//////////////////////////////////////////////////////////////////
// Multiscale fixed size long term rotating log file
// Author: Marijn Weehuizen
//////////////////////////////////////////////////////////////////

#include <time.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <limits.h>

#include "lib.h"

#include "mlog.h"
#define DMID 22
#include "debug.h"

#include "lprintf.h"

int emsg(const char * arg_list, ...);
int imsg(const char * arg_list, ...);
int dmsg(const char * arg_list, ...);
int bug_hunt(char *file, const char *format, ...);

#define MAX(a,b)	((a) > (b) ? (a) : (b))
#define MIN(a,b)	((a) < (b) ? (a) : (b))

int SAMPLE_PERIOD[]={ HOUR_SAMPLE, DAY_SAMPLE, WEEK_SAMPLE, MONTH_SAMPLE, YEAR_SAMPLE };
int SAMPLE_COUNTS[]={ HOUR_COUNT, DAY_COUNT,  WEEK_COUNT,  MONTH_COUNT,  YEAR_COUNT };
int SAMPLE_WINDOW[]={ HOUR_WINDOW, DAY_WINDOW };


char *SAMPLE_TEXT[]={ "HOUR", "DAY", "WEEK", "MONTH", "YEAR" };

int samplemode=MLOG_HOUR; // default sample (mode or MLOG_DAY)
int debug_mlog=FALSE;

MLOG *mlog_new ()
{
	MLOG *pL;
	MLOG_INDEX *pI;
	HISTORY *pd;
	int i;

	pL=malloc(sizeof(MLOG));
	if (pL==NULL){
		return pL;
	}
	memset(pL,0,sizeof(MLOG));

	pL->samplemode = samplemode;
	pL->type = PERF_AVG;
	pL->last1 = pL->data_recent;
	pL->last2 = pL->data_recent+1;
	pd = pL->data;
	for (i=MLOG_HOUR;i<=MLOG_YEAR;i++){
		pI=&pL->index[i];

		pI->pstart = pd;
		pI->pend =  pd + SAMPLE_COUNTS[i]-1;
		pI->precent = pI->pstart;
		pI->poldest = pI->pstart+1;

		pd += SAMPLE_COUNTS[i];
	}
	return pL;
}
int qqq=0;
extern char *g_recent_name;
int mlog_addvalue(MLOG *pL, time_t now, double value, int dodebug)
{
	HISTORY_DETAIL avgvalue={0,0,0,0},*last1,*last2;
	int more=TRUE,td1,td2;
	time_t tlog;
	static time_t last_warn;

	if (pL->samplemode != samplemode){
		imsg("perflog: resampling from %s to %s\n",SAMPLE_TEXT[pL->samplemode],SAMPLE_TEXT[samplemode]);
		mlog_resample(pL, samplemode);
	}

	last1 = pL->last1;
	last2 = pL->last2;

	td1 = (int) (now - last1->time);
	td2 = (int) (now - last2->time);
	
	if(value>1){
//		_asm{int 03}
	}


	tlog = (now / SAMPLE_PERIOD[samplemode]) * SAMPLE_PERIOD[samplemode];
	switch (pL->type){
		case PERF_CTR: // Normalise the between the difs between two sets of points (need three valid points)
			if ( ( td1>(  SAMPLE_PERIOD[samplemode]-SAMPLE_WINDOW[samplemode])  && td1<(  SAMPLE_PERIOD[samplemode]+SAMPLE_WINDOW[samplemode]) ) &&
				 ( td2>(2*SAMPLE_PERIOD[samplemode]-SAMPLE_WINDOW[samplemode])  && td2<(2*SAMPLE_PERIOD[samplemode]+SAMPLE_WINDOW[samplemode]) )){

				// calculate as = part of current delta + part of last delta
				avgvalue.in	= ( (value-last1->in)*(((double)tlog-last1->time)/(now-last1->time))
								+ (last1->in-last2->in)*((last1->time-((double)tlog-SAMPLE_PERIOD[samplemode])))/(last1->time-last2->time) );

				if(avgvalue.in<0){
					bug_hunt("perflog_bughunt.log","perflog: value PERF_CTR unexpectedly <0 %f %d\n", avgvalue.in,42);
					bug_hunt("perflog_bughunt.log","perflog: values %f %f %f %d\n", last2->in,last1->in,value,42);
					//bug_hunt("perflog_bughunt.log","perflog: times  [[%ld]] %ld %ld %ld %d\n", tlog,last2->time,last1->time,now, 42);
					bug_hunt("perflog_bughunt.log","perflog: times %d %d %d %d\n", samplemode, SAMPLE_PERIOD[samplemode], SAMPLE_WINDOW[samplemode],42);
					bug_hunt("perflog_bughunt.log","perflog: NEGATIVE value (%f) set to ZERO\n", avgvalue.in);
					avgvalue.in=0;
				}
				avgvalue.time = tlog;
			}
			break;
		case PERF_AVG: // Just normalise between two points
			if ( td1>(SAMPLE_PERIOD[samplemode]-SAMPLE_WINDOW[samplemode])  && td1<(SAMPLE_PERIOD[samplemode]+SAMPLE_WINDOW[samplemode]) ){
				// calculate as = part of current delta + part of last delta
				avgvalue.in	= ( last1->in + (value-last1->in)*(((double)tlog-last1->time)/(now-last1->time)));
				avgvalue.time = tlog;
			}
			if(avgvalue.in<0){
				bug_hunt("perflog_bughunt.log","perflog: value PERF_AVG unexpectedly <0 %f %d\n", avgvalue.in,42);
				bug_hunt("perflog_bughunt.log","perflog: values %f %f %d\n", last1->in,value,42);
				bug_hunt("perflog_bughunt.log","perflog: times  [[%ld]] %ld %ld %d\n", tlog,last1->time,now, 42);
				bug_hunt("perflog_bughunt.log","perflog: times %d %d %d %d\n", samplemode, SAMPLE_PERIOD[samplemode], SAMPLE_WINDOW[samplemode],42);
				bug_hunt("perflog_bughunt.log","perflog: NEGATIVE value (%f) set to ZERO\n", avgvalue.in);
				avgvalue.in=0;
			}
			break;
		default:
		case PERF_INSTANT:	// Log the latest value as it is pointless averaging instantaneous vlaues
			avgvalue.in	= value;
			if(avgvalue.in<0){
				bug_hunt("perflog_bughunt.log","perflog: value PERF_INSTANT unexpectedly <0 %f %d\n", avgvalue.in,42);
				avgvalue.in=0;
			}
			avgvalue.time = tlog;
			break;

	}

//	if(pL->type==PERF_CTR&&(avgvalue.in>10000000||avgvalue.in<0)) bug_hunt("perflog_bughunt.log","perflog: ready to log %s %f %d\n", g_recent_name,avgvalue.in,42);

	*last2 = *last1;
	last1->time = now;
	last1->in = value;
	last1->inmax = value;

	if (avgvalue.time==0) {
		if(time(NULL)-last_warn>10){
			dmsg("perflog: data point(s) logged but log(s) not updated (%s)\n", last2->time==0 ? "not enough data collected yet" : "last sample was too long ago");
			bug_hunt("perflog_bughunt.log","perflog: data point(s) logged but log(s) not updated (%s)\n", last2->time==0 ? "not enough data collected yet" : "last sample was too long ago");
		}
		last_warn=time(NULL);
		return FALSE;
	}

	mlog_addvalue_core(pL,samplemode,&avgvalue,dodebug);

	return TRUE;
}

// ***** begin of mlog circular buffer abstraction functions - iterator, point addition, point removal, etc

typedef struct _pl_iter {
	MLOG_INDEX *pI;
	HISTORY *p;
	int old2new;
} pl_iter;

enum {PL_OLD2NEW,PL_NEW2OLD};
pl_iter *pl_iter_new()								// New log file section "iterator" 
{
	pl_iter* pli;
	pli = malloc(sizeof(pl_iter));
	if (pli==NULL){
		return NULL;
	}
	memset(pli, 0, sizeof(pl_iter));
	return pli;
}
void pl_iter_free(pl_iter *pli)						// Free the iterator
{
	if (pli){
		free(pli);
		pli=NULL;
	}
}
HISTORY *pl_iter_init(pl_iter *pli,MLOG_INDEX *pl, int order)			// New log file section "iterator" 
{
	if (pli==NULL) return NULL;
	pli->pI = pl;
	if (order==PL_OLD2NEW){
		pli->p = pl->poldest;
		pli->old2new= TRUE;
	}else{
		pli->p = pl->precent;
		pli->old2new = FALSE;
	}
	return pli->p;
}
void pl_iter_setdirn(pl_iter *pli, int order)
{
	if (order==PL_OLD2NEW){
		pli->old2new= TRUE;
	}else{
		pli->old2new = FALSE;
	}
}
HISTORY *pl_iter_cur(pl_iter *pli)					// Select value currently pointed to by iterator
{
	if (pli->old2new){
		if (pli->p == pli->pI->precent)
			return NULL;
	}else{
		if (pli->p == pli->pI->poldest)
			return NULL;
	}
	return pli->p;
}
HISTORY *pl_iter_next(pl_iter *pli)					// Iterate to next value
{
	if ( pli->p > pli->pI->pend || pli->p < pli->pI->pstart ) {
		emsg("perflog: invalid iterator\n");
		lib_abort();
	}
	if (pli->old2new){
		if (pli->p == pli->pI->precent)
			return NULL;
		pli->p--;
		if (pli->p < pli->pI->pstart)
			pli->p = pli->pI->pend;
	}else{
		if (pli->p == pli->pI->poldest)
			return NULL;
		pli->p++;
		if (pli->p > pli->pI->pend)
			pli->p = pli->pI->pstart;
	}
	return pli->p;
}
HISTORY *pl_iter_next_multi(pl_iter *pli,MLOG *pL,int *curscale)
{
	HISTORY *cur;
	if (pli->old2new) return NULL; // Sorry only support PL_NEW2OLD
	cur=pl_iter_next(pli);
	if (cur==NULL && *curscale<MLOG_YEAR){
		(*curscale)++;
		cur = pl_iter_init(pli,&pL->index[*curscale],PL_NEW2OLD);
	}
	return cur;
}
int pl_space_left(MLOG_INDEX *pl)			// Number of free slots left
{
	int free = (int) (pl->precent - pl->poldest - 1);
	if (free<0) 
		free += (int) (pl->pend - pl->pstart + 1);
	return free;
}
int pl_space_used(MLOG_INDEX *pl)			// Number of slots used
{
	int used = (int)(pl->poldest - pl->precent + 1);
	if (used<=0) 
		used += (int)(pl->pend - pl->pstart + 1);
	return used;
}

HISTORY *pl_del_oldest(MLOG_INDEX *pl)		// Delete oldest value
{
	HISTORY *invalid=pl->poldest;
	pl->poldest--;
	if (pl->poldest<pl->pstart) 
		pl->poldest=pl->pend;
	return invalid;
}
HISTORY *pl_del_recent(MLOG_INDEX *pl)		// Delete newest value
{
	HISTORY *invalid=pl->precent;
	pl->precent++;
	if (pl->precent>pl->pend) 
		pl->precent=pl->pstart;
	return invalid;
}

void pl_add_value(MLOG_INDEX *pl, HISTORY_DETAIL *value)	// Add a value to a free slot
{
	if ( pl_space_left(pl)==0 ) {
		emsg("perflog: no free space to add a value, correcting and continuing\n");
		pl_del_oldest(pl);
	}
	
	pl->precent--;
	if (pl->precent < pl->pstart)
		pl->precent = pl->pend;

	pl->precent->time = value->time;
	pl->precent->percent = 0;
	pl->precent->in = /*(unsigned long)*/ TRUNC_ULONG(value->in);
	pl->precent->out = /*(unsigned long)*/ TRUNC_ULONG(value->out);
	pl->precent->inmax = /*(unsigned long)*/ TRUNC_ULONG(value->inmax);
	pl->precent->outmax = /*(unsigned long)*/ TRUNC_ULONG(value->outmax);
}

void verify_pl_spacefn(MLOG_INDEX *pl, int scale)
{
	if (pl_space_left(pl) + pl_space_used(pl) != SAMPLE_COUNTS[scale]){
		emsg("perflog: space_used + free_space does not match expected samples (free:%d used:%d expected:%d)\n",
				pl_space_left(pl), pl_space_used(pl), SAMPLE_COUNTS[scale] );
		lib_abort();
	}
}

// ***** end of mlog circular buffer abstraction functions - iterator, point addition, point removal, etc


// Reentrant routine to add data to one or more levels of a log file
int mlog_addvalue_core(MLOG *pL, int scale, HISTORY_DETAIL *newvalue, int dodebug)
{
	HISTORY_DETAIL avgvalue;
	HISTORY *cur,*removed;
	MLOG_INDEX *pl;
	pl_iter *p;
	int nextneed;
	int ngot=0;
	double tot=0;
	unsigned long max=0;
	time_t timestart=0,timeend=0;
	char bf[BFSZ];

	pl = &pL->index[scale];

// Remove once fully tested	
//verify_pl_spacefn(pl,scale);
//	if(pL->type==PERF_CTR&&newvalue->in>10000000) bug_hunt("perflog_bughunt.log","perflog: add log %s SCALE=%s %f %d\n", g_recent_name,SAMPLE_TEXT[scale],newvalue->in,42);

	pl_add_value(pl,newvalue);

// Remove once fully tested	
//imsg("perflog: %s data point added %s %d to scale %s. Points free %d used %d\n",SAMPLE_TEXT[scale],lib_mail_time(bf,newvalue->time),(unsigned long)newvalue->in,SAMPLE_TEXT[scale],pl_space_left(pl),pl_space_used(pl));
//verify_pl_spacefn(pl,scale);

	// Set values for debug graphs
	if (pL->debugfill && dodebug){
		if (pL->debugfill & DEBUG_STEADY)	{pl->precent->in = 42;}
		if (pL->debugfill & DEBUG_RAMP)		{
			pl->precent->in = ((pL->debugfill & DEBUG_COUNTER)>>8) + 1;
			pL->debugfill = (pL->debugfill & DEBUG_TYPE) | ((long)(pl->precent->in)<<8);
		}
	}
	
	if (pl_space_left(pl)>0) return TRUE;
	// No need to average year samples
	if (scale==MLOG_YEAR){
		pl_del_oldest(pl);
		return TRUE;
	}
	nextneed = SAMPLE_PERIOD[scale+1]/SAMPLE_PERIOD[scale];

	memset(&avgvalue,0,sizeof(HISTORY_DETAIL));
	p = pl_iter_new();
	if (p==NULL) return FALSE;
	cur = pl_iter_init(p,pl,PL_OLD2NEW);
	timestart = (cur->time/SAMPLE_PERIOD[scale+1]) * SAMPLE_PERIOD[scale+1];
	timeend = timestart+SAMPLE_PERIOD[scale+1];
	for (;cur!=NULL && nextneed>0;cur=pl_iter_next(p),nextneed--) {
		if (cur->time<timestart) {
			imsg("perflog: backwards timestep detected (%dsec) adding summary of %d points (instead of %d) to %s scale [%s]\n",
					timeend-cur->time, ngot, ngot+nextneed, SAMPLE_TEXT[scale+1],lib_mail_time(bf,timeend));
//			bug_hunt("perflog_bughunt.log","perflog: backwards timestep detected (%dsec) adding summary of %d points (instead of %d) to %s scale [%s]\n",
//					timeend-cur->time, ngot, ngot+nextneed, SAMPLE_TEXT[scale+1],lib_mail_time(bf,timeend));
			break;
		}
		if (cur->time>timeend) {
			if (cur->time-timeend > SAMPLE_PERIOD[scale]*2)
				imsg("perflog: forwards timestep detected (%dsec) adding summary of %d points (instead of %d) to %s scale [%s]\n",
					cur->time-timeend, ngot, ngot+nextneed, SAMPLE_TEXT[scale+1],lib_mail_time(bf,timeend));
//				bug_hunt("perflog_bughunt.log","perflog: forwards timestep detected (%dsec) adding summary of %d points (instead of %d) to %s scale [%s]\n",
//					cur->time-timeend, ngot, ngot+nextneed, SAMPLE_TEXT[scale+1],lib_mail_time(bf,timeend));
			break;
		}
		removed=pl_del_oldest(pl);
// Remove once fully tested	
//		if (cur!=removed) {
//			emsg("perflog: point removed is not the same as the one we are working on!!!\n");
//			lib_abort();
//		}
// Remove once fully tested	
//dmsg("perflog: %s adding summary point %s %d to scale %s.\n",SAMPLE_TEXT[scale],lib_mail_time(bf,cur->time),cur->in,SAMPLE_TEXT[scale]);

		ngot++;
		tot += cur->in;

		switch(scale){	// Only log max for summay of samples of 5 minutes or greater
			case MLOG_HOUR:		max=0;						break;
			case MLOG_DAY:		max = MAX(max, cur->in);	break;
			default:			max = MAX(max, cur->inmax);	break;
		}
	}
	pl_iter_free(p);

	if (ngot>0) {
		avgvalue.time = timeend;
		avgvalue.in = (long)( tot / ngot);
//		if(pL->type==PERF_CTR&&avgvalue.in>10000000) bug_hunt("perflog_bughunt.log","perflog: recalc %s SCALE=%s %f tot=%f got=%d %d\n", g_recent_name,SAMPLE_TEXT[scale],avgvalue.in,tot,ngot,42);
		avgvalue.inmax = max ;
		mlog_addvalue_core(pL, scale+1, &avgvalue, FALSE);
	}

// Remove once fully tested	
//dmsg("perflog: %s after summarise action %s %d to scale %s. Points free %d used %d\n",SAMPLE_TEXT[scale],lib_mail_time(bf,newvalue->time),(unsigned long)newvalue->in,SAMPLE_TEXT[scale],pl_space_left(pl),pl_space_used(pl));

	return TRUE;
}

char *double_limit(double value, char *bf, int bfsz, int limit);
int mlog_fillgraphbufer(MLOG *pL, int graphtime, int graphscale, HISTORY*buf, int xpix, int ypix, double yscale, int cutoff)
{
	HISTORY *cur=NULL;
	double avg_in, value;
	unsigned long max;
	pl_iter *p;
	int ntot,sampletime,curscale,x,backfill=0,missing=0;
	int timestart=0,timeend=0;
//	char bf[BFSZ],bf2[BFSZ-1];
	
	p = pl_iter_new();
	if (p==NULL) return FALSE;

	curscale = samplemode;
	timestart = (graphtime/SAMPLE_PERIOD[graphscale]) * SAMPLE_PERIOD[graphscale];
	timeend = timestart+SAMPLE_PERIOD[graphscale];
	cur = pl_iter_init(p,&pL->index[curscale],PL_NEW2OLD);

	// traverse graph pixels starting at rightmost pixel, working back in time
	for(x=0;x<xpix;x++){
		
		ntot=0;
		avg_in=0;
		sampletime=0;
		max=0;

		// Get 0 or more points from the graph matching time window changing scales if needed
		for (;cur && cur->time>timestart;cur=pl_iter_next_multi(p,pL,&curscale)) {
			if (cur->time>timeend) {
// Remove once fully tested	
//imsg("perflog: %s ignoring point %s %d to graph point %d\n",SAMPLE_TEXT[curscale],lib_mail_time(bf,cur->time),cur->in,x);
				continue;
			}
			value = (double)cur->in*SAMPLE_PERIOD[curscale];

			if (cutoff>0 && cur->in>cutoff){
				continue;
			}

			// Add single point to total
			ntot++;
			avg_in += value;
			

//			if(avg_in>10000000){
//				bug_hunt("perflog_bughunt.log","perflog: Pre High!! %d %d %d %d\n", curscale, SAMPLE_PERIOD[curscale], cur->in, 42);
//				bug_hunt("perflog_bughunt.log","perflog: Pre High!! *2 %f %d\n", avg_in,42);
//			} 
			
			sampletime += SAMPLE_PERIOD[curscale];
			if (graphscale>MLOG_DAY && curscale>=MLOG_DAY) {
				switch(curscale){	// Only log max for summary of samples of 5 minutes or greater
					case MLOG_HOUR:		max=0;						break;
					case MLOG_DAY:		max = MAX(max, cur->in);	break;
					default:			max = MAX(max, cur->inmax);	break;
				}
			}
// Remove once fully tested	
//imsg("perflog: %s adding point %s %d to graph point %d\n",SAMPLE_TEXT[curscale],lib_mail_time(bf,cur->time),cur->in,x);
		}

// No backfilling required anymore
//		// Try backfill the last point on a scale if we have no points
//		if (ntot==0 && x==xpix-1 && backfill){
//			cur=pl_iter_next_multi(p,pL,&curscale);
//			if (cur) {
//				avg_in = (double)cur->in*SAMPLE_PERIOD[curscale];
//				sampletime = SAMPLE_PERIOD[curscale];
//				ntot=1;
//			}
//		}

		// Set graph point (handling backfilling of missing data points)
		if (ntot==0 || sampletime==0) {
			buf[x].in = 0;		
			buf[x].inmax = 0;
			buf[x].out = 0;
			buf[x].outmax = 0;
			missing++;
			backfill++;
		} else {
			buf[x].in = /*(long)*/ TRUNC_ULONG(avg_in/sampletime/yscale);		
			buf[x].inmax = /*(long)*/ TRUNC_ULONG(max/yscale);
			
//			if(pL->type==PERF_CTR&&buf[x].in>10000000){
//				char tmp1[BFSZ],tmp2[BFSZ];
//				bug_hunt("perflog_bughunt.log","perflog: graph point high!! %d %d %s %s %d %d\n", ntot, sampletime, double_limit(yscale,tmp1,BFSZ,9), double_limit(avg_in,tmp2,BFSZ,9), buf[x].in, 42);
//			}
			buf[x].out = 0;
			buf[x].outmax = 0;
			if (backfill>0 && backfill<x) {
				while(backfill>0){
//					buf[x-backfill].in=buf[x].in;
					buf[x-backfill].in=0;				// Backfilling to zero is more appropriate
					backfill--;
				}
			}
		}
// Remove once fully tested	
//imsg("perflog: graph point set %s to %s pont %d value %d [%d]\n", lib_mail_time(bf,timestart),lib_mail_time(bf2,timeend),x, buf[x].in, ntot);

		// Move on to the next point
		timestart -= SAMPLE_PERIOD[graphscale];
		timeend -= SAMPLE_PERIOD[graphscale];
	}
	pl_iter_free(p);
	if (missing>0){ // Changed to dmsg by chrisp as annoyed a user, hope it's not important.
		dmsg("perflog: %d of %d graph samples have no data (this is ok, previous values used)\n",missing,xpix);
	}
	return TRUE;
}


int mlog_setsamplemode(int mode)
{
	if (samplemode==mode) return TRUE;
	samplemode=mode;
	return TRUE;
}
int mlog_getsampleperiod()
{
	return SAMPLE_PERIOD[samplemode];
}

int mlog_getsamplemode()
{
	return samplemode;
}

void mlog_setdebug(int debug)
{
	debug_mlog=debug;
}

int mlog_resample(MLOG *pL, int mode)
{
	int ngot;
	time_t t;
	double tot;
	MLOG_INDEX *pl;
	pl_iter *p;
	HISTORY_DETAIL avgvalue;
	HISTORY *cur=NULL,*p2,*removed;
	time_t timestart=0,timeend=0;

	if (pL->samplemode==mode) return TRUE;
	
	p = pl_iter_new();
	if (p==NULL) return FALSE;

	if (mode==MLOG_DAY){
		// grab all highres data points and add them to low res data points
		cur = pl_iter_init(p,&pL->index[MLOG_HOUR],PL_OLD2NEW);
		for(;cur!=NULL;cur=pl_iter_next(p)){
			tot=ngot=0;
			timestart = (cur->time/SAMPLE_PERIOD[MLOG_DAY]*SAMPLE_PERIOD[MLOG_DAY]);
			timeend =  (timestart+SAMPLE_PERIOD[MLOG_DAY]);
			while (cur!=NULL && cur->time>timestart && cur->time<=timeend){
				tot+=cur->in;
				ngot++;
				cur=pl_iter_next(p);
			}
			avgvalue.in = (long)tot/ngot;
			avgvalue.time = timeend;
			mlog_addvalue_core(pL,MLOG_DAY,&avgvalue,FALSE);
		}
	}
	if (mode==MLOG_HOUR){
		// Move single low res data point to start of high res data points
		cur = pl_iter_init(p,&pL->index[MLOG_DAY],PL_NEW2OLD);
		pl=&pL->index[MLOG_HOUR];
		p2 = pl->precent = pl->pstart;
		for(t=cur->time; t>cur->time-SAMPLE_PERIOD[MLOG_DAY]; t-=SAMPLE_PERIOD[MLOG_HOUR],p2++){
			p2->in = cur->in;
			p2->inmax = 0;
			p2->time = t;
			pl->poldest=p2;
		}		
		removed=pl_del_recent(&pL->index[MLOG_DAY]);
		if (cur!=removed) {
			emsg("perflog: point removed is not the same as the one we are working on!!!\n");
			lib_abort();
		}
	}
	pL->samplemode=mode;
	pl_iter_free(p);

	return TRUE;
}

int	mlog_getscale(char* type)
{
	int scale;
	for(scale=MLOG_YEAR;scale>samplemode;scale--){
		if (strcmpnc(type,SAMPLE_TEXT[scale])==0) break;
	}
	return scale;
}

int bug_hunt(char *file, const char *format, ...);

// Load file (with blank initialisation if it does not exist)
int mlog_load (MLOG **hL, char* file, int debugfill, int type)
{
	FILE *fi;
	HISTORY *p=NULL;
	char bf[BFSZ];
	int nolog=FALSE;
	int i,scale,firstboundary=TRUE,points, sample;
	long value,max;
	time_t t,last_t,tnext;
	long long longt;
	MLOG *pL;
	int debug;

	pL=mlog_new();
	if (pL==NULL){
		emsg("perflog: could not allocate memory for log file\n");
		*hL=NULL;
		return FALSE;
	}
	pL->debugfill = debugfill;
	pL->type = type;

	*hL=pL;
	debug=0;
	if ((fi = fopen(file,"r")) != NULL) {
		imsg("perflog: loading log file (%s)\n", file);
		scale=samplemode;
		fgets(bf,BFSZ,fi);
		points=0;
		last_t=0;
		while(!feof(fi)) {
			// Deal with scale changes
			if (bf[0]=='#' && scale<MLOG_YEAR){
				if (points>SAMPLE_COUNTS[scale]){
					emsg("perflog: too many data points in %s section of file (%d, max %d), has been corrected (once only correction)\n", SAMPLE_TEXT[scale], points, SAMPLE_COUNTS[scale]);
				}
				points=0;
				last_t=0;
				if (firstboundary){
					if (strncmp(bf+2,SAMPLE_TEXT[MLOG_HOUR],lib_strlen(SAMPLE_TEXT[MLOG_HOUR]))==0){
						pL->samplemode = MLOG_HOUR;
					}else{
						pL->samplemode = MLOG_DAY;
					}
					scale=pL->samplemode;
					p=pL->index[scale].pstart;
					firstboundary=FALSE;
				} else {
					scale++;
					p=pL->index[scale].pstart;
				}
				fgets(bf,BFSZ,fi);
				continue;
			}
			if (!sscanf(bf,"%lld %ld %ld\n",&longt,&value,&max)){
				emsg("perflog: error parsing file in %s \n",file);
			}
			t = (time_t) longt;
			if (p==NULL) {
				fclose(fi);
				emsg("perflog: no valid data (1) in file %s \n",file);		
				goto file_bad;
			}
			if (points<SAMPLE_COUNTS[scale]-1 && t!=last_t){
				p->time = t;
				p->in = value;
				p->inmax = max;
				pL->index[scale].poldest=p;

				p++;
			}
			last_t=t;
			points++;
			t-=SAMPLE_PERIOD[scale];
			fgets(bf,BFSZ,fi);
		}
		fclose(fi);

		if (scale!=MLOG_YEAR || points==0) {
			sprintf(bf,"%s_corrupt",file);
			remove(bf);
			rename(file,bf);
			emsg("perflog: corrupt/missing data (2) in file %s, reinitialising, renamed original file to %s \n",file,bf);
			goto file_bad;
		}

		// Resample the most recent data points if the file sampling mode does not match the current mode
		if (pL->samplemode!=samplemode) {
			mlog_resample(pL,samplemode);
		}
	}else{
file_bad:
		imsg("perflog: no log file initialising blank log (%s)\n", file);
		t = ( (time(NULL)/SAMPLE_PERIOD[samplemode]) * SAMPLE_PERIOD[samplemode]);
		for (i=samplemode;i<=MLOG_YEAR;i++){
			sample=1;
			if (i==MLOG_YEAR)
				tnext = (t - (SAMPLE_COUNTS[i]-10)*SAMPLE_PERIOD[i]);
			else
				tnext = ( (t - (SAMPLE_COUNTS[i])*SAMPLE_PERIOD[i]) / SAMPLE_PERIOD[i+1]) * SAMPLE_PERIOD[i+1] + (SAMPLE_PERIOD[i+1]);
			for (p=pL->index[i].pstart; /*p!=next_sn*/t>tnext && sample<=SAMPLE_COUNTS[i]-1; p++, sample++){
				p->time=t;
				p->in = p->inmax = 0;
				t -= SAMPLE_PERIOD[i];
				// For testing
				if (pL->debugfill & DEBUG_FILL){
					if (pL->debugfill & DEBUG_STEADY)	{p->in=42; p->inmax=(long)(1.2*p->in);}
					if (pL->debugfill & DEBUG_RAMP)		{p->in=debug;p->inmax=(long)(1.2*debug); debug++;}
				}
			}
			pL->index[i].poldest=p-1;
		}
	}

// Remove once fully tested	
//	for (i=pL->samplemode;i<=MLOG_YEAR;i++){
//		dmsg("perflog: %s space_used + free_space matches samples (free:%d used:%d expected:%d)\n",	
//				SAMPLE_TEXT[i],pl_space_left(&pL->index[i]), pl_space_used(&pL->index[i]), SAMPLE_COUNTS[i] );
//	}

	return TRUE;
}

// Write log file
int mlog_save (MLOG *pL, char* file)
{
	FILE *fo;
	char oldfile[BFSZ],newfile[BFSZ];
	int i,sanitycheck;
	HISTORY*cur;
	MLOG_INDEX *pl;
	pl_iter *p;

	sprintf(newfile, "%s_tmp", file);
	sprintf(oldfile, "%s_old", file);

	if ((fo = fopen(newfile,"w")) == NULL) {
		emsg("perflog: could not open file\n");
		return FALSE;
	}

	for (i=samplemode;i<=MLOG_YEAR;i++){
		fprintf(fo,"# %s samples\n", SAMPLE_TEXT[i] );

		pl = &pL->index[i];
		sanitycheck=0;
		p = pl_iter_new();
		if (p==NULL) break;
		cur = pl_iter_init(p,pl,PL_NEW2OLD);
		for (;cur!=NULL;cur=pl_iter_next(p)) {
		
			fprintf(fo,"%d %lu %lu\n", (int) cur->time, (unsigned long)cur->in, (unsigned long) cur->inmax );

			if (sanitycheck++>1000) {
				emsg("perflog: mlog data structures seems to be corrupt while saving %s (SERIOUS FAULT(2) - log file is now probably invalid) scale:%s points found:%d \n",file,SAMPLE_TEXT[i],sanitycheck);
				break;
			}
		} 
		pl_iter_free(p);
// Remove once fully tested	
//dmsg("perflog: saved %d data points\n",sanitycheck);
	}
	fclose(fo);

	rename(file,oldfile);
	rename(newfile,file);
	remove(oldfile);

	return TRUE;
}

unsigned long mlog_lastvalue(MLOG *pL)
{
	return (unsigned long) pL->last1->in;
}


// Mlog debugging functions

void mlog_writevalue_detail(MLOG *pL, FILE* fo,char* s, HISTORY_DETAIL *p, int scale)
{
	char bf[BFSZ];
	fprintf(fo,"    %s=  value=%f time=[%s]\n",s,p->in, lib_mail_time(bf,p->time) );
	return;
}

void mlog_writevalue(MLOG *pL, FILE* fo,char* s, HISTORY *p, int scale)
{
	char bf[BFSZ];
	char xtmp[BFSZ];
	MLOG_INDEX *index;

	if (scale==-1){
		fprintf(fo,"    %s=  value=%d time=[%s]\n",s,(int) p->in, lib_mail_time(bf,p->time) );
		return;
	}

	index = &pL->index[scale];
	if (p > index->pend)	p -= SAMPLE_COUNTS[scale];
	if (p < index->pstart)	p += SAMPLE_COUNTS[scale];

	fprintf(fo,"    %s=%3d value=%4d time=[%s] (%s)\n",s, (int) (p - index->pstart),(int)  p->in, lib_mail_time(bf,p->time),time_to_str(p->time,xtmp));
}

int mlog_debugsave(MLOG *pL, char* file)
{
	FILE *fo;
	int scale,i;
	MLOG_INDEX *pl;
	HISTORY *cur;
	char bf[BFSZ];
	pl_iter *p;

	remove("perflog_old.txt");
	rename("perflog.txt","perflog_old.txt");

	if ((fo = fopen(file,"w")) == NULL) {
		return FALSE;
	}

	mlog_writevalue_detail(pL, fo, "last-1  ", pL->last1, -1);
	mlog_writevalue_detail(pL, fo, "last-2  ", pL->last2, -1);

	p = pl_iter_new();
	for (scale=samplemode; scale<=MLOG_YEAR; scale++){
		pl = &pL->index[scale];

		fprintf(fo,"\nSection: %s \n", SAMPLE_TEXT[scale]);

		fprintf(fo,"    start=%d recent=%d oldest=%d end=%d \n",
					(int)(pl->pstart - pl->pstart), (int)(pl->precent - pl->pstart),
					(int)(pl->poldest - pl->pstart),(int)( pl->pend - pl->pstart) );
		mlog_writevalue(pL, fo, "recent   ", pl->precent, scale);

		if (p==NULL) break;
		cur = pl_iter_init(p,pl,PL_NEW2OLD);
		for(i=1;i<15;i++){
			cur=pl_iter_next(p);
			sprintf(bf,"recent%c%2d",'+',i);
			mlog_writevalue(pL, fo, bf, cur, scale);
		}

		fprintf(fo,"\n");

		cur = pl_iter_init(p,pl,PL_OLD2NEW);
		for(i=0;i<15;i++){
			cur=pl_iter_next(p);
		}
		pl_iter_setdirn(p,PL_NEW2OLD);
		for(i=14;i>=1;i--){
			cur=pl_iter_next(p);
			sprintf(bf,"oldest%c%2d",'-',i);
			mlog_writevalue(pL, fo, bf, cur, scale);
		}
		mlog_writevalue(pL, fo, "oldest   ", pl->poldest, scale);
	}
	pl_iter_free(p);

	fclose(fo);
	return TRUE;
}
void mlog_verify_p(HISTORY *p, MLOG_INDEX *pI, char* location)
{
	int i = sizeof(HISTORY);

	if (p>pI->pend || p<pI->pstart || (pI->pend-pI->pstart>700) ){
		emsg("PERFLOG FAULT - p not valid anymore [%s]\n",location);
		emsg("perflog: X1 precent=%d oldest=%d  [p=%ld :: pstart=%ld precent=%ld poldest=%ld plast=%ld]\n",
			pI->precent-pI->pstart, pI->poldest-pI->pstart,
			p,
			pI->pstart, pI->precent, pI->poldest, pI->pend);
		lib_abort();
	}
}

#ifdef _MLOG_TEST_
int mlog_test()
{
	int i,t,v,i1,i2,samplemode;
	MLOG *pL;

	mlog_load(&pL,"mylogfile.log",&samplemode);
	mlog_debugsave(pL,"perflog.txt");
	mlog_save(pL,"perflog.log");

	t=time(NULL);
	v=20000;
	i=i1=i2=1;
	while(TRUE){
		mlog_addvalue(pL,t,v);
		mlog_debugsave(pL, "perflog.txt");
		mlog_save (pL, "perflog.log");

		if (i1%6==0){
			if (i2%4==0){
				i=i;
			}
			i2++;
		}
		lib_msleep (200);
		t+=300;v+=1;i++;i1++;
	}
	return TRUE;
}

#endif

int mlog_reset_invalid_data(MLOG *pL, time_t reset_start, time_t reset_end, int *changes)
{
	int i,count=0;
	MLOG_INDEX *pI;
	HISTORY *cur;
	pl_iter *p;
	
	p = pl_iter_new();
	for (i=MLOG_YEAR;i>=MLOG_HOUR;i--){
		pI=&pL->index[i];
		cur = pl_iter_init(p,pI,PL_OLD2NEW);
		for(;cur!=NULL;cur=pl_iter_next(p)){
			if(cur->time >= reset_start && cur->time <= reset_end){
				cur->in=0;
				cur->inmax=0;
				cur->out=0;
				cur->outmax=0;
				count++;
			}
		}
	}
	pl_iter_free(p);
	*changes=count;
	return TRUE;
}
int mlog_locate_invalid_data(MLOG *pL, unsigned long limit, time_t *invalid_start, time_t *invalid_end, int *changes)
{
	int i,first=TRUE,count=0;
	MLOG_INDEX *pI;
	HISTORY *cur;
	pl_iter *p;
	
	p = pl_iter_new();
	for (i=MLOG_YEAR;i>=MLOG_HOUR;i--){
		pI=&pL->index[i];
		cur = pl_iter_init(p,pI,PL_OLD2NEW);
		for(;cur!=NULL;cur=pl_iter_next(p)){
			if(cur->in>limit || cur->inmax>limit || cur->out>limit || cur->outmax>limit){
				if(first){
					*invalid_start=cur->time;
					first=FALSE;
				}
				*invalid_end=cur->time;
				count++;
			}
		}
	}
	pl_iter_free(p);
	*changes=count;
	return TRUE;
}
// Bad limit defined to catch points of value = 2.1GB / 20 data points per average (hour->day) = 107M
// Any rate counter that has over 100 million going through it a second is probably bad.
// Although possible exception is bandwidth_* counters. We can always disable this for them once this issue
// is resolved an iuf someone complains about it getting limited!!
#define BAD_LIMIT 100000000
int mlog_validate_data2(MLOG *pL, time_t reset_start, time_t reset_end, int *changes)
{
	int i,count=0;
	MLOG_INDEX *pI;
	HISTORY *cur;
	pl_iter *p;
	
	p = pl_iter_new();
	for (i=MLOG_YEAR;i>=MLOG_HOUR;i--){
		pI=&pL->index[i];
		cur = pl_iter_init(p,pI,PL_OLD2NEW);
		for(;cur!=NULL;cur=pl_iter_next(p)){
			if(cur->time >= reset_start && cur->time <= reset_end){
				if(cur->in>BAD_LIMIT || cur->inmax>BAD_LIMIT || cur->out>BAD_LIMIT || cur->outmax>BAD_LIMIT){
					cur->in=0;
					cur->inmax=0;
					cur->out=0;
					cur->outmax=0;
					count++;
				}
			}
		}
	}
	pl_iter_free(p);
	*changes=count;
	return TRUE;
}

int mlog_rescale_data2(MLOG *pL, double rescale, int *changes)
{
	int i,count=0;
	MLOG_INDEX *pI;
	HISTORY *cur;
	pl_iter *p;
	
	p = pl_iter_new();
	for (i=MLOG_YEAR;i>=MLOG_HOUR;i--){
		pI=&pL->index[i];
		cur = pl_iter_init(p,pI,PL_OLD2NEW);
		for(;cur!=NULL;cur=pl_iter_next(p)){
			cur->in *= rescale;
			cur->inmax *= rescale;
			cur->out *= rescale;
			cur->outmax *= rescale;
			count++;
		}
	}
	pl_iter_free(p);
	*changes=count;
	return TRUE;
}
