/*
simple_malloc is the default fast one...

use_debug.txt would make it use the padding one.
use_dmalloc.txt makes it use the old one.


*/

#include <stdlib.h>
#include <ctype.h>
#include <stdarg.h>
#include <string.h>
#include <stdio.h>
#include <time.h>
#ifdef WIN32
#include <malloc.h>
#endif
//#include "file64.h"

#include "lib.h"

#include "thread.h"
//#include "lstat.h"
#ifdef USE_NED
#include "nedmalloc.h"
#endif

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <memory.h>
#include "isofix.h"
//#include "bigint.h"

#ifdef CRTDBG
#include <crtdbg.h>
#endif

// So we don't get malloc/free over-ridden
#define IS_DMALLOC_C 1
#include "dmalloc.h"
#undef malloc
#undef free
#undef calloc
#undef realloc
#undef strdup



// unix debugging malloc over-ride.
#ifdef UDMALLOC
#include <dmalloc.h>
#endif

#ifdef CRTCHRISP // COMPILED IN RELEASEDB BUILD... 
#ifdef WIN32
#ifdef _DEBUG

#include <malloc.h>
#include <crtdbg.h>
#define malloc(a) _malloc_dbg(a,_CLIENT_BLOCK,__FILE__, __LINE__ )
#define calloc(a,b) _calloc_dbg(a,b,_CLIENT_BLOCK,__FILE__, __LINE__ )
#define realloc(a,b) _realloc_dbg(a,b,_CLIENT_BLOCK,__FILE__, __LINE__ )
#define free(a) _free_dbg(a,_CLIENT_BLOCK)
#endif
#endif 
#endif

typedef struct MBLOCK {void **data; struct MBLOCK *next; struct MBLOCK *last; int size; char *file; int line; char k; char id; char isbig; } Block;
typedef struct BLOCKLIST {Block *used; Block *free; int nused; int nfree; int sz;  } Blocklist;

#ifndef TRUE
#define FALSE 0
#define TRUE (!FALSE)
#endif
static int use_smalloc;
void nofrag_insert(Block **first, Block *b);
void nofrag_remove(Block **first, Block *b);
static void (*abort_fn)(int x);
//static int free_check=FALSE; // SEt to true to check free'd memory isn't modified.
// Chrisp changing this to find memory bug... Monday ,1 Feb 2010
static int free_check=TRUE; // SEt to true to check free'd memory isn't modified., do not set to false, it is set to false by function
// if yo uset it to false and the functino sets it true, then it will crash because memory has hnot been nicely cleared...
//void hits_add(char *file, int line);
#ifdef DO_THREADS
//Mutex *th_mutex_new(char *name);
static Mutex *m;
#endif
#ifdef USE_NED
static int use_simple=FALSE;
static int use_ned=FALSE; // both should be true in future.
#else
static int use_ned=FALSE;
static int use_simple=FALSE; // this should be true in future. 
#endif

static int freepc=0; // % of frees to keep track of...
static int padpc=0; 
int simple_padsz(int b);
static int npad;

void *(*psimple_malloc)(int n,char *src, int line) = simple_malloc;
void *(*psimple_realloc)(void *x, int n,char *src, int line) = simple_realloc;
void *(*psimple_calloc)(int num, int size,char *src, int line) = simple_calloc;
char *(*psimple_strdup)(const char *s,char *src, int line) = simple_strdup;
void (*psimple_free)(void *x, char *src, int line) = simple_free;

void *(*psimple_mallocid)(int dmid, int n,char *src, int line);
void *(*psimple_callocid)(int dmid,int num, int size,char *src, int line);
int pad_freelist(int *xi);
void track_init(void);
void track_wait(int h);
void track_release(int h);

int pad_isok(int *xi, char *reason) ;
static int (*emsg)(const char * arg_list, ...);
static int (*imsg)(const char * arg_list, ...);
static int ngfree;
static int gfreesz;

void *safe_memset(void *dst, int val,size_t size, void *base, int len)
{

	if ((char *)dst < (char *)base || size < 0 || (char *)dst+size > (char *)base+len) {
		if (emsg) emsg("safe_memset: failed %p %d %p %d\n",dst,size,base,len);
		lib_abort();
	}
	return memset(dst,val,size);
}

int set_freepc(int x )
{
	freepc = x;
	return TRUE;
}
int set_padpc(int x )
{
	padpc = x;
	return TRUE;
}
int d_malloc_set_debug(int x )
{
	free_check = x;	
	return TRUE;
}


#ifdef TESTING
int main(int argc, char *argv[]) 
{
	// Do some tests on my malloc function.
	char *m[1000];
	int sz[1000];
	int i;
	int b;
	char *s;
	for (i=0; i<1000; i++) {
		int want = myrand()%3000;
		if (want<0) want = -want;
		m[i] = d_malloc(want);
		sz[i] = want;
		memset(m[i],i%100,sz[i]);
	}
	for (i=0; i<1000000; i++) {
		int k;
		int want = myrand()%8000;
		if (want<0) want = -want;
		k = myrand() % 1000;

		s=m[k];
		if (s!=NULL) {
			for (b=0; b<sz[k]; b++) {
				if (s[b]!=k%100) lib_abort();
			}
	
			d_free(m[k]); m[k] = NULL;
		}
		if (k%3==1) {
			m[k] = d_malloc(want);
			sz[k] = want;
			memset(m[k],k%100,sz[k]);
		}
	}
	for (i=1; i<1000; i++) {
		if (m[i]) d_free(m[i]);
	}
	nofrag_stats(stdout);


}
#endif
static int total_big;
#define MAXFREQ 10091
static int dofreq=TRUE;
static int iscol;
static int freq_line[MAXFREQ];
static int freq_n[MAXFREQ];
static int freq_col[MAXFREQ];
static long freq_sz[MAXFREQ]; // bigint 
//static long freq_msec[MAXFREQ]; // bigint 
static char *freq_file[MAXFREQ];

void d_malloc_set_emsg(void (*vemsg)(const char * arg_list, ...))
{
	emsg = (int (*)(const char * arg_list, ...) )vemsg;
}
void d_malloc_init(int (*vemsg)(const char * arg_list, ...))
{
	static int done;
	int rv=0;
	if (done) return ;
	done = TRUE;
#ifdef is_linux
	rv = mallopt(-7, 1);  // M_ARENA_TEST
	rv = mallopt(-8, 1);  // M_ARENA_MAX
#endif
	emsg = vemsg;
	imsg = vemsg;
#ifdef DO_THREADS
	if (m==NULL) m = th_mutex_new("dmalloc");
#endif
}
static int mutex_line;
#ifndef MUTEX_TIME
//#define MUTEX_TIME 600000 chrisp changed so it will timeout BEFORE other mutexes so we will know if it might be a deadlock or really a memory mutex issue. 
#define MUTEX_TIME 100000
#endif



static int mutex_i;
static int (*mutex_cb)(int vm, int isget, int seconds, char *file, int line);
int dmalloc_set_mutex(int (*cb)(int vm, int isget, int seconds, char *file, int line), int m)
{
		mutex_i = m;
		mutex_cb = cb;
		return TRUE;
}

static int nwaiting;
int dmalloc_nwaiting(void)
{
	return nwaiting;
}
int d_mutex_wait(int line)
{
	if (mutex_cb) return mutex_cb(mutex_i,TRUE,600,"dmalloc",line);
#ifdef DO_THREADS
	if (m==NULL) return TRUE;
	if (th_mutex_wait_remember(m,20, __FILE__, line,FALSE)) {
		nwaiting = 0;
		mutex_line = line;
		iscol = FALSE;
		return TRUE;
	}
	if (!th_mutex_get(m)) { //,MUTEX_TIME)) { /* 10 minutes */
		nwaiting++;
		printf("Unable to get mutex: held by %d\n",mutex_line);
		if (!th_mutex_wait(m,10000)) { /* another 10 seconds, incase it came back to life*/
				lib_abort();
		}
		nwaiting = 0;
	}
	iscol = TRUE;
	nwaiting = 0;
	mutex_line = line;
#endif
	return TRUE;
}
void d_mutex_release(void)
{
	if (mutex_cb) { mutex_cb(mutex_i,FALSE,600,"dmalloc",__LINE__); return; }
#ifdef DO_THREADS
	if (m==NULL) return;
	th_mutex_release(m);
	mutex_line = 0;
#endif
}
int nofrag_grow_start(Blocklist *bl, int k,int g);
static Blocklist blocks[32];
static int nb,zzmax;
static void *saved_bit;
int nofrag_init(void)
{
	int sz=8;
	int i;
	if (saved_bit==NULL) saved_bit = real_malloc(3000000);
	for (i=0; i<30; i++) {
//		if (sz>900000) break; 
		if (sz>5000000) break; 
		blocks[i].sz = sz;
		zzmax = sz;
		sz *= 2;

	}
	nb = i;
	if (lib_exists("c:\\windows\\nofrag.grow")) {
		i=0;
		printf("Nofrag growing malloc area\n");
		nofrag_grow_start(&blocks[i],i,390000); i++;
		nofrag_grow_start(&blocks[i],i,790000); i++;
		nofrag_grow_start(&blocks[i],i,640000); i++;
		nofrag_grow_start(&blocks[i],i,160000); i++;
		nofrag_grow_start(&blocks[i],i,125000); i++;
		nofrag_grow_start(&blocks[i],i,155000); i++;
		nofrag_grow_start(&blocks[i],i,31500); i++;
		nofrag_grow_start(&blocks[i],i,17000); i++;
		nofrag_grow_start(&blocks[i],i,16000); i++;
		nofrag_grow_start(&blocks[i],i,30000); i++;
		nofrag_grow_start(&blocks[i],i,40000); i++;
		nofrag_grow_start(&blocks[i],i,8000); i++;
		nofrag_grow_start(&blocks[i],i,4000); i++;
		nofrag_grow_start(&blocks[i],i,150); i++;
		nofrag_grow_start(&blocks[i],i,160); i++;
		nofrag_grow_start(&blocks[i],i,10); i++;
		nofrag_grow_start(&blocks[i],i,190); i++;
		nofrag_grow_start(&blocks[i],i,30); i++;
		nofrag_grow_start(&blocks[i],i,8); i++;
		nofrag_grow_start(&blocks[i],i,8); i++;
		printf("Nofrag growing mallc area - done\n");
	}

	return TRUE;
}

char *mystrndup (const char *s, size_t n)
{
  size_t len = strlen (s); 
  char *xnew;
  if (len>n) len = n;
  xnew = malloc (len + 1);

  if (xnew == NULL)
    return NULL;

  xnew[len] = '\0';
  return memcpy (xnew, s, len);
}


static int show_free=FALSE;
int dump_show_free(int x)
{
	show_free = x;
	return TRUE;
}



#define MAXLINES 50000
int nofrag_leak(int tot, char *fname, int line);
int nofrag_dump(char *fname)
{
	// Go through all block lists and record totals for memory usage ....source:line...
	// For each 'source' file not done
	//	Count mallocs and total for each line.
	//	  Display...
	Block *b;
	int totused=0;
	int *lines,*totals;
	int *lines_f,*totals_f;
	int *lastsz;
	int i;
	int line;
	int totfree=0;
	FILE *fout;
	int h;
	char fname2[BFSZ];
	Wentry *w;
	Hwordlist *wl=NULL;

	//if (use_smalloc) return s_dump(fname);
	if (use_simple) return TRUE;
	if (saved_bit) real_free(saved_bit); saved_bit = NULL;
	ncpy(fname2,fname,BFSZ-1);
	ncat(fname2,"2",BFSZ-1);
	if (lib_exists(fname)) rename(fname,fname2);
	fout = fopen(fname,"w");
	if (fout==NULL) return FALSE;

	fprintf(fout,"TOTAL    |--- NOT FREED -----| |--- USED & FREED ---|\n");
	fprintf(fout,"bytes       bytes     blocks      bytes     blocks\n");

	hword_new(&wl);
	lines = calloc(MAXLINES,sizeof(int));
	lastsz = calloc(MAXLINES,sizeof(int));
	totals = calloc(MAXLINES,sizeof(int));
	lines_f = calloc(MAXLINES,sizeof(int));
	totals_f = calloc(MAXLINES,sizeof(int));
//	printf("find source files\n");
	for (i=0; i<nb; i++) {
		Blocklist *bl = &blocks[i];
		d_mutex_wait(__LINE__);
		bl = &blocks[i];
		for (b=bl->used;b!=NULL;b=b->next) {
			if (!hword_find(wl,b->file)) {
				hword_add(wl,b->file);
			}
		}
		d_mutex_release();
	}
	for (i=0; i<nb; i++) {
		Blocklist *bl = &blocks[i];
		d_mutex_wait(__LINE__);
		bl = &blocks[i];
		for (b=bl->free;b!=NULL;b=b->next) {
			if (!hword_find(wl,b->file)) hword_add(wl,b->file);
		}
		d_mutex_release();
	}

	for (h=0; h<HWORDHASH; h++) {
		for (w=wl->table[h];w!=NULL;w=w->next) {
//			  printf("source %s\n",w->word);
			 memset(lines,0,sizeof(int)*MAXLINES);
			 memset(totals,0,sizeof(int)*MAXLINES);
			 memset(lines_f,0,sizeof(int)*MAXLINES);
			 memset(totals_f,0,sizeof(int)*MAXLINES);
			 for (i=0; i<nb; i++) {
				Blocklist *bl = &blocks[i];
				d_mutex_wait(__LINE__);
				bl = &blocks[i];
				for (b=bl->used;b!=NULL;b=b->next) {
					if (strcmp(b->file,w->word)==0) {
						line = b->line;
						line = line % MAXLINES;
						lines[line]++;
						totals[line] += b->size;
						lastsz[line] = b->size;
					}
				}
				for (b=bl->free;b!=NULL;b=b->next) {
					if (strcmp(b->file,w->word)==0) {
						line = b->line;
                        line = line % MAXLINES;
						lines_f[line]++;
						totals_f[line] += b->size;
					}
				}
				d_mutex_release();
			 }
			 for (i=0; i<MAXLINES; i++) {
				 int overhead = 60; 
	//			 if (!show_free) if (totals[i]==0) continue;
				 int leak ;
					 if (lines[i]>0 ||lines_f[i]>0) {
						 leak = nofrag_leak(lines[i],w->word,i);
						 fprintf(fout,"%8d %8d=used %8d=n %8d=free %8d=nfree %8d=wast %8d=lsz %8d=leak %8d=leaked %s:%d\n"
						,totals_f[i]+totals[i]+(lines[i]+lines_f[i])*overhead,totals[i],lines[i],totals_f[i],lines_f[i],overhead*(lines[i]+lines_f[i]),lastsz[i],leak,leak*lastsz[i],w->word,i);
					 }
			 }
		}
	}
	free(lines); free(totals);
	free(lines_f); free(totals_f);
	fclose(fout);
	hword_free(&wl);
	return TRUE;

}
#define LEAKHASH 1000
typedef struct LEAK {char *src; int n; struct LEAK *next;} Leak;
static Leak *leaks[LEAKHASH];
int nofrag_leak(int tot, char *fname, int line)
{
	char bf[BFSZ];
	int h;
	int ret=0;
	Leak *l;
	if (fname==NULL) return 0;
	sprintf(bf,"%.99s:%d",fname,line);
	h = lib_str_hash(bf,LEAKHASH)+line;
	h = h % LEAKHASH;
//	printf("searching %s h=%d\n",bf,h);
	for (l=leaks[h];l!=NULL;l=l->next) {
		if (strcmp(l->src,bf)==0) {
			ret = l->n;
			l->n = tot;
//	printf("found %s h=%d\n",bf,h);
			return tot - ret ;
		}
	}
	l = calloc(1,sizeof(Leak));
	if (l == NULL) lib_abort();
	l->src = strdup(bf);
//	printf("ADding %s h=%d\n",bf,h);
	d_mutex_wait(__LINE__);
	l->next = leaks[h];
	l->n = tot;
	leaks[h] = l;
	d_mutex_release();
	return ret; // first time, so null... 
}
int nofrag_stats(FILE *f)
{
	int i;
	int nu=0;
	int nf=0;
	Block *b;
	int totused=0;
	int totfree=0;
	d_mutex_wait(__LINE__);
	for (i=0; i<nb; i++) {
		Blocklist *bl = &blocks[i];
		fprintf(f,"Block[%d] size=%d, nused=%d/%d tot=%dk wasted=%dk\n",i,bl->sz,bl->nused,bl->nused+bl->nfree,bl->sz*(bl->nused+bl->nfree)/1024,bl->sz*(bl->nfree)/1024);
		totfree += bl->sz * bl->nfree;
		totused += bl->sz * bl->nused;
		nu = 0; nf = 0;
		for (b=bl->used;b!=NULL;b=b->next) nu++;
		for (b=bl->free;b!=NULL;b=b->next) nf++;
		if (nu!=bl->nused) fprintf(f,"Block mismatch1 %d %d\n",nu,bl->nused);
		if (nf!=bl->nfree) fprintf(f,"Block mismatch2 %d %d\n",nf,bl->nfree);
	}
	d_mutex_release();
	fprintf(f,"Total memory usage %dmb of %dmb, bigtotal %dmb overhead %dmb\n",(totused)/(1024*1024),(totfree+totused)/(1024*1024),total_big/(1024*1024),(nu+nf)*60/(1024*1024));
	return TRUE;
}

static int nsimple;
static bigint simple_bytes;

int d_total_mb(void)
{
	int i;
	int nu=0;
	int nf=0;
	int nall=0;
	unsigned int totused=0;
	int totfree=0;
	d_mutex_wait(__LINE__);
	for (i=0; i<nb; i++) {
		Blocklist *bl = &blocks[i];
		totfree += bl->sz * bl->nfree;
		totused += bl->sz * bl->nused;
		nu += bl->nused;
		nall += bl->nfree + bl->nused;
	}
	d_mutex_release();
	return totused/(1024*1024);	
}
char * d_info(char *info)
{
	int i;
	int nu=0;
	int nf=0;
	int nall=0;
	int totused=0;
	int totfree=0;
	for (i=0; i<nb; i++) {
		Blocklist *bl = &blocks[i];
		totfree += bl->sz * bl->nfree;
		totused += bl->sz * bl->nused;
		nu += bl->nused;
		nall += bl->nfree + bl->nused;
	}
	sprintf(info,"Memory DMALLOC (%d) %dmb/%dmb  simple %d %dmb, wasted %dmb",nu,totused/(1024*1024),(totused+totfree)/(1024*1024),nsimple,(int)(simple_bytes/1000000),(nall*60)/1000000);
	if (use_smalloc) {
		sprintf(info,"Memory SMALLOC debugging, use tellmail dump for details");
	} else if (use_simple) {
		sprintf(info,"Memory SIMPLE (%d) %dmb/%dmb  simple %d %dmb(est), wasted %dmb, freelist=%d padlist=%d freepc=%d paddpc=%d",nu,totused/(1024*1024),(totused+totfree)/(1024*1024),nsimple,(int)(simple_bytes/1000000),(nall*60)/1000000,ngfree,npad,freepc,padpc);
	}
#ifdef USE_NED
	if (use_ned) {
		sprintf(info,"Memory NED (%d) %dmb/%dmb  simple %d %dmb, wasted %dmb",nu,totused/(1024*1024),(totused+totfree)/(1024*1024),nsimple,(int)(simple_bytes/1000000),(nall*60)/1000000);
	}
#endif
	return info;
}


int nofrag_bucket(int sz)
{
	int lo=0;
	int hi=nb-1;
	int k;
	k = nb/2;
	for (;TRUE;) {
		if (blocks[k].sz>=sz) {
			hi = k;
			k = (hi-lo)/2 + lo;	
		} else {
			lo = k+1;
			k = (hi-lo)/2 + lo;	
		}
		if (hi-lo<1) {
			k = lo;
			break;
		}
	}		 
	if (blocks[k].sz<sz) {
		k++;
		if (blocks[k].sz<sz) lib_abort();
	}
	return k;	
}
/*#define MAXHITS 10001
static int hits_n[MAXHITS];
static int hits_line[MAXHITS];
static char *hits_file[MAXHITS];
void hits_report(char *fname)
{
	int i;
	FILE *f;
	f = fopen(fname,"w");
	if (f==NULL) return;
	for (i=0; i<MAXHITS;i++) {
		if (hits_file[i]!=NULL) fprintf(f,"%08d %s:%d\n",hits_n[i],hits_file[i],hits_line[i]);
	}
	fclose(f);

}
void hits_add(char *file, int line)
{
	int hline = line % MAXHITS;
	hits_line[hline] = line;
	hits_file[hline] = file;
	hits_n[hline]++;
}*/
void *d_realloc(void *x, int sz, char *file, int line)
{
	void **px = x;
	void *n;
	int szcopy;
	Block *b;
//	hits_add(file,line);	
	if (use_simple) {
		return psimple_realloc(x,sz,file,line);
	}

	//if (strstr(file,"gd")!=NULL) printf("realloc %d %p %s:%d\n",sz,x,file,line);
	
	if (x==NULL) return d_mallocid(0,sz,file,line);
	d_check_pointer(x,file,line);
	n = d_mallocid(0,sz,file,line);
	if (n==NULL) return NULL;
	b = px[-2];
	szcopy = b->size;
	if (sz<szcopy) szcopy = sz;
	//if (strstr(file,"gd")!=NULL) printf("realloc %d %p %s:%d copying %d  bytes\n",sz,x,file,line,szcopy);
	memcpy(n,x,szcopy);
	d_free(x,file,line);
	return n;
}
void *d_callocid(int dmid, size_t num, size_t size, char *fname, int line)
{
	void *x;
//printf("calloc %d %d %s:%d\n",num,size,fname,line);
	//if (strstr(fname,"gd")!=NULL) printf("calloc %d %p %s:%d\n",num,size,fname,line);
	if (use_simple) {
		if (psimple_callocid) return psimple_callocid(dmid,(int)num,(int)size,fname,line);
		return psimple_calloc((int)num,(int)size,fname,line);
	}

	x = d_mallocid(dmid,(int)(num*size),fname,line);
	if (x==NULL) {
		emsg("In %s:%d, calloc(%d) failed\n", fname, line, size*num);
		lib_abort();
	}
	memset(x,0,num*size);
	return x;
}
void dmalloc_ignore_leak_do(char *file, int line)
{
}
void d_dump(void)
{
	nofrag_dump("dump.log");
}
char *d_strdupid(int dmid, const char *s, char *file, int line)
{
	char *m;
	int len;
	if (s==NULL) return NULL;
	len = lib_strlen(s)+1;
	m = d_mallocid(dmid,len,file,line);
	memcpy(m,s,len);
	return m;
}
static int (*die_fn)(char *reason);
void d_set_die(int (*my_die_fn)(char *reason))
{
	die_fn = my_die_fn;
}

void dmalloc_set_dump(int xdump_to_stdout, int xdump_to_emsg)
{
//	dump_to_stdout=xdump_to_stdout;
//	dump_to_emsg=xdump_to_emsg;
}
int d_log_big(char *fname)
{
		return TRUE;
}
int d_check_pointer(char *x, char *src, int line)
{
	void **px = (void **) x;
	Block *b;
	intptr_t m;
	int end;

	if (use_simple) {
		return TRUE;
	}
#ifdef USE_NED
	if (use_ned) {
		return TRUE;
	}
#endif

	if (x==NULL) return TRUE;
	b = px[-2];
	m = (intptr_t) px[-1]; // dodgy.. px is a pointer to addresses not a pointer to ints...
	if (b->data != px-2) {
		emsg("d_check_pointer1 %s:%d\n",src,line);
		lib_abort(); // another double check	
	}
	if (m!=0x11223344) { // Check the magic number.
		emsg("d_check_pointer2 %s:%d %x\n",src,line, (int)m);
		lib_abort();	
	}
	end = (b->size+(sizeof(char *)-1))/sizeof(char *) ;
	if (px[end]!=(void *) 0x55443322) { // Check the magic number at the end of the block too.
		emsg("d_check_pointer3 %s:%d size=%d %x\n",src,line,b->size,px[end]);
		lib_abort();
	}
	return TRUE;
}
int d_check_pointer2(char *x, char *src, int line)
{
	void **px = (void **) x;
	Block *b;
	intptr_t m;
	int end;

	if (use_simple) {
		return TRUE;
	}
#ifdef USE_NED
	if (use_ned) {
		return TRUE;
	}
#endif

	if (x==NULL) return TRUE;
	b = px[-2];
	m = (intptr_t) px[-1]; // dodgy.. px is a pointer to addresses not a pointer to ints...
	if (b->data != px-2) {
		emsg("d_check_pointer1 %s:%d\n",src,line);
		return FALSE;
	}
	if (m!=0x11223344) { // Check the magic number.
		emsg("d_check_pointer2 %s:%d %x\n",src,line, (int)m);
		return FALSE;
	}
	end = (b->size+(sizeof(char *)-1))/sizeof(char *) ;
	if (px[end]!=(void *) 0x55443322) { // Check the magic number at the end of the block too.
		emsg("d_check_pointer3 %s:%d size=%d %x\n",src,line,b->size,px[end]);
		return FALSE;
	}
	return TRUE;
}


int d_check_pointer_clear(char *x, char *src, int line)
{
	void **px = (void **) x;
	Block *b;
	intptr_t m;
	int end;

	if (use_simple) {
		return TRUE;
	}
#ifdef USE_NED
	if (use_ned) {
		return TRUE;
	}
#endif

	if (x==NULL) return TRUE;
	b = px[-2];
	m = (intptr_t) px[-1];
	if (b->data != px-2) {
		emsg("d_check_pointer1 %s:%d\n",src,line);
		lib_abort(); // another double check	
	}
	m = (intptr_t)  (px[-1]);
	if (m!=0x11223344) { // Check the magic number.
		printf("d_check_pointer2_clear %s:%d %x\n",src,line, (int)m);
		emsg("d_check_pointer2_clear %s:%d %x\n",src,line, (int)m);
		lib_abort();	
	}
	px[-1] = (void *) 0x33333333;
	end = (b->size+(sizeof(char *)-1))/sizeof(char *) ;
	m = (intptr_t) px[end];
	if (m!=0x55443322) { // Check the magic number at the end of the block too.
		emsg("d_check_pointer3 %s:%d size=%d %x\n",src,line,b->size, (int)m);
		lib_abort();
	}
	px[end] = (void *) 0x66666666;
	return TRUE;
}
int d_free_idle(void)
{
	return TRUE;
}
void d_check(void)
{

}
int d_free_check(void)
{
	return TRUE;
}
void dmalloc_set_emsg(int (*vemsg)(const char * arg_list, ...))
{
	emsg=vemsg;
}
int d_free_set_big(int x)
{
	return TRUE;
}

void *d_malloc_checked(int x, char *file, int line)
{
	return d_mallocid(0,x,file,line);
}
void d_malloc_check(void)
{
	int k;
	Blocklist *bl=NULL;
	Block *b;

	 d_mutex_wait(__LINE__);
	 for (k=0;k<15;k++) {
		bl = &blocks[k];
		if (bl==NULL) continue;
		for (b = bl->used;b!=NULL;b=b->next) {
			int l = b->size/sizeof(char *);
			if (!d_check_pointer2((char *)(b->data+2),"dmalloc_check",0)) {
				printf("d_malloc_check: Corruption from pointer %s:%d\n",b->file,b->line);
				emsg("d_malloc_check: Corruption from pointer %s:%d\n",b->file,b->line);
				d_mutex_release();
				lib_abort();
			}
		}
	 }
	 d_mutex_release();
	
}
void d_free_test(void)
{
	int k;
	Blocklist *bl=NULL;
	Block *b;

	if (free_check) { // Check contents of free'd data has not been changed 
	 d_mutex_wait(__LINE__);
	 for (k=0;k<15;k++) {
		bl = &blocks[k];
		if (bl==NULL) continue;
		for (b = bl->free;b!=NULL;b=b->next) {
			int i;
			int l = b->size/sizeof(char *);
			if (b->id != 22) lib_abort(); // Shoudl be set to this while in freelist.
			for (i=0; i<l; i++) {
				if (b->data[i]!=(void *) b->size ) {  // 0x4455667788
					char *s = NULL;
					free_check=FALSE;
					d_mutex_release();
					printf("Crashing corrupted %s:%d free test %d %d\n",b->file,b->line,(int) b->data[i],(int) b->size );
					emsg("Crashing corrupted %s:%d\n",b->file,b->line);
					imsg("Crashing corrupted %s:%d\n",b->file,b->line);
					*s = 0;
				}
			}
		}
	 }
	 d_mutex_release();
	}
}
static int dmalloc_hit_counter;
int dmalloc_hits(void);
int dmalloc_hits_reset(void)
{
	dmalloc_hit_counter = 0;
	return 0;
}
int dmalloc_hits(void)
{
	return dmalloc_hit_counter;
}
static int m_clean;
int d_set_malloc_clean()
{
	m_clean = TRUE;
	return TRUE;
}
int d_freq(void)
{
	int i;
	FILE *f;
	f = fopen("freq.dat","w");
	if (f==NULL) return 0;
	for (i=0; i<MAXFREQ;i++) {
		char *z =freq_file[i];
		if ((freq_line[i]!=0) && (z!=NULL)) {
//			fprintf(f,"%09d %09d %s:%d\n",(int)(freq_sz[i]/(1024*1024)),freq_n[i],z,freq_line[i]);			
			fprintf(f,"%09d col=%d %s:%d\n",freq_n[i],freq_col[i],z,freq_line[i]);			
//			fprintf(f,"%09d %09d %s:%d\n",freq_msec[i],freq_n[i],z,freq_line[i]);			
		}
	}
	fclose(f);
	return 0;
}
static unsigned long dmtot;
static unsigned long crash_limit;
void d_set_malloc_limit(int x)
{
	crash_limit = (unsigned int) x*1000000;
}


int  nofrag_grow_one(Blocklist *bl,int k, int sz )
{
	Block *b;
	int len;
	int waslen;
	int tot=0;
	b = calloc(1,sizeof(Block));
	if (b==NULL) return FALSE;
	len = bl->sz/sizeof(void *) + 3;
	waslen = len;

#ifdef CRTDBG
	tot = len*sizeof(void*);
	b->data = _malloc_dbg(tot,_CLIENT_BLOCK ,__FILE__,__LINE__);
	memset(b->data,0,tot);
#else
	b->data = calloc(sizeof(void *),len);
#endif
	if (b->data==NULL) return FALSE;
	b->k = k;
	b->size = sz; // how much we allocated... 
	b->id = 22;
	b->file = "prealloc";
	b->line = 1;

	if (free_check) { // Check contents of free'd data has not been changed 
		int l = b->size/sizeof(char *);
		int i;
		for (i=0; i<l; i++) {
			b->data[i] = (void *) b->size; //0x4455667788;
		}
	}

	nofrag_insert(&bl->free,b);
	bl->nfree++;
	return TRUE;
}
int nofrag_grow(Blocklist *bl, int k, int sz)
{
	int j;
	int g=1;
	int ret=0;
	if (bl->sz<10000) g = 10;
	if (bl->sz<5000) g = 50;
	if (bl->sz<1000) g = 500;
	if (bl->sz<500) g = 5000;
	if (bl->sz<100) g = 10000;
//	printf("Grow block list %d %d by %d\n",k,bl->sz,g);
	for (j=0;j<g;j++) ret = nofrag_grow_one(bl,k,sz);
	return ret;
}
int nofrag_grow_start(Blocklist *bl, int k, int g)
{
	int j;
	for (j=0;j<g;j++) nofrag_grow_one(bl,k,bl->sz);
	return TRUE;
}
void *d_mallocid(int mid, int x, char *file, int line)
{
//	int msec;
	int k;
	Blocklist *bl=NULL;
	Block *b;
	int end=0;
	int len=0;
	int lh = line%MAXFREQ;
	dmalloc_hit_counter++;

	if (dofreq) { // count frequency of malloc calls for performance analysis.
		char c;
		freq_line[lh] = line; //if (line==5878) lib_abort();
		freq_file[lh] = file;
		freq_n[lh]++;
		freq_sz[lh] += x;
		//msec = lstat_lib_msec();
		// probe the string to make sure it's valid. 
		c = *file;
	}

	if (use_simple) {
		void *ret;
		if (psimple_mallocid) {
			ret = psimple_mallocid(mid,x,file,line);
			//if (dofreq) freq_msec[lh] +=  lstat_lib_msec()- msec ;
			return ret;
		}
		ret = psimple_malloc(x,file,line);
		//if (dofreq) freq_msec[lh] +=  lstat_lib_msec()- msec ;
		return ret;
	}


//	x+=8; // testing...

// chrisp debugging line to remove
//if (x>1000000) printf("malloc %d %s:%d\n",x,file,line);

	if ((x<0) || (x>300000000)) { // negative or more than 300mb, then crash hard... 
		emsg("MALLOC of silly amount of memory %s:%d %d\n",file,line,x);
		d_dump_crash();
		lib_abort();
		return NULL;
	}

	d_mutex_wait(__LINE__);
	if (iscol) freq_col[lh]++;
	dmtot += x;
	if (crash_limit>0) if (dmtot>crash_limit) {
		crash_limit = 0;
		emsg("MALLOC limit reached %s:%d %d %d\n",file,line,x,dmtot);
		d_dump_crash();
		lib_abort();
	}
	if (nb==0) nofrag_init();

	if (x>zzmax) {
		b = calloc(1,sizeof(Block));
		if (b==NULL) goto return_null;
		len = (x/sizeof(void *)) + 4;
		b->data = calloc(sizeof(void *),len);
		if (b->data==NULL) goto return_null;
		b->k = 0;
		b->size = x;
		b->id = 33;
		b->isbig = TRUE; // stand alone block, no linked list....
		total_big += x;
		goto ret;
	}
	k = nofrag_bucket(x);
	bl = &blocks[k];
	b = bl->free;

	if (b==NULL) {
		if (!nofrag_grow(bl,k,x)) goto return_null;
		b = bl->free;
		if (b==NULL)  goto return_null;
	}

/*	if (b==NULL) { // create new one
		b = calloc(1,sizeof(Block));
		if (b==NULL) goto return_null;
		len = bl->sz/sizeof(void *) + 3;
		b->data = calloc(sizeof(void *),len);
		if (b->data==NULL) goto return_null;
		b->k = k;
		b->size = x;
		b->id = 33;
	} else 
	*/
	{ // remove from head of free list
		if (b->id != 22) {
			emsg("Memory block corrupted, line number %d, id=%d\n",b->line,b->id);
			emsg("Memory block corrupted, File %s\n",b->file);
			lib_abort(); // Shoudl be set to this while in freelist.
		}
		nofrag_remove(&bl->free,b);
		if (free_check) { // Check contents of free'd data has not been changed 
			int i;
			int l = b->size/sizeof(char *);
			for (i=0; i<l; i++) {
				if (b->data[i]!=(void *) b->size ) {  // 0x4455667788
					char *s = NULL;
					d_mutex_release();
					free_check=FALSE;
					emsg("Crashing corrupted %s:%d\n",b->file,b->line);
					printf("Crashing corrupted %s:%d free test2 %d %d\n",b->file,b->line,(int) b->data[i],(int) b->size );
					*s = 0;
				}
			}
		}
		b->id = 33;
		b->size = x; // Current allocated size
		bl->nfree--;
		len = bl->sz/sizeof(void *) + 2;
	}	

	// Insert onto dbl link, 'used' linked list.
	nofrag_insert(&bl->used,b);
	bl->nused++;

ret:
	b->file = file;
	b->line = line;
	// Return the block
//	memset(b->data,2,len);
	b->data[0] = b; // The block for this object.
	b->data[1] = (void *) 0x11223344; // Magic number.
	end = (b->size+(sizeof(char *)-1))/sizeof(char *) + 2;
	b->data[end] = (void *) 0x55443322; // Magic number.

	// pad from end of buffer to magic number..
	{
		char *m = (char *) (&b->data[end]);
		char *e = (char *) (b->data+2);
		e = e + x;
		for (;e<m;e++) {
			*e = 0x7e;
		}
	}
	d_mutex_release();
	//if (strstr(file,"gd")!=NULL) printf("malloc %d %p %s:%d\n",x,(b->data+2),file,line);

	{
		void *ret = ( b->data+2);
		if (m_clean) memset(ret,02,x);
	}
//printf("dmalloc returning %p\n",(void *) b->data+2);
	//if (dofreq) freq_msec[lh] +=  lstat_lib_msec()- msec ;

	return b->data+2;
return_null:
	d_mutex_release();
	d_dump_crash();

	return NULL;
}
void d_dump_crash(void)
{
		static int done_dump;
		if (!done_dump) {
			done_dump = TRUE;
			d_dump();
		}

}
void nofrag_remove(Block **first, Block *b)
{
	if (b->last!=NULL) {
		b->last->next = b->next;		
	} else {
		if (*first!=b) {
			emsg("Memory block corrupted, line number %d, id=%d\n",b->line,b->id);
			emsg("Memory block corrupted, File %s\n",b->file);
			lib_abort(); // This should be true....  ONE KNOWN RECENT CRASH HERE: 6 feb 2015
		}
		b->last = NULL;
		*first = b->next;
	}
	if (b->next!=NULL) b->next->last = b->last;

	b->last = NULL;
	b->next = NULL;
}
void nofrag_insert(Block **first, Block *b)
{
	b->next = *first;
	if (*first !=NULL) (*first)->last = b;
	b->last = NULL;
	*first = b;
}
void nofrag_debug(Blocklist *bl, Block *th)
{
	Block *b;
	int max=0;
	for (b=bl->used;b!=NULL;b=b->next,max++) {
		if (max>10000) break;
		if (th==b) continue;
		if (b->data==th->data) lib_abort();
	}
	max = 0;
	for (b=bl->free;b!=NULL;b=b->next,max++) {
		if (max>10000) break;
		if (th==b) continue;
		if (b->data==th->data) lib_abort();
	}
}
int d_is_smalloc(void)
{
	return use_smalloc;
}
int d_malloc_set_smalloc(int x)
{
	use_ned = FALSE;
	use_smalloc = TRUE;
	use_simple = TRUE;

//	psimple_mallocid = s_malloc;
//	psimple_realloc = s_realloc;
//	psimple_callocid = s_calloc;
//	psimple_free = s_free;
	return TRUE;
}

int d_malloc_set_ned(int x)
{
	use_simple = TRUE;
#ifdef USE_NED
	use_ned = x;	
#endif
	return TRUE;
} 

int d_malloc_dodgy(void *p, int len, int showit)
{
	// see if end of block boundary can be found.
	char *x = (char *) p;
	int xpad;
	if (showit ) printf("dodgy test %d \n",len);
	for (;len>0;len--,x++) {  //0x11223344
		if (showit) printf("%x ",*x);
		for (xpad=0;(*x==0x7e) && (xpad<4);xpad++) x++; // skip over padding...
		if (*x==0x22) {  // 55443322
			x++;
			if (*x==0x33) {
				x++;
				if (*x==0x44) {
					x++;
					if (*x==0x55) return TRUE;
				}
			}
		}
	}
	if (showit) printf("\n");
	return FALSE;
}
int d_malloc_used(void)
{
	if (use_simple) return FALSE;
	if (use_ned) return FALSE;
	return TRUE;
}


void d_free(void *x, char *file, int line)
{
	void **px = x;
	Block *b;
	int k;
	static int fcnt;
	Blocklist *bl;

	if (x==NULL) return;
	if (use_simple) {
		psimple_free(x,file,line);
		return; 
	}
//printf("d_free %p %s:%d\n",x,file,line);
	//if (strstr(file,"gd")!=NULL) printf("free %p %s:%d\n",x,file,line);
//	d_check_pointer(x,file,line);


	d_check_pointer(x,file,line);

//	px[-1] = (void *) 0x12121212; // Clear magic number to prevent double free.
	b = px[-2];
	k  = b->k;

	if (b->isbig) {
		total_big -= b->size;
		dmtot -= b->size;
		free(b->data);
		free(b);
		return;
	}
	bl = &blocks[k];

// Scan the 'used' linked list for overlapping memory....
//	nofrag_debug(bl,b);


	if (x==NULL) return;
	if (b->id!=33) lib_abort();
	b->id = 22;
	// Remove from 'used' list....
	d_mutex_wait(__LINE__);
	dmtot -= b->size;
	bl = &blocks[k];
	nofrag_remove(&bl->used,b);
	bl->nused--;
	
	d_check_pointer_clear(x,file,line);

	// And insert into 'free' list
	nofrag_insert(&bl->free,b);
	bl->nfree++;

	if (free_check) { // Check contents of free'd data has not been changed 
		int l = b->size/sizeof(char *);
		int i;
		for (i=0; i<l; i++) {
			b->data[i] = (void *) b->size; //0x4455667788;
		}
	}

	d_mutex_release();
}
unsigned long dmalloc_total(void)
{
	return dmtot;
}
void d_set_abort(void (*fn)(int x))
{
	abort_fn = fn;
#ifdef DO_LOW
	low_set_abort(fn);
#endif
}
void d_lib_abort(int n)
{

	if (abort_fn!=NULL) abort_fn(n);
	lib_abort();
}





/*
header
	
code
length
DATA...

*/
#define SIMPLE_CODE 0x12340000
#define SIMPLE_PAD 0x12350000
#define SIMPLE_FREED 0x33330000
#ifdef WIN64
static int salign = 2;
#else
static int salign = 0;
#endif

#define MAXTOT 20000
static int stot_line[MAXTOT];
static char *stot_file[MAXTOT];
static int stot_used[MAXTOT];
static int stot_lost[MAXTOT];
static int stot_lostn[MAXTOT];
static int stot_col[MAXTOT];
static int stot_n[MAXTOT];
static int trackmem = FALSE;
#define SEX 2
#define S_SIZE 0
#define S_CODE 1
int simple_abort(int *xi,char *reason, char *src, int line)
{
	int b;
	emsg("Crashing because malloc fault (%s) %s:%d xi=%p data=%p \n",reason,src,line,xi,xi+SEX+salign);
	printf("Crashing because malloc fault (%s) \n",reason);
	b = xi[S_SIZE];
	printf("Crashing because malloc fault2 (%s) %x len=%d %x b=%d \n",reason,xi[S_CODE],xi[S_SIZE],xi[b+SEX+salign],b);
	d_lib_abort(74);
	return TRUE;
}

typedef struct FREELIST {void *p; int nb; char *src; int line; struct FREELIST *next;} Flist;
static Flist *fhead;
void gfree_free(void);
void simple_free_raw(void *x, char *src, int line);
void pad_freeone(int *xi);
void gfree_add(void *x, int nb, char *src, int line)
{
	Flist *h = calloc(1,sizeof(Flist));
	h->nb = nb;
	h->p = x;
	memset(x,23,nb*sizeof(int));

	if (ngfree>5000 || gfreesz>20000000) gfree_free();
	d_mutex_wait(__LINE__);
	h->next = fhead;
	h->src = src;
	h->line = line;
	fhead = h;
	ngfree++;
	gfreesz += nb*4;
	d_mutex_release();
}
char *show_ascii(char *tmp, char *s, int len)
{
	char *out=tmp;
	char t2[BFSZ];
	char c;
	int i;
	if (len>100) len = 100;
	for (i=0; i<len; i++) {
		c = s[i];
		if (c==23) *out++ = '+';
		else if (c==0x70) *out++ = '*';
		else if (isprint(c)) *out++ = c;
		else {
			sprintf(t2,"%02x",(unsigned int) c);
			*out++ = '/';
			*out++ = t2[0];
			*out++ = t2[1];
		}
	}
	*out++ = 0;
	return tmp;
}
void gfree_check_one(void *p, int nb, char *src, int line)
{
	int *xi = (int *)p;
	char *s;
	int i;
	int len;
	char bf[BFSZ];
	char tmp[BFSZ];
	xi-=SEX+salign;
	// check each item is not corrupted...
	s = p;
	len = nb*4;
	for (i=0; i<len;i++) {
		if (*s!=23) {
			sprintf(bf,"gfree_check failed at %d of %d found %d instead of 23 %s:%d ngfree=%d [%s]\n",i,len,(int) *s,src,line,ngfree,show_ascii(tmp,s,len-i));
			printf("gfree aborting %s\n",bf);
			simple_abort(xi,bf,src,line);
		}
	}
}
void gfree_check(void)
{
	Flist *f;
	int nfree=0;
	time_t start = time(NULL);
	int elap;
	if (ngfree==0) return;
	d_mutex_wait(__LINE__);
	start=time(NULL);
	for (f=fhead;f!=NULL;f=f->next) {
		gfree_check_one(f->p,f->nb,f->src,f->line);
		nfree++;
	}
	elap = (int)(time(NULL)-start);
	d_mutex_release();
	if (elap>1) emsg("gfree_check took %d seconds %d\n",elap,nfree);
}

void gfree_free(void)
{
	Flist *f,*next;
	gfree_check();
	d_mutex_wait(__LINE__);
	for (f=fhead;f!=NULL;f=next) {
		next = f->next;
		simple_free_raw(f->p,f->src,f->line);
		free(f);
	}
	fhead = NULL;
	ngfree = 0;
	gfreesz = 0;
	d_mutex_release();
}
void simple_free(void *x, char *src, int line)
{
	int b;
	static int fcnt;
	void *ret=x;
	int xline;
	int code;
	int *xi = (int *)x;
	if (x==NULL) return;
	xi-=SEX+salign;
	b = xi[S_SIZE]; // b = number of blocks...
	code = xi[S_CODE]&0xffff0000;
	xline = xi[S_CODE] & 0xffff; // line number of allocation.

	if (code== SIMPLE_PAD) {
		int sz = xi[S_SIZE];
		char reason[BFSZ];
		char reason2[BFSZ];
		if (!pad_isok(xi,reason)) {
			sprintf(reason2,"simple_free check pad %s",reason);
			simple_abort(xi,reason2,src,line);
		}
	}

	if (freepc>0) {
		fcnt++;
		if ( fcnt % 100 < freepc) {
			if (b<100000) {
				//d_mutex_wait(__LINE__);
				gfree_add(x,b,src,line);
				//gfree_check_one(x,b,src,line); // TEST IT NOW REMOVE THIS LINE LATER...
				//d_mutex_release();
				return; // don't free it as well!  
			}
		}
	}
	simple_free_raw(x,src,line);
} 
void simple_free_raw(void *x, char *src, int line)
{
	int b;
	void *ret=x;
	int both;
	char bf[BFSZ];
	int xline;
	int code;
	int *fixit=NULL;
	int *xi = (int *)x;
	if (x==NULL) return;

	xi-=SEX+salign;
	code = xi[S_CODE]&0xffff0000;
	b = xi[S_SIZE]; // b = number of blocks...
	xline = xi[S_CODE] & 0xffff; // line number of allocation.

	nsimple--;

	if (trackmem) {
		int h = xline%MAXTOT;
		if (h>=0) {
			track_wait(h);
			stot_used[h] -= b*sizeof(int);
			simple_bytes -= b*sizeof(int);
			stot_n[h]--;
			track_release(h);
		}
	} else {
		simple_bytes -= b*sizeof(int);
	}
	if (code== SIMPLE_PAD) {
		int sz = xi[S_SIZE];
		char reason[BFSZ];
		int padsz = simple_padsz(sz);
		if (!pad_isok(xi,reason)) {
			sprintf(bf,"PADfree: %s:%d %s\n",src,line,reason);
			simple_abort(xi,bf,src,line);
		}

		pad_freelist(xi);
		code = SIMPLE_CODE;
		fixit = xi -  padsz;
		//if (!pad_isok(xi)) simple_abort(xi,"pad failure on free"); done in pad_freelist()...
	} 

	if (code  == SIMPLE_FREED) {		
		simple_abort(xi,"simple_free4",src,line);
	}
	if (code != SIMPLE_CODE) {
		emsg("Bad simple freea %s:%d b=%d xi=%p xi[0]=%x (start clode lost)\n",src,line,b,xi,xi[S_CODE]);
		simple_abort(xi,"simple_free1",src,line);
	}
// when created: xi[b+SEX+salign] = b + line%MAXTOT ;
	both = xi[b+SEX+salign]; // both 
	if (both-xline!=b) {
		emsg("Bad simple free2 %s:%d b=%d both=%d %d %s:%d (end block length/line lost)\n",src,xline,b,both,xline,src,line);
		simple_abort(xi,"simple_free2",src,line);
	}
#ifdef USE_NED
	if (use_ned) {
		nedfree(xi);
		return;
	}
#endif
	xi[S_CODE] = SIMPLE_FREED;
	xi[b+SEX+salign] = SIMPLE_FREED+1;  // mARK BEGINNIN AND END SO WE WILL KNOW IF WE TRY AND FREE THEM AGAIN...
	if (fixit) {
//		printf("Using fixit %p %p\n",xi,fixit);
		xi = fixit;
	}
	free(xi);
}
int simple_track(int x)
{
	if (x) track_init();
	trackmem = x;
	return TRUE;
}

int simple_stop(void)
{
	trackmem = FALSE;
	return TRUE;
}


int simple_dump(char *fname)
{
	FILE *f;
	int i;
	bigint total=0;
	int increase=0;
	track_init();
	f = fopen(fname,"w");
	if (f==NULL) return FALSE;
	trackmem = TRUE; //Turns on tracking when we do the first dump.
	for (i=0; i<MAXTOT; i++) {
		int lost = stot_used[i]-stot_lost[i];
		if (lost>0) increase += lost;
		total += stot_used[i];
		if (stot_file[i]!=NULL) fprintf(f,"%11d %10d %d %d %s:%d %d\n",stot_used[i]-stot_lost[i],stot_used[i],stot_n[i]-stot_lostn[i],stot_n[i],stot_file[i],stot_line[i],stot_col[i]);
		if (stot_lost[i] < stot_used[i]) stot_lost[i] = stot_used[i];
		if (stot_lostn[i] < stot_n[i]) stot_lostn[i] = stot_n[i];
	}
	fprintf(f,"%11d %10dmb %s:%d %d\n",increase,(int) (total/1000000),"total",0,0);
	fclose(f);
	return TRUE;
}
#ifdef DO_THREADS
#define TRACKN 10
static Mutex *tm[TRACKN];
#endif 
void track_init(void)
{
	int i;
#ifdef DO_THREADS
	if (tm[0]==NULL) {
		for (i=0;i<TRACKN;i++) {
			tm[i] = th_mutex_new("track");
		}
	}
#endif
}
void track_wait(int h)
{
#ifdef DO_THREADS
	th_mutex_get(tm[h%TRACKN]);
#endif
}
void track_release(int h)
{
#ifdef DO_THREADS
	th_mutex_release(tm[h%TRACKN]);
#endif
}
int simple_padsz(int b)
{
	int pad = (b*4)/10;
	if (pad>1000) pad = 1000;
	if (pad<10) pad = 10;
	return pad/4; // convert bytes to blocks.  
}
void pad_clear(int *xi, int padsz, int *base, int len)
{
	int *pre,*post;
	int sz = xi[S_SIZE];
	
	pre = xi - padsz;
	post = xi + (sz + SEX+1+salign);
	//printf("pre %p tp %p post %p  bytes %d (end=%p)\n",pre,pre+padsz,post,padsz,post+padsz*sizeof(int));
	safe_memset(pre,0x70,padsz*sizeof(int),base,len);
	safe_memset(post,0x70,padsz*sizeof(int),base,len);
}
int pad_isok(int *xi, char *reason) 
{
	// return TRUE IF IT'S not corrupt.
	if (xi==NULL) {
		printf("pad_isok passed null\n");
		strcpy(reason,"null code");
		return FALSE;
	}
	{
		int *pre,*post,i;
		int sz = xi[S_SIZE];
		int padsz = simple_padsz(sz);
		int code = xi[S_CODE]&0xffff0000;


		if (code!=SIMPLE_PAD) {
			printf("pad_isok wrong code %d\n",code);
			strcpy(reason,"code wrong");
			return FALSE;
		}


		pre = xi - padsz;
		post = xi + (sz + SEX+1+salign);
	//printf("scanning pre=%p to %p,  post=%p, postend=%p\n",pre,pre+padsz,post,post+padsz);
		for (i=0;i<padsz;i++) {
			char tmp[BFSZ];
			if (pre[i]!=0x70707070) {
				sprintf(reason,"pre error found %x at %d [%s]",pre[i],i,show_ascii(tmp,(char *) (pre+i),(padsz-i)*4) );
				printf("Problem pre[%d] %x\n",i,pre[i]);
				return FALSE;
			}
			if (post[i]!=0x70707070) {
				sprintf(reason,"post error found %x at %d [%s]",post[i],i,show_ascii(tmp,(char *) (post+i),(padsz-i)*4));
				printf("Problem post[%d] %x\n",i,post[i]);
				return FALSE;
			}
		}
		return TRUE;
	}
}
typedef struct PADLIST {void *xi; char *src; int line;  struct PADLIST *next;} Padlist;
#define PHASH 10000
int pad_hash(void *xi)
{
	return (int) ( ((long long) xi) %PHASH);
}
static Padlist *plist[PHASH];
void pad_insert(int *xi, char *src, int line)
{
	// Add to hashed linked list of padded blocks.
	int h = pad_hash(xi);
	Padlist *p;
	npad++;
	p = calloc(1,sizeof(Padlist));
	track_wait(h);
	p->xi = xi;
	p->src = src;
	p->line = line;
	p->next = plist[h];
	plist[h] = p;
	track_release(h);
}
void pad_scan(void)
{
	Padlist *p;
	char reason[BFSZ];
	char bf[BFSZ];
	int elap;
	time_t start;
	static int upto;
	track_init();
	upto++;
	if (upto>PHASH) upto = 0;

	track_wait(upto);
	start = time(NULL);
	for (p=plist[upto];p!=NULL;p=p->next) {
		if (!pad_isok(p->xi,reason)) {
			sprintf(bf,"pad_scan failed %s:%d %s\n",p->src,p->line,reason);
			simple_abort(p->xi,bf,p->src,p->line);
		}
	}
	elap = (int)(time(NULL)-start);
	track_release(upto);
	if (elap>1) emsg("pad_scan took %d seconds\n",elap);

}
int pad_freelist(int *xi)
{
	// If it is padded, then free the linked list entry.
	// and return true;
	int sz = xi[S_SIZE];
	int code = xi[S_CODE]&0xffff0000;
	if (code!=SIMPLE_PAD) return FALSE;

	// Scan the list of padded things and remove problem
	pad_freeone(xi);		
	return TRUE;
}
void pad_freeone(int *xi)
{
	Padlist *p,**last;
	int h = pad_hash(xi);
	track_wait(h);
	last = &plist[h];
	for (p=plist[h];p!=NULL;p=p->next) {
		if (p->xi==xi) {
			*last = p->next;
			free(p);
			npad--;
			goto done;
		}
		last = &(p->next);
	}
done:
	track_release(h);
}


void *simple_malloc(int n, char *src, int line)
{
	int *base;
	char reason[BFSZ];
	void *ret;
	int *xi;
	int padsz = 0;
	int tot=0;
	static int fcnt;
	//void *x;
	int b = (sizeof(int)+n)/sizeof(int); // 0,1,2,3 = 1,  4,5,6,7=2
	int align = 0;
	nsimple++;
	
	if (padpc>0) {  // decide if we should pad this one....
		fcnt++;
		if ( fcnt % 100 < padpc) {
			padsz = simple_padsz(b);
		}
	}

#ifdef USE_NED
	if (use_ned) {
		xi = nedmalloc((b+SEX+1)*sizeof(int));
	} else {
		xi = malloc((b+SEX+1)*sizeof(int));
	}
#else 
	tot = ((intptr_t)b+SEX+1+salign+padsz*2)*sizeof(int);
	base = xi = malloc(tot);
//	if (padsz) printf("padsz %d, b=%d total=%d rawxi=%p rawend=%p\n",padsz,b,tot,xi,xi+tot);
#endif
	if (xi==NULL) {
		emsg("MALLOC FAILED %d %s:%d\n",n,src,line);
		simple_abort(xi,"Malloc failed in simple_malloc()",src,line);
	}
	if (padsz) xi += padsz;// Not times sizeof int
	xi[S_SIZE] = b;
	xi[S_CODE] = SIMPLE_CODE + line%MAXTOT;
	xi[b+SEX+salign] = b + line%MAXTOT ;
	if (padsz) {
//		printf("padclear  %d xi=%p\n",padsz,xi);
		xi[S_CODE] = SIMPLE_PAD + line%MAXTOT;
		pad_clear(xi,padsz,base,tot);
		if (!pad_isok(xi,reason)) simple_abort(xi,reason,src,line);
		pad_insert(xi,src,line);
	}
	

	if (trackmem) {
		int h;
		h = line%MAXTOT;
		if (stot_file[h]) if ((stot_line[h] != line) || (stot_file[h] != src)) stot_col[h]=1;
		stot_line[h] = line;
		stot_file[h] = src;
		track_wait(h);
		stot_used[h] += b*sizeof(int);
		simple_bytes += b*sizeof(int);
		stot_n[h]++;
		track_release(h);
	} else {
		simple_bytes += b*sizeof(int);
	}
	ret = (void *) (xi+SEX+salign);

	return ret;
}
void *simple_realloc(void *x, int n, char *src, int line)
{
	int *xi = (int *)x;
	void *ret;
	int b = (sizeof(int)+n)/sizeof(int); // 0,1,2,3 = 1,  4,5,6,7=2
	int xline;
	int b2;
	int both;
	int code;
	int oldsz;
	if (x==NULL) return simple_malloc(n,src,line);
	xi-= SEX+salign;
	xline = xi[S_CODE] & 0xffff;
	code = xi[S_CODE]&0xffff0000;
	oldsz = xi[S_SIZE];
	b2 = xi[S_SIZE];
	if (n==0) { // realloc could be used to free memory!  stupid code!!!
		simple_free(x,src,line);
		return NULL; 
	}
	if (code==SIMPLE_PAD) {
		void *r = simple_malloc(n,src,line);
		int tocopy = oldsz*4;
		if (n<tocopy) tocopy = n;
//		printf("simple_realloc_pad %s:%d %p %d oldsz=%d newsz=%d\n",src,line,x,n,oldsz*4,n,tocopy);
		memcpy(r,x,tocopy);
		simple_free(x,src,line);
		return r;
	}
	if ((xi[S_CODE]&0xffff0000) != SIMPLE_CODE) simple_abort(xi,"simple_realloc3",src,line);

	both = xi[b2+SEX+salign];
	if (both-xline!=b2) {
		emsg("Bad simple free3 %s:%d b=%d %d %d\n",src,line,b2,both,xline);
		simple_abort(xi,"simple_free2",src,line);
	}


	if (trackmem) {
		int h = xline%MAXTOT;  // using the original line! 
		if (h>=0) {
			track_wait(h);
			stot_used[h] -= b2*sizeof(int);
			simple_bytes -= b2*sizeof(int);
			track_release(h);
		}
	} else {
		simple_bytes -=b2*sizeof(int);
	}



	if (use_ned) {
#ifdef USE_NED
		xi = nedrealloc(xi,(b+SEX+1)*sizeof(int) );
#endif
	} else {
		xi = realloc(xi,((intptr_t)b+SEX+1+salign)*sizeof(int) );
	}
	if (xi==NULL) {
		char bf[BFSZ];
		sprintf(bf,"realloc failed, out of memory %s:%d bytes %d",src,line,n);
		simple_abort(xi,bf,src,line);
	}
	xi[S_CODE] = SIMPLE_CODE + xline%MAXTOT;  // re-store the original line. 
	xi[S_SIZE] = b;
	xi[b+SEX+salign] = b + xline%MAXTOT;

	if (trackmem) {
		int h = xline%MAXTOT;  // Using the original line. 
		if (h>=0) {
			track_wait(h);
			stot_used[h] += b*sizeof(int);
			simple_bytes += b*sizeof(int);
			track_release(h);
		}
	} else {
		simple_bytes += b*sizeof(int);
	}
	ret = (void*)(xi + SEX + salign);

	return ret;
}
void *simple_calloc(int num, int size, char *src, int line)
{
	void *x = simple_malloc(num*size,src,line);
	memset(x,0, (intptr_t)num*size);
	return x;
}
char *simple_strdup(const char *s, char *src, int line)
{
	char *n = simple_malloc(lib_strlen(s)+1,src,line);
	strcpy(n,s);
	return n;
}



static int nreal;
void real_free(void *x){
	nreal--;
	free(x);
}

void *real_malloc(int n){
	nreal++;
	return malloc(n);
}
void *real_realloc(void *x, int sz)
{
	if (x==NULL) nreal++;
	return realloc(x,sz);
}
void *real_calloc(int num, int size)
{
	nreal++;
	return calloc(num,size);
}
char *real_strdup(const char *s)
{
	nreal++;
	return strdup(s);
}
/* 

[pointer to memory]
[next]
[last]
[src]
[line]


	New code, allcoate 'storage'*2 for anything under 5k 

	Keep two dble linked lists:

	1) allocated memory
	2) freed memory

*/
// THIS CODE WILL ONLY WORK ON 32BIT MACHINES... IT'S FOR DEBUGGING...! 
// why??? 

#ifdef IS64
typedef struct MEMBLOCK { struct ALINK* lnk; long long len; long long id; long long pad; } Memblock;
#else
typedef struct MEMBLOCK { struct ALINK* lnk;  int len; int id; int pad; } Memblock;
#endif
typedef struct ALINK {Memblock *m; int id; struct ALINK *next; struct ALINK *last; char *src; int line;} Alink;
Alink *ll_add(Alink **head,Memblock *m, char *src, int line) ;
Alink *ll_add_raw(Alink **head,Alink *add);
void ll_del(Alink **head, Alink *x);
void cp_free_real(void *x,char *src, int line);

#define CP_OFFSET 4
enum {C_ALINK,C_LEN,C_ID,C_PAD};
static int dolists=TRUE;
static Alink *listused,*listfree;

int cp_abort(void** xi, char* reason)
{
	int b;
	emsg("Crashing because malloc fault (%s) \n", reason);
	b = (int)xi[S_SIZE];
	emsg("Crashing because malloc faultcp (%s) len=%d id=%d pad=%d \n", reason, (int)xi[C_LEN], (int)xi[C_ID],(int) xi[C_PAD]);
	d_lib_abort(74);
	return TRUE;
}

void cp_check(Memblock *m, char *src, int line)
{
	void **xi = (void **)  m;
	int len = m->len;
	int i;
	int pad = m->pad;
	int b = (sizeof(int*) +len-1)/ sizeof(int*); // 0,1,2,3 = 1,  4,5,6,7=2
	int rest;
	char *s;

	if (m->id!=99) {
		printf("ID LOST FOR THIS BLOCK %s:%d %d\n",src,line,(int)m->id);
		emsg("ID LOST FOR THIS BLOCK %s:%d %d\n",src,line,(int)m->id);
		cp_abort(xi, "cp_free3");
	}
	s = (char *) (xi+CP_OFFSET);
	rest = (b* sizeof(int*) -len)+ pad* sizeof(int*);
	for (i=0; i<rest; i++) {
		if (s[len+i]!=66) {
				d_mutex_release();
				printf("cp_check failed2 %s:%d c=%d(66) len=%d i=%d rest=%d\n",src,line,(int)s[len+i],len,i,rest); 
				emsg("cp_check failed2 %s:%d c=%d len=%d i=%d rest=%d\n",src,line,s[len+i],len,i,rest); 
				cp_abort(xi, "cp_free4");
		}
	}
	return;
}
static int ncpfree;
void cp_free_later(void);

void cp_free(void *x, char *src, int line)
{
	int **xi = ((int **)x)-CP_OFFSET;
	int len = (int) xi[C_LEN];
	Alink *a = (Alink *) xi[C_ALINK];
	// memset(x,22,len); // invalidate the contents...  lets not do this as we want odd behaviour...
	d_mutex_wait(__LINE__);

	if (a->id !=8888) {
		emsg("Invalid cp_free %s:%d %x %d\n",src,line,x,a->id);
		cp_abort(xi, "cp_free2");
	}
	ll_del(&listused,a);
	ll_add_raw(&listfree,a);
	ncpfree++;
	if (ncpfree>1000) {
		cp_free_later();
	}
	d_mutex_release();
}
void cp_free_later(void)
{
	// Go through listfree and free it all really.
	Alink *l,*next;
	d_mutex_wait(__LINE__);
	for (l=listfree;l!=NULL;l=next) {
		if (l->id!=8888) { 
			printf("free_later found bad entry in list %d\n",l->id);		
			emsg("free_later found bad entry in list %d %s:%d\n",l->id,l->src,l->line);		
			lib_abort();
		}
		next = l->next;
		cp_check(l->m,l->src,l->line);
		cp_free_real(l->m,l->src,l->line);
		ll_del(&listfree,l);
		l->id = 7777;
		free(l);  // this one doesn't cause the crash
	}
	listfree = NULL;
	ncpfree = 0;
	d_mutex_release();
}
void cp_check_some(void)
{
	// Process the linked list for a second... 
	time_t start = time(NULL);
	Alink *l;
	int x=0;
	static int skip;
	d_mutex_wait(__LINE__);
	for (l=listfree;l!=NULL;l=l->next) {
		if (l->id!=8888) {	
			emsg("cp_check_some found bad entry in listfree %d %s:%d\n",l->id,l->src,l->line);		
			lib_abort(); 
		}
		cp_check(l->m,l->src,l->line);
	}
	for (l=listused;l!=NULL;l=l->next) {
		if (x<skip) continue; // don't recheck last ones again
		if (l->id!=8888) {		
			emsg("free_later found bad entry in listused %d %s:%d\n",l->id,l->src,l->line);		
			lib_abort();
		}
		cp_check(l->m,l->src,l->line);
		x++;
		if (x%1000==1) if (time(NULL)-start>10) {
			skip = x;
			break;
		}
	}
	if (l==NULL) skip = 0;
	d_mutex_release();
}
void cp_free_real(void *x,char *src, int line)
{
	Memblock *m = (Memblock *) x;
	cp_check(m,src,line);
	m->id = 7788;
	free(x); // THIS ONE DOES. 
}

void *cp_malloc(int len, char *src, int line)
{
	int b = (sizeof(int *)+len-1)/sizeof(int *); // 0,1,2,3 = 1,  4,5,6,7=2  or .... 0,1,2,3,4,5,6,7 = 1
	char *x;
	int **xi;
	int rest;
	int want;
	int pad = len/ sizeof(int*)+1;
	if (pad>250) pad = 250;

	want = ((intptr_t)b + CP_OFFSET + pad) * sizeof(int*);
	xi = malloc(want);
	//printf("Malloc LEN=%d want=%d\n", len, want);
	
	xi[C_LEN] = (void *) len;
	xi[C_PAD] = (void *) pad;
	xi[C_ID] = (void *) 99;
	x = (char *) (xi+CP_OFFSET);
	rest = (b* sizeof(int*) -len)+ pad* sizeof(int*);
	// printf("xi=%p memset %p len=%d rest=%d want=%d",xi, x, len,rest,want);
	memset(x+len,66,rest);
	
	d_mutex_wait(__LINE__);
	xi[C_ALINK] = (int *) ll_add(&listused,(Memblock *) xi,src,line);


	// Next line just a double check it's ok before we proceed.
	//cp_check((Memblock *) xi, src, line);

	d_mutex_release();
//	printf("malloc %x %s:%d\n",x,src,line);

	return x;
}
void *cp_realloc(void *old, int n, char *src, int line)
{
	void *b;
	int min;
	int **xi = (int **) old; 

	if (n==0) {
		if (old) cp_free(old,src,line); 
		return NULL;
	}
	b = cp_malloc(n,src,line);
	xi -= CP_OFFSET; 
	if (old) {
		min = (int) xi[C_LEN];
		if (min>n) min = n;
		memcpy(b,old,min); // if storage reduced in size, then only copy as much as it is now...
		cp_free(old,src,line);
	}
	return b;
}
void *cp_calloc(int num, int size, char *src, int line)
{
	void *x = cp_malloc(num*size,src,line);
	memset(x,0, (intptr_t)num*size);
	return x;
}
char *cp_strdup(const char *s, char *src, int line)
{
	char *n = cp_malloc(lib_strlen(s)+1,src,line);
	strcpy(n,s);
	return n;
}
int d_malloc_set_pad(void)
{
	use_simple = TRUE;
	psimple_malloc = cp_malloc;
	psimple_realloc = cp_realloc;
	psimple_calloc = cp_calloc;
	psimple_strdup = cp_strdup;
	psimple_free = cp_free;
	return TRUE;
}
int d_malloc_set_simple(int x)
{
	use_simple = x;
	psimple_malloc = simple_malloc;
	psimple_realloc = simple_realloc;
	psimple_calloc = simple_calloc;
	psimple_strdup = simple_strdup;
	psimple_free = simple_free;
	return TRUE;
}


Alink *ll_add(Alink **head,Memblock *m, char *src, int line) 
{
	Alink *add = calloc(1,sizeof(Alink));
	Alink *h= *head;
	add->m = m;
	add->src = src;
	add->line = line;
	add->id = 8888;
	ll_add_raw(head,add);
	return add;
}
Alink *ll_add_raw(Alink **head,Alink *add)
{
	Alink *h= *head;
	if (add->id!=8888) {
		printf("ll_del LOST id %d *=(8888)\n",add->id); 
		emsg("ll_del LOST id %d *=(8888)\n",add->id); 
		lib_abort(); 
	}
	add->last = NULL;
	add->next = NULL;
	if (h) h->last = add;
	*head = add;
	add->next = h;
	return add;
}
int ll_show(Alink *first)
{
	Alink *h;
	for (h=first;h!=NULL;h=h->next) {
		printf("[%p] <--%p  mem=(%p)  -->%p  %s:%d\n",h,h->last,h->m,h->next,h->src,h->line);
	}
	return TRUE;
}
/*
a  0< ->a  
b  a<  >c
c  b<  >0

*/

void ll_del(Alink **head, Alink *x)
{
//	if (head) ll_show(*head);
	if (x->id!=8888) {
		printf("ll_del4 LOST id %d (8888)\n",x->id); 
		emsg("ll_del4 LOST id %d (8888)\n",x->id); 
		lib_abort();
	}
	if (head) if (*head==x) *head = x->next; 
	if (x->last) x->last->next = x->next;
	if (x->next) x->next->last = x->last;
}





















