/*
todo:	
	Add history so duplicates are rejected in normal way.
  
    
	Check file open/close
	Check malloc/free/strdup
	Test items are allowed thru for first 15 minutes only
---------------------------------------------------------------------------------------
  
    The magical hiding of missing parts functions :-)

Hash on subject, 2000 entries.
Write index with Position/Length/Partno,
Also in index, writepoint,totallen,erasedpoint
Read index on startup.
Extend file if necessary.

What happens to existing items that have partially arrived? 
	Command to run 20 minutes after starting it, to send all
	stored items into the server. (background job?)

Add a command to delete partial posts not in the last 100 items.
Must have 'n' buckets, as 2 gig isn't enough.
	
	b1,b2,b3,b4,b5
	Expire all in next bucket.
	If index is full, then, expire bucket
	If 


*/
#ifdef __MWERKS__ /*RHP we will take this to mean we are making a Mac version*/
#include "mac.h"
#endif

#include <stdlib.h>
#include <stdio.h>
#include <ctype.h>
#include <string.h>
#include <assert.h>
#ifdef VMS
#include <unixio.h>
#define unlink remove
#endif
#ifndef NO_FCNTL
#include <fcntl.h>
#else
/*RHP not for mac */
#ifndef Mac
#include <file.h>
#endif
#endif

#ifdef WIN32
#include <windows.h>
#endif
#include <errno.h>
#include "thread.h"

#include "emsg.h"
#include "sary.h"
#include "str.h"
#include "lib.h"
#include "group.h"
#include "hist.h"
#include "suck.h"
#include "chan.h"
#include "feed.h"
#include "expire.h"
#include "control.h"
#include "item.h"
#include "db.h"

typedef struct PART {int bkt; int bktid; int offset; int len; int maxparts; int part;
	int grp,num; 
	struct PART *next;} Part;
typedef struct PARTS {char *subj; char *mid; Part *parts; struct PARTS *next; } Parts;

#include "parts.h"
#undef mymalloc
#undef myfree
#define mymalloc(x) mymalloc2(x,M_PARTS)
#define myfree(x) myfree2(x,M_PARTS)

int part_thread_send(Parts *pts);
static int nsend,ndrop;
static int dbupto,dbclear,dbend,dbbkt,ndbbkt;
#define MAXDBBKT  2000
static int bktid[MAXDBBKT];
static int part_inprogress = FALSE;
static int part_disable=TRUE;
#define PART_HASH 2000
static Parts *ptable[PART_HASH];
static int isfast;
static int stat_nparts;
static double stat_partsize;
static int stat_initems;
static double stat_insize;
static int stat_okitems;
static int stat_nrej;
static double stat_oksize;
static double stat_dropsize;
static int stat_nfailed;
static int let_thru;
static int nhist;
Parts *pts_dup(Parts *pts);
static int time_zero;
void partmsg(char * arg_list, ...);
int part_clear_oldest(void);
int part_idx_add_new(Part *p, char *subj, char *mid);
int part_lost(Parts *pts, Part *pone);
Parts *parts_find_only(int h, char *subj, char *mid);
int part_send_one(Part *p, int isthread, int *donesync);
int part_thread_complete(void *x);
int parts_free(Parts *pts);
#include "mutex.h"

static int nsync,nasync;
int part_sync(char *s);
int part_age(void);
static int age_lost;
char *part_status(void)
{
	static char bf[BFSZ];
	if (part_disable) return "";
	lprintf(bf,BFSZ-1,"Parts %d:%d %.0fmb, In %d %.0fmb, Complete %d %.0fmb, %.0f%% %.0f%%, Rej %.0f%%\r\nDrop %.0fmb, Err %d Rotated %d minutes ago, Upto %d %0.2f %d, Thread send %d drop %d, sync %d/%d\r\n"
		,stat_nparts,nhist,stat_partsize/1000000
		,stat_initems,stat_insize/1000000
		,stat_okitems,stat_oksize/1000000
		,stat_okitems*100.0/(stat_initems+1)
		,stat_oksize*100.0/(stat_insize+1)
		,stat_nrej*100.0/(stat_initems+1)
		,stat_dropsize/1000000.0
		,stat_nfailed
		,part_age()/60,dbbkt,(double) dbupto/100000000.0,ndbbkt,nsend,ndrop
		,nsync,nasync);	
	return bf;
}
int part_age(void)
{
	/* Find the diff in age between dbbkt and dbbkt + 1 */
	int next=dbbkt+1;
	int last = dbbkt-1;

	if (isfast) return age_lost;
	if (next>=ndbbkt) next=0;
	if (last<0) last=0;
	imsg("part_age last,  %s %d\n",part_file(last),lib_cdate(part_file(last)));
	imsg("part_age next, %s %d\n",part_file(next),lib_cdate(part_file(next)));
	return lib_cdate(part_file(last)) - lib_cdate(part_file(next));
}
typedef struct HPARTS {Parts *parts; struct HPARTS *next; } Hparts;
static Hparts *htable[PART_HASH];

int part_hist(char *mid)
{
	int h;
	Hparts *p;

	if (isfast) return FALSE;
	if (part_disable) return FALSE;
	h = str_hash(mid,PART_HASH);
	for (p=htable[h];p!=NULL;p=p->next) {
		if (strcmp(mid,p->parts->mid)==0) return TRUE;
	}
	return FALSE;
}
int part_hist_add(Parts *pts)
{
	Hparts *p;
	int h = str_hash(pts->mid,PART_HASH);
	p = mymalloc(sizeof(Hparts));
	memset(p,0,sizeof(Hparts));
	p->parts = pts;
	p->next = htable[h];
	nhist++;
	htable[h] = p;
	return TRUE;
}
int part_hist_del(Parts *pts)
{
	Hparts *p,**last;
	int h = str_hash(pts->mid,PART_HASH);
	last = &(htable[h]);
	for (p=htable[h];p!=NULL;p=p->next) {
		if (pts==p->parts) {
			*last = p->next;
			myfree(p);
			nhist--;
			return TRUE;
		}
		last = &(p->next);
	}
	emsg("Could not find history parts entry to free\n");
	dnews_ex("no history parts entry");
	return FALSE;	
}
#define INDEX_LIMIT 10000
int part_index_size(void)
{
	static int x;
	int n;
	if (x>0) return x;
	x = atoi(config_get(C_PART_INDEX));
	if (x==0) {
		n = atoi(config_get(C_PART_BUCKETS));
		x = INDEX_LIMIT;
		if (n>50) x = 20000;
		if (n>100) x = 40000;
	}
	return x;
}

static int store_write;
static Parts *store_pts;
static Part *store_p;
int part_check(char *xsubject, char *mid, char *item, int len)
{
	/* Check if it's a multi part post, if it is process it */ 
	/* If we are processing one, then return false to avoid loop */
	/* Find parts, and add this one, if we now have the 'set' then process them */

	char subj[BFSZ];
	char subject[BFSZ];
	char fname[BFSZ];
	int part,partmax;
	int h,j;
	Parts *pts;
	int gap;
	Part *p;

	store_write = FALSE;
	if (part_inprogress) return FALSE; 
	if (part_disable) return FALSE;

	ncpy(subject,xsubject,302);
	if (!parts_subject(subject, subj, fname, &part, &partmax)) return FALSE;
	if (strlen(subj)>299) return FALSE;
	if (partmax<2) return FALSE; /* No need eh :-) */
	if (strncmpnc(subject,"re: ",4)==0) return FALSE; /* WILL miss some but better safe */

	dmsg("part: Incoming multi part message %d/%d  %.180s\n",part,partmax,xsubject);

	/* Ok, here we have a part of a multi part message */
	if (!isfast) {
	 part_findgap(len); /* Move to next bucket before we allocate a part struct */
	}
	h = str_hash(subj,PART_HASH);
	if (part==0) {
		pts = parts_find_only(h,subj,mid);
		if (pts==NULL) {
			/* Zeroth part, might have arrived 'after' the rest, so send it through */	
			return FALSE; 
		}
	}
	pts = parts_find_add(h,subj,mid);
	p = mymalloc(sizeof(Part)); memset(p,0,sizeof(Part));
	p->bkt = 0;
	p->grp = 0;
	p->num = 0;
	p->offset = 0;
	p->len = 0;

	p->next = pts->parts;
	pts->parts = p;
	p->maxparts = partmax;
	p->part = part;

	stat_start(X_PART_STORE);
	if (!isfast) part_store(p,item,len);
	stat_end(X_PART_STORE);

	stat_nparts++; stat_partsize += p->len;
	if (!isfast) part_idx_add(p,subj,mid);

	stat_initems++; stat_insize += p->len;

	if (isfast) {
		store_pts = pts;
		store_p = p;
		db_setpart(TRUE);
		store_write = FALSE;
		if (parts_complete(store_pts)) {
			store_write = TRUE;
		}
		return FALSE;
	}


	if (parts_complete(pts)) {
		part_inprogress = TRUE;
		stat_start(X_PART_SEND);
		parts_send(pts);
		stat_end(X_PART_SEND);
		parts_del(h,pts);
		part_inprogress = FALSE;
	}
	stat_start(X_PART_CLEAR);
	/* Set to 5 instead of 10000 for testing */ 
	gap = part_index_size()/10;

	if (stat_nparts>part_index_size()) {
	 for (j=dbbkt+1;(stat_nparts>(part_index_size()-gap)) && (j<ndbbkt);j++) {
		imsg("part: Too many parts in index %d, clearing bucket %d, In dnews.conf part_index=%d\n",stat_nparts,j,part_index_size());
		part_clear_bkt(j);
	 } 
	 for (j=0;(stat_nparts>(part_index_size()-gap)) && (j<dbbkt);j++) {
		imsg("part: Too many parts in index %d, clearing bucket %d\n",stat_nparts,j);
		part_clear_bkt(j);
	 }
	 part_index_rewrite();
	}
	stat_end(X_PART_CLEAR);
	if (let_thru > lib_gmtime()) {
		imsg("part: Letting multi part item thru still as we have just started %d, part_index=%d\n",(let_thru-lib_gmtime())/60,part_index_size());
		return FALSE;
	}
	return TRUE; /* Tells item_suck not to store this item (it's already happenned) */  
}
int part_write_complete(void)
{
	Parts *pts = store_pts;
	int h;

	if (!isfast) return TRUE;
	if (!store_write) {
		stat_start(X_PART_EXPIRE);
		if (stat_nparts>part_index_size()) part_clear_oldest();
		stat_end(X_PART_EXPIRE);
		return TRUE;
	}

	/* Store into real index, if buckets still exist, not deleted*/
	stat_start(X_PART_SEND);
	parts_send(pts);
	stat_end(X_PART_SEND);
	h = str_hash(pts->subj,PART_HASH);
	parts_del(h,pts);
	part_inprogress = FALSE;
	return TRUE;
}
int part_send_one_idx(Part *p)
{
	db_setpart(FALSE); /* Just to make sure */
/*imsg("Sending in part to index bkt=%d g=%d:%d offset=%d len=%d\n",p->bkt,p->grp,p->num,p->offset,p->len);*/
	if (db_expired(p->bkt)) {
		imsg("Part of multi part message already expired\n");
		return FALSE;
	}
	if (p->len>0) db_add_idx(p->bkt,p->grp,p->num,p->offset,p->len);
	stat_okitems++;
	stat_oksize+= p->len;
	return TRUE;
}
int part_callback(int fid,int grp,int num,int pnt,int len)
{
	Part *p = store_p;
	if (!isfast) return TRUE;

	p->bkt = fid;
	p->grp = grp;
	p->num = num;
	p->offset = pnt;
	p->len = len;
	stat_insize += p->len;
	stat_partsize += p->len;
/*imsg("Storing idx info bkt=%d g=%d:%d offset=%d len=%d\n",p->bkt,p->grp,p->num,p->offset,p->len);*/
	part_idx_add_new(store_p,store_pts->subj,store_pts->mid);


	return TRUE;
}
int parts_del(int h, Parts *pts)
{
	Parts **next,*pnext,*p;

	next = &ptable[h];
	for (p=ptable[h];p!=NULL;p=pnext) {
		pnext = p->next;
		if (p==pts) {
			*next = p->next;
			parts_free_parts(pts,FALSE);
			return TRUE;
		}
		next = &(p->next);
	}


	emsg("Unable to find part to free\n");
	return FALSE;
}

int parts_free_parts(Parts *pts, int doreport)
{
	Part *p,*pnext;
	int t = 0;

	part_hist_del(pts);
	for (p=pts->parts;p!=NULL;p=pnext) {
		pnext = p->next;
		stat_nparts--; stat_partsize -= p->len;
		t += p->len;
		myfree(p);		
	}
	if (doreport) {
		;
	}
	mystrfree(pts->subj,SD_PARTS);
	mystrfree(pts->mid,SD_PARTS);
	pts->subj = NULL;
	pts->mid = NULL;
	myfree(pts);
	return t;
}
Parts *parts_find_only(int h, char *subj, char *mid)
{
	Parts *pts;
	for (pts=ptable[h]; pts!=NULL; pts=pts->next) {
		if (strcmp(pts->subj,subj)==0) {
			dmsg("part: Found existing multi part index entry %d (%.90s)\n",h,subj);
			return pts;
		} 
	}
	return NULL;
}
Parts *parts_find_add(int h, char *subj, char *mid)
{
	Parts *pts;
	for (pts=ptable[h]; pts!=NULL; pts=pts->next) {
		if (strcmp(pts->subj,subj)==0) {
			dmsg("part: Found existing multi part index entry %d (%.90s)\n",h,subj);
			return pts;
		} 
	}
	pts = mymalloc(sizeof(Parts)); memset(pts,0,sizeof(Parts));
	pts->subj = mystrdup(subj,SD_PARTS);
	pts->mid = mystrdup(mid,SD_PARTS);
	pts->next = ptable[h];
	ptable[h] = pts;
	dmsg("part: Created new multi part index entry %d (%.90s)\n",h,subj);
	part_hist_add(pts);
	return pts;
}
int parts_complete(Parts *pts)
{
	char gotit[1000];
	int maxparts;
	Part *p = pts->parts;
	int i;

	if (p==NULL) return FALSE;
	maxparts = p->maxparts;

	for (p=pts->parts;p!=NULL;p=p->next) {
		if (maxparts<p->maxparts) maxparts = p->maxparts;
	}

	if (maxparts>900) return TRUE;
	
	for (i=0; i<=maxparts; i++) gotit[i] = FALSE;
	for (p=pts->parts;p!=NULL;p=p->next) {
		if (p->part<0) continue;
		if (p->part>maxparts) continue;
		gotit[p->part] = TRUE;
	}
/*	for (i=1; i<=maxparts; i++) imsg("got part %d of %d\n",i,gotit[i]);*/
	for (i=1; i<=maxparts; i++) if (!gotit[i]) return FALSE;
	return TRUE;
}
#define PART_OLD file_work("part.old")
int part_init(void)
{
	FILE *f;
	/* Read in index, extend bucket. etc..*/
	time_zero = time(NULL);
	if (config_true(C_PART_FAST)) isfast = TRUE;
	if (isfast) part_disable = FALSE;
	if (!isfast) if (!part_file_init()) return FALSE;
	part_idx_read();
	imsg("part: after init, upto %d in bucket %d of %d\n",dbupto,dbbkt,ndbbkt); 
	let_thru = lib_gmtime() + 60*15;
	if (lib_exists(PART_OLD)) {
		let_thru = 0;		
	} else {
		f = fopen(PART_OLD,"w");
		if (f!=NULL) fclosep(f);
	}
	return TRUE;
}
static FILE *fc;

char *part_file(int bkt)
{
        static char bf[BFSZ];
	
	if (strlen(config_get(C_PART_DIR))==0) {
		emsg("part_dir must be defined, and the directory must exist!\n");
		dnews_ex("part_dir not defined");
	}
        lprintf(bf,BFSZ-1,"%spart%d.dat",config_getu(C_PART_DIR),bkt);
        return bf;
}


int part_file_init(void)
{
	/* Create the directory */
	/* If the files dont exist, create them all at the specified size */
	int i;
	int start;
	int elap;
	FILE *f;

	dbend = 100000000;
	part_disable = FALSE;
	if (isfast) return TRUE;
	if (strlen(config_get(C_PART_DIR))==0) {
		imsg("part: part processing disabled as part_dir not defined (this is ok)\n");
		part_disable = TRUE;
		return FALSE;
	}

	ndbbkt = atoi(config_get(C_PART_BUCKETS));
	if (ndbbkt==0) {
		ndbbkt = 3;
	}
	if (ndbbkt>=MAXDBBKT) ndbbkt = MAXDBBKT-1;
	for (i=0; i<ndbbkt; i++) {
		bktid[i] = 0;
	}
	for (i=0; i<ndbbkt; i++) {
		start = lib_gmtime();

		f = fopen(part_file(i),"r");
		if (f!=NULL) {fclosep(f); continue;}

		part_file_create(i);
#ifdef WIN32
		elap = time(NULL) - start;
		for (;elap<3;) {
			imsg("Limiting bucket creation speed to 30mb/second to avoid blowing badly designed operating systems\n");
			lib_sleep(1);
			elap = lib_gmtime() - start;
		}
#endif
	}
	return TRUE;
}
int part_file_create(int bkt)
{
	FILE *f;
	if (isfast) return TRUE;
	f = fopen(part_file(bkt),"r");
	if (f!=NULL) {fclosep(f); return TRUE;}
	lib_createdir(part_file(0));
	wmsg("part: Creating part bucket, may take some time... %s\n",part_file(bkt));
	f = fopen(part_file(bkt),"w+b");
	if (f==NULL) {filemsg("part: Could not create %s\n",part_file(bkt)); dnews_ex("part_create"); abort();}
	if (fseek(f,dbend,SEEK_SET)!=0) {
		filemsg("part: Could not extend bucket to 100mb %s\n",part_file(bkt));
		fclosep(f);
		remove(part_file(bkt));
		dnews_ex("part_extend_failed");
		abort();
	}
	if (fwrite("1",1,1,f)!=1) {
		filemsg("part: Could not extend bucket to 100mb %s\n",part_file(bkt));
		remove(part_file(bkt));
		dnews_ex("part_extend_failed");
		abort();
	}
	if (fclosep(f)!=0) {
		filemsg("part: 2 Could not extend bucket to 100mb %s\n",part_file(bkt));
		remove(part_file(bkt));
		dnews_ex("part_extend_failed");
		abort();
	}
	imsgf("part: Created bucket successfully \n");
	return TRUE;	
}
static int cur_bkt= -1;
int part_open(int bkt)
{
	char fname[BFSZ];
	static char xbf[33000];
	if (fc!=NULL) if (cur_bkt==bkt) return TRUE;
	if (fc!=NULL) {
		fclosep(fc);
		fc = NULL;
	}

	strcpy(fname,part_file(bkt));
	fc = fopen(fname,"r+b");
	if (fc==NULL) {
		filemsg("Could not open (%s) \n",fname);
		return FALSE;
	}
        setvbuf(fc,NULL,_IOFBF,32767);
	cur_bkt = bkt;	
	return TRUE;
}
static FILE *lastf;
FILE *part_open_f(int bkt)
{
	FILE *f;
	char fname[BFSZ];
	static int lastbkt;
	if (lastbkt==bkt) if (lastf!=NULL) return lastf;
	if (lastf!=NULL) fclose(lastf); lastf = NULL;
	strcpy(fname,part_file(bkt));
	f = lastf = fopen(fname,"rb");
	if (f==NULL) {
		tmsg("Could not open (%s) %s\n",fname,strerror(errno));
		return FALSE;
	}
        setvbuf(f,NULL,_IOFBF,32767);
	return f;
}
int part_close_f(void)
{
	if (lastf!=NULL) fclose(lastf);
	lastf = NULL;
	return TRUE;
}
int part_findgap(int len)
{
	/* If would go past end then move onto next bucket */
	/*imsg("part: findgap (%d)  upto=%d  dbbkt=%d\n",len,dbupto,dbbkt);*/
	if (dbupto+len>dbend) {
		part_sync("findgap");
		dbupto = 0;
		dbbkt++;
		if (dbbkt>=ndbbkt) dbbkt = 0;
		bktid[dbbkt]++;
		imsg("part: Moving onto next bucket %d  %d+%d>%d\n",dbbkt,dbupto,len,dbend);
		part_clear_bkt(dbbkt);
		part_index_rewrite();
	}
	return TRUE;
}
int part_clear_oldest(void)
{
	int h;
	Part *p;
	Parts *pts;
	int t;
	int bkt = -1;
	int bkt_age=time(NULL)+100000;

	if (!isfast) return TRUE;
	imsg("part: search for oldest bucket to remove from index\n");
	/* Find oldest bucket we are using in our index */ 
	for (h=0;h<PART_HASH;h++) {
		for (pts=ptable[h];pts!=NULL;pts=pts->next) {
			for (p=pts->parts;p!=NULL;p=p->next) {
				t = db_get_age(p->bkt);
				if (t<bkt_age) {
					bkt = p->bkt;
					bkt_age = t;
				}
			}
		}
	}
	if (bkt<0) {
		emsg("part_clear_oldest unable to find oldest bucket in index \n"); 
		return FALSE;
	}	
	if (bkt>=0) {
		imsg("part: Trimming items in bkt %d age %d\n",bkt,bkt_age);
		age_lost = time(NULL) - bkt_age;
		part_clear_bkt(bkt);
		part_index_rewrite();
	}
	return TRUE;
}
int part_clear_bkt(int bkt)
{
	int h;
	int killit;
	Part *p;
	Parts *pts,*pnext,**last;
	int done=FALSE;
	int nkill=0;

	if (bkt==0) time_zero = time(NULL);

	for (h=0;h<PART_HASH;h++) {
		last = &ptable[h];
		for (pts=ptable[h];pts!=NULL;pts=pnext) {
			pnext = pts->next;
			killit = FALSE;
			done = FALSE;
			for (p=pts->parts;p!=NULL;p=p->next) {
				if (p->bkt==bkt) {
					if (!done) part_lost(pts,p);
					done = TRUE;
					killit = TRUE;
				}
				if (p->len==0) killit = TRUE;
			}
			if (killit) {
				nkill++;
				*last = pts->next;
				stat_dropsize += parts_free_parts(pts,TRUE);
			} else {
				last = &(pts->next);
			}
		}
	}
	return TRUE;
}
int part_lost(Parts *pts, Part *pone)
{
	char gotit[1000];
	int maxparts;
	Part *p = pts->parts;
	int i;
	int have=0;
	static int tot;
	int miss=0;
	static FILE *f;

	if (p==NULL) return FALSE;
	maxparts = p->maxparts;

	for (p=pts->parts;p!=NULL;p=p->next) {
		if (maxparts<p->maxparts) maxparts = p->maxparts;
	}

	if (maxparts>900) return TRUE;
	
	for (i=0; i<=maxparts; i++) gotit[i] = FALSE;
	for (p=pts->parts;p!=NULL;p=p->next) {
		if (p->part<0) continue;
		if (p->part>maxparts) continue;
		have++;
		gotit[p->part] = TRUE;
	}
	for (i=1; i<=maxparts; i++) if (!gotit[i]) miss++;
	
	if (f==NULL) f = fopen(file_log("partlost.log"),"w");
	if (f==NULL) return TRUE;

	if (miss==0) return TRUE; /* Nothing missing */
	fprintf(f,"%d/%d/%d %.300s %.100s\n",miss,have,maxparts,pts->subj,pts->mid);
	tot+= 100;
	if (tot>1000000) {
		fclose(f); f = NULL;
		tot = 0;
	}
	return TRUE;
}
int part_showindex(int chan)
{
	int h;
	Part *p;
	Parts *pts;
	char bf[BFSZ];
	for (h=0;h<PART_HASH;h++) {
		for (pts=ptable[h];pts!=NULL;pts=pts->next) {
			lprintf(bf,BFSZ-1,"Item %.70s\r\n",pts->subj); chan_write_s(chan,bf);
			for (p=pts->parts;p!=NULL;p=p->next) {
			 lprintf(bf,BFSZ-1,"Part %d/%d at %d len %d bucket %d\r\n"
				 ,p->part,p->maxparts,p->offset,p->len,p->bkt); 
			 chan_write_s(chan,bf);
			}
		}
	}
	return TRUE;
}
static char *outbf;
static int outsz,outlen,outat,outsync;
static FILE *outf;
static int th_dbbkt = -1;
int part_sync(char *because)
{
	int r=TRUE;
	int start = lib_msec();
//	imsg("part_sync line %s\n",because);
	mutex_wait(M_PART);	
	if (outlen!=0) outsync = TRUE;
	else r=FALSE;
	mutex_release(M_PART);	

	for (;outsync;) lib_msleep(1); /* Wait until it is synced */
	
	mutex_wait(M_PART);	
	if (outf!=NULL) fflush(outf);
	mutex_release(M_PART);	
//	imsg("part_sync done %dmsec %s\n",lib_msec()-start,because);
	return r;
}
int part_thread(void *x)
{
	char *s;
	int sync=FALSE;
	int len;
//	lib_boost_thisthread(); /* So the buffer is always empty with luck .*/
	lib_boost_thisthread(); /* So the buffer is always empty with luck .*/
	for (;TRUE;) {
		mutex_wait(M_PART);
		if (outlen==outat) {
			outat = 0; outlen = 0;	
			outsync = FALSE;
		}
		if (outlen>outat) {
			s = outbf+outat;
			len = outlen-outat;
			if ((outlen+100000)>outsz) outsync = TRUE; /* Force a cycle near the end */
			sync = outsync;
			if (!sync) mutex_release(M_PART);
			fwrite(s,1,len,outf);
			if (sync) nsync++; else nasync++;
			if (sync) {
				outat = 0; outlen = 0;	
				outsync = FALSE;
			} else {
				mutex_wait(M_PART);
				outat += len;
				if (outat==outlen) { /* cycle around if we can */
					outat = 0; outlen = 0;	
					outsync = FALSE;
				}
			}
		}
		if (outlen==outat) {
			outat = 0; outlen = 0;	
			outsync = FALSE;
		}
		mutex_release(M_PART);
		lib_msleep(20);
	}
	return TRUE;
}
int part_open_write(int bkt)
{
	char fname[BFSZ];
	strcpy(fname,part_file(bkt));
	if (outf!=NULL) fclose(outf);
	outf = fopen(fname,"r+b");
	if (outf==NULL) {
		filemsg("Could not open (%s)\n",fname);
		return FALSE;
	}
	fseek(outf,dbupto,SEEK_SET);
        setvbuf(outf,NULL,_IOFBF,128000);
	return TRUE;
}
int part_store_thread(Part *p, char *item,int len) 
{
	static Thread *t;
	if (dbbkt!=th_dbbkt) {
		part_sync("store_thread_bucket_change");
		if (!part_open_write(dbbkt)) return FALSE;
		th_dbbkt = dbbkt;
	}
	mutex_wait(M_PART);
	if (outbf==NULL) {
		outsz = conf_item_max()+102000;
		if (outsz<10000000) outsz = 10000000;
		outbf = mymalloc(outsz+100);
#ifdef DOTHREADS
		t = th_new(part_thread,NULL);
		if (t==NULL) {
			emsg("Thread part creation failed %s\n",strerror(errno));
			return FALSE;
		}
		imsg("part: Output thread created \n");
		t = th_new(part_thread_complete,NULL);
		if (t==NULL) {
			emsg("Thread part creation failed %s\n",strerror(errno));
			return FALSE;
		}
		imsg("part: Output2 thread created \n");
#endif
	}
	if (len>=outsz) goto failed;
	if ((outlen + len)>=outsz) {
		mutex_release(M_PART);
		part_sync("need more space");
		mutex_wait(M_PART);
	}
	if ((outlen + len)>=outsz) goto failed;
	memcpy(outbf+outlen,"{",1);
	outlen++;
	memcpy(outbf+outlen,item,len);
	outlen += len;
	memcpy(outbf+outlen,"}",1);
	outlen++;
	p->offset = dbupto;
	p->len = len+2;
	p->bkt = dbbkt;
	dbupto += p->len;
failed:
	mutex_release(M_PART);

	return TRUE;
}
int part_store(Part *p, char *item,int len) 
{
	if (config_true(C_PART_THREAD)) return part_store_thread(p,item,len);
	if (!part_open(dbbkt)) return FALSE;
	if (fseek(fc,dbupto,SEEK_SET)!=0) goto failed;
	p->offset = dbupto;
	p->len = len+2;
	p->bkt = dbbkt;
	if ((int) fwrite("{",1,1,fc)!=1) goto failed;
	if ((int) fwrite(item,1,len,fc)!=len) goto failed;
	if ((int) fwrite("}",1,1,fc)!=1) goto failed;
/*	if (fflush(fc)!=0) goto failed;*/
	dbupto += p->len;
	return TRUE;
failed:
	return FALSE;
}
#define FILE_PART_IDX file_work("part.idx")
#define FILE_PART_IDX_NEW file_work("part.idx_new")
static  FILE *fidx;
typedef struct PIDX {int bkt; int offset; int len; int maxparts; int part; int dbupto; int dbclear; int dbbkt; short subj_len; short mid_len; char ver;  } Pidx;
int part_idx_open(void)
{
	fidx = fopen(FILE_PART_IDX,"ab");
	if (fidx==NULL) {
		emsg("couldn't open %s\n",FILE_PART_IDX);
		return FALSE;
	}
	return TRUE;
}
int part_idx_add(Part *p, char *subj, char *mid)
{
	Pidx idx;
	if (fidx==NULL) if (!part_idx_open()) return FALSE;
	idx.ver = 10; /* Version */
	idx.offset = p->offset;
	idx.len = p->len;
	idx.part = p->part;
	idx.bkt = p->bkt;
	idx.maxparts = p->maxparts;
	idx.dbupto = dbupto;
	idx.dbclear = dbclear;
	idx.dbbkt = dbbkt;
	idx.subj_len = strlen(subj) ;
	idx.mid_len = strlen(mid) ;
	if (idx.subj_len>300) idx.subj_len=300;
	if (idx.mid_len>300) idx.mid_len=300;
	fwrite(&idx,sizeof(Pidx),1,fidx);
	fwrite(subj,1,idx.subj_len,fidx);
	fwrite(mid,1,idx.mid_len,fidx);
	
	return TRUE;
}
int part_idx_add_new(Part *p, char *subj, char *mid)
{
	Pidx idx;
	if (fidx==NULL) if (!part_idx_open()) return FALSE;
	idx.ver = 11; /* Version */
	idx.offset = p->offset;
	idx.len = p->len;
	idx.part = p->part;
	idx.bkt = p->bkt;
	idx.maxparts = p->maxparts;
	idx.dbupto = p->grp;
	idx.dbclear = p->num;
	idx.dbbkt = dbbkt;
	idx.subj_len = strlen(subj) ;
	idx.mid_len = strlen(mid) ;
	if (idx.subj_len>300) idx.subj_len=300;
	if (idx.mid_len>300) idx.mid_len=300;
	fwrite(&idx,sizeof(Pidx),1,fidx);
	fwrite(subj,1,idx.subj_len,fidx);
	fwrite(mid,1,idx.mid_len,fidx);
	
	return TRUE;
}
char fstrdup(FILE *f, int len);
int part_idx_read(void)
{
	Pidx idx;
	FILE *f;
	Parts *pts;
	char subj[BFSZ],mid[BFSZ];
	Part *p;
	int h;
	int nbad = 0;
	int n=0;
	int nold=0;
	int nnew=0;
	double tot=0;
	imsg("part: Reading part index\n");
	f = fopen(FILE_PART_IDX,"rb");
	if (f==NULL) return TRUE;
	
	for (;!feof(f);) {
		if (fread(&idx,sizeof(idx),1,f)<1) break;
		if (idx.subj_len>BFSZ-1) idx.subj_len = BFSZ -1;
		if (idx.mid_len>BFSZ-1) idx.mid_len = BFSZ -1;
		if (idx.subj_len<0) idx.subj_len = 0;
		if (idx.mid_len<0) idx.mid_len = 0;
		if (idx.subj_len>0) if (fread(subj,1,idx.subj_len,f)<1) {
			filemsg("part: subject read failed %d\n",idx.subj_len);
			break;
		}
		if (idx.mid_len>0) if (fread(mid,1,idx.mid_len,f)<1) {
			filemsg("part: mid read failed\n");
			break;
		}
		if (idx.ver==10) {
			if (isfast) {nold++; continue;}
			subj[idx.subj_len] = 0;
			mid[idx.mid_len] = 0;
			h = str_hash(subj,PART_HASH);
			pts = parts_find_add(h,subj,mid);
			p = mymalloc(sizeof(Part)); memset(p,0,sizeof(Part));
			stat_nparts++; stat_partsize += idx.len;
			p->next = pts->parts;
			pts->parts = p;
			p->offset = idx.offset;
			p->len = idx.len;
			p->part = idx.part;
			p->bkt = idx.bkt;
			p->maxparts = idx.maxparts;
			dbupto = idx.dbupto;
			dbbkt = idx.dbbkt;
			dbclear = idx.dbclear;
		} else 	if (idx.ver==11) { /* Index only */
			if (!isfast) {nnew++;  continue;}
			subj[idx.subj_len] = 0;
			mid[idx.mid_len] = 0;
			h = str_hash(subj,PART_HASH);
			pts = parts_find_add(h,subj,mid);
			p = mymalloc(sizeof(Part)); memset(p,0,sizeof(Part));
			stat_nparts++; stat_partsize += idx.len;
			p->next = pts->parts;
			pts->parts = p;
			p->offset = idx.offset;
			p->len = idx.len;
			p->part = idx.part;
			p->bkt = idx.bkt;
			p->maxparts = idx.maxparts;
			p->grp = idx.dbupto;
			p->num = idx.dbclear;
			dbupto = idx.dbupto;
			dbbkt = idx.dbbkt;
			dbclear = idx.dbclear;
		} else nbad++;
		n++;
		tot += idx.len;
	}
	fclosep(f);
	imsg("part: read %d/%d parts %d bad, %d old, %d new, into index total %.2fmb/%dmb\n",stat_nparts,n,nbad,nold,nnew,tot/1000000,ndbbkt*dbend/1000000);
	return TRUE;
}
int part_index_rewrite(void);

int part_index_rewrite(void)
{
	Pidx idx;
	FILE *f;
	int h;
	char fname[BFSZ];
	Parts *pts;
	Part *p;
	int n=0;
	int failed=FALSE;
	int np=0;
	if (fidx!=NULL) fclosep(fidx);
	fidx = NULL;
	part_close_f();

	f = fopen(FILE_PART_IDX_NEW,"wb");
	if (f==NULL) {	filemsg("part: Could not create (%s)\n",FILE_PART_IDX_NEW); return FALSE; }
	for (h=0;h<PART_HASH;h++) {
	  for (pts=ptable[h];pts!=NULL;pts=pts->next) {
		  for (p=pts->parts; p!=NULL; p = p->next) {
			idx.ver = 10; /* Version */
			idx.offset = p->offset;
			idx.len = p->len;
			idx.part = p->part;
			idx.bkt = p->bkt;
			idx.maxparts = p->maxparts;
			idx.dbupto = dbupto;
			idx.dbbkt = dbbkt;
			idx.dbclear = dbclear;
			idx.subj_len = strlen(pts->subj) ;
			idx.mid_len = strlen(pts->mid) ;
			if (isfast) {
				idx.ver=11;  
				idx.dbupto = p->grp;
				idx.dbclear = p->num;
			}
			if (idx.subj_len>300) idx.subj_len=300;
			if (idx.mid_len>300) idx.mid_len=300;
			fwrite(&idx,sizeof(Pidx),1,f);
			fwrite(pts->subj,1,idx.subj_len,f);
			if (fwrite(pts->mid,1,idx.mid_len,f)<1) failed=TRUE;
			n++;
		  }
		  np++;
	  }
	}
	if (fclosep(f)!=0) failed=TRUE;
	if (failed) {
		filemsg("Writing part index failed - is workarea disk full? \n");
		return FALSE;
	}
	strcpy(fname,FILE_PART_IDX);
	if (remove(fname)!=0) filemsg("part: Remove of (%s) failed\n",fname);
	if (rename(FILE_PART_IDX_NEW,fname)!=0) filemsg("part: Rename to (%s) failed\n",fname);
	imsg("part: Re-written file part index with %d items %d parts\n",np,n);
	return TRUE;
}
/* Send in the parts in order just to  be fancy */
static Parts *send_que;
int parts_send_tothread(Parts *pts)
{
	Parts *p;
	/* Add to the queue of items to be sent in */
	mutex_wait(M_PART_SEND);
	if (nsend<1000) {
		p = pts_dup(pts);
		p->next = send_que;
		send_que = p;
		nsend++;
	} else {
		wmsg("Dropping item as part send queue is full\n");
		ndrop++;
	}
	mutex_release(M_PART_SEND);
	return TRUE;
}
int part_thread_complete(void *x)
{
	int sync=FALSE;
	Parts *p;
	lib_boost_thisthread(); /* So the buffer is always empty with luck .*/
	for (;TRUE;) {
		mutex_wait(M_PART_SEND);
		if (nsend==0) {
			lib_msleep(20);
			mutex_release(M_PART_SEND);
			continue;
		}
		p = send_que;
		send_que = send_que->next;
		nsend--;
		mutex_release(M_PART_SEND);
		if (!part_thread_send(p)) {
			mutex_wait(M_PART_SEND);
			tmsg("part: failed multi part post \n");
			ndrop++;
			mutex_release(M_PART_SEND);
		}
		mutex_wait(M_MAIN);
		parts_free(p);
		mutex_release(M_MAIN);
	}
	return TRUE;
}
int parts_free(Parts *pts)
{
	Part *p,*pnext;
	int t = 0;

	for (p=pts->parts;p!=NULL;p=pnext) {
		pnext = p->next;
		myfree(p);		
	}
	mystrfree(pts->subj,SD_PARTS);
	mystrfree(pts->mid,SD_PARTS);
	pts->subj = NULL;
	pts->mid = NULL;
	myfree(pts);
	return t;
}
int part_thread_send(Parts *pts)
{
	int max = pts->parts->maxparts;
	int n=0;
	int nok=0;
	int i;
	int donesync = FALSE;
	Part *p;
	int ret= TRUE;
	for (p=pts->parts; p!=NULL; p=p->next) {
		if (p->bkt<0) ret = FALSE;
		if (p->bkt>=ndbbkt) ret = FALSE;
		if (p->bktid != bktid[p->bkt]) {
			tmsg("bktid has changed [%d] %d != %d\n",p->bkt,p->bktid,bktid[p->bkt]);
			return FALSE;
		}
		if (!ret) {
			tmsg("Range check failed\n");
			return FALSE;
		}
	}
	/* 
		Now read each one from disk, in order, then
		submit after grabbing main mutex 
	*/
	for (p=pts->parts;p!=NULL;p=p->next) {
		if (max<p->maxparts) max = p->maxparts;
	}
	
	for (i=0; i<=max; i++) {
		for (p=pts->parts;p!=NULL;p=p->next) if (p->part==i) break;
		if (p!=NULL) {
			if (part_send_one(p,TRUE,&donesync)) nok++; 
			n++;
		}
	}
	tmsg("part: parts_complete, Stored %d parts of %d\n",n,max);
	if (nok>0) if (nok != n) {
		stat_nfailed++;
		tmsg("part: Post only partially successful %d/%d/%d\n",nok,n,max);
	}
	return TRUE;

	return ret;
}
Parts *pts_dup(Parts *pts)
{
	Parts *head=NULL;
	Part *p=NULL;
	Part *n;
	Part **last;
	
	head = mymalloc(sizeof(Parts));
	*head = *pts;
	head->subj = mystrdup(pts->subj,SD_PARTS);
	head->mid = mystrdup(pts->mid,SD_PARTS);
	head->next = NULL;
	head->parts = NULL;
	last = &(head->parts);
	for (p=pts->parts; p!=NULL; p=p->next) {
		n = mymalloc(sizeof(Part));
		*n = *p;
		if (n->bkt<0) n->bkt = 0;
		if (n->bkt>=ndbbkt) n->bkt = 0;
		n->bktid = bktid[n->bkt];
		n->next = NULL;
		*last = n;
		last = &n->next;
	}
	return head;
}
int parts_send(Parts *pts)
{
	Part *p;
	int max = pts->parts->maxparts;
	int n=0;
	int nok=0;
	int i;
	int donesync = FALSE;

	if (config_true(C_PART_THREAD)) return parts_send_tothread(pts);
	for (p=pts->parts;p!=NULL;p=p->next) {
		if (max<p->maxparts) max = p->maxparts;
	}
	
	for (i=0; i<=max; i++) {
		for (p=pts->parts;p!=NULL;p=p->next) if (p->part==i) break;
		if (p!=NULL) {
			if (isfast) {
				part_send_one_idx(p); nok++;
			} else {
				if (part_send_one(p,FALSE,&donesync)) nok++; 
			}
			n++;
		}
	}
	dmsg("part: parts_complete, Stored %d parts of %d\n",n,max);
	if (nok>0) if (nok != n) {
		stat_nfailed++;
		imsg("part: Post only partially successful %d/%d/%d\n",nok,n,max);
		partmsg("part: Post only partially successful %d/%d/%d %.80s\n",nok,n,max,pts->subj);
	}
	return TRUE;
}
void hist_set_part(int x);
int part_send_one(Part *p, int isthread, int *donesync)
{
	String *xitem;
	static char *mem;
	static int memsz;
	FILE *f;
	if (p->bkt<0) return FALSE;
	if (isthread && (p->bktid != bktid[p->bkt])) {
		tmsg("Bucket has been rotated bugger\n");
		return FALSE;
	}
	if (!(*donesync)) if (p->bkt==th_dbbkt) {
		*donesync = TRUE;
		part_sync("Sending parts into real db");

	}
	if (mem==NULL) {
		mutex_wait(M_MAIN);
		memsz = conf_item_max()+102000;
		mem = mymalloc(memsz+100);
		mutex_release(M_MAIN);	
	}

	f = part_open_f(p->bkt);
	if (f==NULL) {
		tmsg("part: Cannot send part as bucket file disappeared %s\n",strerror(errno));
		part_close_f();
		return FALSE;
	}
	if (fseek(f,p->offset,SEEK_SET)!=0) {
		tmsg("part: Cannot send part as seek failed\n");
		goto failed;
	}
	if (p->len<0) { tmsg("FATAL ERROR, part length less than zero\n");goto failed;}
	if (p->len+10>=memsz) {tmsg("FATAL ERROR, part length is too large %d\n",p->len); goto failed;}
	if ((int) fread(mem,1,p->len,f)!=(p->len)) {
		tmsg("part: Cannot send part as fread2 failed\n"); goto failed;
	}
	if (mem[0]!='{') {tmsg("Header gone\n"); goto failed;}
	if (mem[p->len-1]!='}') {tmsg("tail gone\n"); goto failed;}
	mutex_wait(M_MAIN);
	part_inprogress = TRUE;
	xitem = str_new();
        str_setlen(xitem,p->len);
	memcpy(str_s(xitem),mem+1,p->len-1);
	str_setlen(xitem,p->len-2);
	hist_set_part(TRUE);
	if (!item_suck(xitem)) {
		hist_set_part(FALSE);
		imsg("part: One part of multi part message rejected %.200s\n",msg_reason());
		partmsg("part: One part of multi part message rejected %.200s\n",msg_reason());
		stat_nrej++;
		goto failed2;
	} else {
		hist_set_part(FALSE);
		dmsg("part: Sent part %d/%d ok\n",p->part,p->maxparts);
		stat_okitems++;
		stat_oksize+= p->len;
	}	
	str_free(xitem); xitem = NULL;
	part_inprogress = FALSE;
	mutex_release(M_MAIN);
	part_close_f();
	return TRUE;
failed2:
	part_inprogress = FALSE;
	str_free(xitem); xitem = NULL;
	mutex_release(M_MAIN);
failed:
	part_close_f();
	return FALSE;
}


void partmsg(char * arg_list, ...)
{
        va_list arg_ptr;
        char *format;  /* Shouldn't be necessary */
	char output[9000];
	FILE *f;
	time_t t;
        va_start(arg_ptr, arg_list);
        format = arg_list;
        vsnprintf(output, BFSZ-1, format, arg_ptr);
        output[BFSZ-1] = 0;

	f = fopen(file_log("parterr.log"),"a");
	if (f==NULL) return;
	time(&t);
	fprintf(f,"%.19s: s%d %s",ctime(&t),slave_n(),output);
	fclosep(f);
	if (lib_length_file(file_work("parterr.log"))>200000) {
		remove(file_work("parterr.log"));
	}
}