/*
reset dbend
fix fault with fflush() it is null, shouldn't be possiblem ake it not possible

  when we close a part file must first wait for it to be flushed.
  add function to start thread.
  make it a fifo!!!

  ....................................



todo:	
rewrite part_thread_send
	flush any output file we need before opening input file of same bucket.
	remove 15 minute delay
	remove creation of buckets on startup.

---------------------------------------------------------------------------------------
  
    The magical hiding of missing parts functions :-)

Hash on subject, 2000 entries.
Write index with Position/Length/Partno,
Also in index, writepoint,totallen,erasedpoint
Read index on startup.
Extend file if necessary.

What happens to existing items that have partially arrived? 
	Command to run 20 minutes after starting it, to send all
	stored items into the server. (background job?)

Add a command to delete partial posts not in the last 100 items.
Must have 'n' buckets, as 2 gig isn't enough.
	
	b1,b2,b3,b4,b5
	Expire all in next bucket.
	If index is full, then, expire bucket
	If 


*/
#ifdef __MWERKS__ /*RHP we will take this to mean we are making a Mac version*/
#include "mac.h"
#endif

#include <stdlib.h>
#include <stdio.h>
#include <ctype.h>
#include <string.h>
#include <assert.h>
#ifdef VMS
#include <unixio.h>
#define unlink remove
#endif
#ifndef NO_FCNTL
#include <fcntl.h>
#else
/*RHP not for mac */
#ifndef Mac
#include <file.h>
#endif
#endif

#ifdef WIN32
#include <windows.h>
#endif
#include <errno.h>
#include "thread.h"

#include "emsg.h"
#include "sary.h"
#include "str.h"
#include "lib.h"
#include "group.h"
#include "hist.h"
#include "suck.h"
#include "chan.h"
#include "feed.h"
#include "expire.h"
#include "control.h"
#include "item.h"
#include "db.h"
int ttmsg(const char * arg_list, ...);

typedef struct PART {
	int bkt; int bktid; int offset; int len; int maxparts; int part;
	int grp,num; 
	struct PART *next;} Part;
typedef struct PARTS {char *subj; char *mid; Part *parts; struct PARTS *next; } Parts;
typedef struct PFILE {
	FILE *f; int upto; int bkt; 
	char *bf;
	int bfsz;
	int bfat;
	char *bfout;
	int flushed;
	int flush;
	int busy; /* file handle may be in use */
} Pfile;
static Pfile pfiles[20];

#include "parts.h"
#undef mymalloc
#undef myfree
#define mymalloc(x) mymalloc2(x,M_PARTS)
#define myfree(x) myfree2(x,M_PARTS)
Pfile *part_findgap(int len);
int part_close_all(void);
int part_store(Pfile *pf, Part *p, char *item,int len) ;
int part_storing(void *x);

int part_open_all(void);
int part_next_set(void);
int part_thread_send(Parts *pts);
static int nsend,ndrop;
static int dbclear,dbend,dbbkt,ndbbkt;
#define MAXDBBKT  2000
static int bktid[MAXDBBKT];
static int part_inprogress = FALSE;
static int part_disable=TRUE;
#define PART_HASH 12000
static Parts *ptable[PART_HASH];
static int stat_nparts;
static double stat_partsize;
static int stat_initems;
static double stat_insize;
static int stat_okitems;
static int stat_nrej;
static double stat_oksize;
static double stat_dropsize;
static int stat_nfailed,stat_readfail;
static int nhist;
Parts *pts_dup(Parts *pts);
static int time_zero;
void partmsg(char * arg_list, ...);
int part_clear_oldest(void);
int part_idx_add_new(Part *p, char *subj, char *mid);
int part_lost(Parts *pts, Part *pone);
Parts *parts_find_only(int h, char *subj, char *mid);
int part_send_one(Parts *pts, Part *p, int isthread, int *donesync);
int part_thread_complete(void *x);
int parts_free(Parts *pts);
int part_swap(void);
#include "mutex.h"

static int nsync,nasync;
int part_age(void);
static int age_lost;
char *part_status(void)
{
	static char bf[BFSZ];
	if (part_disable) return "";
	lprintf(bf,BFSZ-1,"Parts %d:%d %.0fmb, In %d %.0fmb, Complete %d %.0fmb, %.0f%% %.0f%%, Rej %.0f%%\r\nDrop %.0fmb, Err %d errread %d Rotated %d minutes ago, Upto %d %d, Thread send %d drop %d, sync %d/%d\r\n"
		,stat_nparts,nhist,stat_partsize/1000000
		,stat_initems,stat_insize/1000000
		,stat_okitems,stat_oksize/1000000
		,stat_okitems*100.0/(stat_initems+1)
		,stat_oksize*100.0/(stat_insize+1)
		,stat_nrej*100.0/(stat_initems+1)
		,stat_dropsize/1000000.0
		,stat_nfailed,stat_readfail
		,part_age()/60,dbbkt,ndbbkt,nsend,ndrop
		,nsync,nasync);	
	return bf;
}
int part_age(void)
{
	/* Find the diff in age between dbbkt and dbbkt + 1 */
	int next=dbbkt+1;
	int last = dbbkt-1;

	if (next>=ndbbkt) next=0;
	if (last<0) last=0;
	imsg("part_age last,  %s %d\n",part_file(last),lib_cdate(part_file(last)));
	imsg("part_age next, %s %d\n",part_file(next),lib_cdate(part_file(next)));
	return lib_cdate(part_file(last)) - lib_cdate(part_file(next));
}
typedef struct HPARTS {Parts *parts; struct HPARTS *next; } Hparts;
static Hparts *htable[PART_HASH];

int part_hist(char *mid)
{
	int h;
	Hparts *p;

	if (part_disable) return FALSE;
	h = str_hash(mid,PART_HASH);
	for (p=htable[h];p!=NULL;p=p->next) {
		if (strcmp(mid,p->parts->mid)==0) return TRUE;
	}
	return FALSE;
}
int part_hist_add(Parts *pts)
{
	Hparts *p;
	int h = str_hash(pts->mid,PART_HASH);
	p = mymalloc(sizeof(Hparts));
	memset(p,0,sizeof(Hparts));
	p->parts = pts;
	p->next = htable[h];
	nhist++;
	htable[h] = p;
	return TRUE;
}
int part_hist_del(Parts *pts)
{
	Hparts *p,**last;
	int h = str_hash(pts->mid,PART_HASH);
	last = &(htable[h]);
	for (p=htable[h];p!=NULL;p=p->next) {
		if (pts==p->parts) {
			*last = p->next;
			myfree(p);
			nhist--;
			return TRUE;
		}
		last = &(p->next);
	}
	emsg("Could not find history parts entry to free\n");
	dnews_ex("no history parts entry");
	return FALSE;	
}
#define INDEX_LIMIT 10000
int part_index_size(void)
{
	static int x;
	int n;
	if (x>0) return x;
	x = atoi(config_get(C_PART_INDEX));
	if (x==0) {
		n = atoi(config_get(C_PART_BUCKETS));
		x = INDEX_LIMIT;
		if (n>50) x = 20000;
		if (n>100) x = 40000;
	}
	return x;
}

static int store_write;
static Parts *store_pts;
static Part *store_p;
#define LOGIN file_log("partin.log")
int part_log_in(char *str)
{
	static FILE *f;
	if (f==NULL) f = fopen(LOGIN,"a");
	if (f==NULL) return FALSE;
	if (strstr(str,"rar")==NULL) return FALSE;
	fprintf(f,"%s",str);
	if (file_length(fileno(f))>10000000) {
		fclose(f);
		f = NULL;
		remove(LOGIN);
	}
	return TRUE;
}
int part_check(char *xsubject, char *mid, char *item, int len)
{
	/* Check if it's a multi part post, if it is process it */ 
	/* If we are processing one, then return false to avoid loop */
	/* Find parts, and add this one, if we now have the 'set' then process them */
	static int last;
	char subj[BFSZ];
	char subject[BFSZ];
	char fname[BFSZ];
	int part,partmax;
	int h,j;
	Pfile *pf;
	char tmp[BFSZ];
	Parts *pts;
	int gap;
	Part *p;
	char subjectlwr[BFSZ];

	store_write = FALSE;
	if (part_inprogress) return FALSE; 
	if (part_disable) return FALSE;
	ncpy(subject,xsubject,302);
	if (!parts_subject(subject, subj, fname, &part, &partmax)) return FALSE;
	if (strlen(subj)>299) return FALSE;
	if (partmax<2) return FALSE; /* No need eh :-) */
	if (strncmpnc(subject,"re: ",4)==0) return FALSE; /* WILL miss some but better safe */
	ncpy(subjectlwr,subject,BFSZ-1);
/*
Doesn't make sense, as reposts are junk anyway, we don't want them.
	strlwr(subjectlwr);
	if (strstr(subjectlwr,"repost")!=NULL) return FALSE;
	*/

	if (config_true(C_PART_VANISH)) {
	    int max = atoi(config_get(C_PART_VANISH_MAX));
	    if (partmax > max) {
		imsg("Vanishing multi part item %d parts, max is %d, %s\n",partmax,max,subj);
		return TRUE;
	    } else {
		imsg("Not vanishing %d max is %d\n",partmax,max);
	    }
	}
		
//imsg("part: Incoming multi part message %d/%d  %.180s\n",part,partmax,xsubject);
	dmsg("part: Incoming multi part message %d/%d  %.180s\n",part,partmax,xsubject);
	lprintf(tmp,BFSZ-1,"%d/%d %.300s\n",part,partmax,xsubject);
	part_log_in(tmp);
	/* Ok, here we have a part of a multi part message */

	h = str_hash(subj,PART_HASH);
	if (part==0) {
		pts = parts_find_only(h,subj,mid);
		if (pts==NULL) {
			/* Zeroth part, might have arrived 'after' the rest, so send it through */	
			return FALSE; 
		}
	}
	dmsg("part: find gap now\n");
	/* Find gap first as finding gap may free the part we are trying to complete, bad...*/
	pf = part_findgap(len);
	dmsg("part: parts_find_add\n");

	pts = parts_find_add(h,subj,mid);
	dmsg("part: parts_find_add-done\n");
	p = mymalloc(sizeof(Part)); memset(p,0,sizeof(Part));
	p->bkt = 0;
	p->grp = 0;
	p->num = 0;
	p->offset = 0;
	p->len = 0;

	p->next = pts->parts;
	pts->parts = p;
	p->maxparts = partmax;
	p->part = part;

	stat_start(X_PART_STORE);
	dmsg("part: part_store\n");
	part_store(pf,p,item,len);
	dmsg("part: part_store-done\n");
	stat_end(X_PART_STORE);

	stat_nparts++; stat_partsize += p->len;
	part_idx_add(p,subj,mid);
	dmsg("part: part_idx_add done\n");

	stat_initems++; stat_insize += p->len;

	dmsg("part: part_isitcomplete\n");
	if (parts_complete(pts)) {
		dmsg("part: part_is complete\n");
//		imsg("part: Found complete item %d %s\n",pts->parts,pts->subj);
		part_inprogress = TRUE;
		stat_start(X_PART_SEND);
		dmsg("part: part_is complete-sending, (%s) max=%d\n",pts->subj,pts->parts->maxparts);
		parts_send(pts);
		dmsg("part: part_is complete-sending-done\n");
		stat_end(X_PART_SEND);
		parts_del(h,pts);
		part_inprogress = FALSE;
	}
	dmsg("part: parts_complete done\n");
	stat_start(X_PART_CLEAR);
	/* Set to 5 instead of 10000 for testing */ 
	gap = part_index_size()/10;

	dmsg("part: clear some space\n");
	if (stat_nparts>part_index_size()) {
	 for (j=dbbkt+1;(stat_nparts>(part_index_size()-gap)) && (j<ndbbkt);j++) {
		imsg("part: Too many parts in index %d, clearing bucket %d, In dnews.conf part_index=%d\n",stat_nparts,j,part_index_size());
		part_clear_bkt(j);
	 } 
	 for (j=0;(stat_nparts>(part_index_size()-gap)) && (j<dbbkt);j++) {
		imsg("part: Too many parts in index %d, clearing bucket %d\n",stat_nparts,j);
		part_clear_bkt(j);
	 }
	 if (lib_gmtime()-last>600) {
		 last = lib_gmtime();
		 part_index_rewrite();
	 }
	}
	dmsg("part: clear some space-done\n");
	stat_end(X_PART_CLEAR);
	return TRUE; /* Tells item_suck not to store this item (it's already happenned) */  
}
int part_send_one_idx(Part *p)
{
	db_setpart(FALSE); /* Just to make sure */
/*imsg("Sending in part to index bkt=%d g=%d:%d offset=%d len=%d\n",p->bkt,p->grp,p->num,p->offset,p->len);*/
	if (db_expired(p->bkt)) {
		imsg("Part of multi part message already expired\n");
		return FALSE;
	}
	if (p->len>0) db_add_idx(p->bkt,p->grp,p->num,p->offset,p->len);
	stat_okitems++;
	stat_oksize+= p->len;
	return TRUE;
}
int part_callback(int fid,int grp,int num,int pnt,int len)
{
	return TRUE;
}
int parts_del(int h, Parts *pts)
{
	Parts **next,*pnext,*p;

	next = &ptable[h];
	for (p=ptable[h];p!=NULL;p=pnext) {
		pnext = p->next;
		if (p==pts) {
			*next = p->next;
			parts_free_parts(pts,FALSE);
			return TRUE;
		}
		next = &(p->next);
	}


	emsg("Unable to find part to free\n");
	return FALSE;
}

int parts_free_parts(Parts *pts, int doreport)
{
	Part *p,*pnext;
	int t = 0;

	part_hist_del(pts);
	for (p=pts->parts;p!=NULL;p=pnext) {
		pnext = p->next;
		stat_nparts--; stat_partsize -= p->len;
		t += p->len;
		myfree(p);		
	}
	if (doreport) {
		;
	}
	mystrfree(pts->subj,SD_PARTS);
	mystrfree(pts->mid,SD_PARTS);
	pts->subj = NULL;
	pts->mid = NULL;
	myfree(pts);
	return t;
}
Parts *parts_find_only(int h, char *subj, char *mid)
{
	Parts *pts;
	for (pts=ptable[h]; pts!=NULL; pts=pts->next) {
		if (strcmp(pts->subj,subj)==0) {
			dmsg("part: Found existing multi part index entry %d (%.90s)\n",h,subj);
			return pts;
		} 
	}
	return NULL;
}
Parts *parts_find_add(int h, char *subj, char *mid)
{
	Parts *pts;
	for (pts=ptable[h]; pts!=NULL; pts=pts->next) {
		if (strcmp(pts->subj,subj)==0) {
			dmsg("part: Found existing multi part index entry %d (%.90s)\n",h,subj);
			return pts;
		} 
	}
	pts = mymalloc(sizeof(Parts)); memset(pts,0,sizeof(Parts));
	pts->subj = mystrdup(subj,SD_PARTS);
	pts->mid = mystrdup(mid,SD_PARTS);
	pts->next = ptable[h];
	ptable[h] = pts;
	dmsg("part: Created new multi part index entry %d (%.90s)\n",h,subj);
	part_hist_add(pts);
	return pts;
}
int parts_complete(Parts *pts)
{
	char gotit[1000];
	int maxparts;
	int ngot=0;
	Part *p = pts->parts;
	int i;

	if (p==NULL) return FALSE;
	maxparts = p->maxparts;

	for (p=pts->parts;p!=NULL;p=p->next) {
		if (maxparts<p->maxparts) maxparts = p->maxparts;
	}

	if (maxparts>900) return TRUE;
	
	for (i=0; i<=maxparts; i++) gotit[i] = FALSE;
	for (p=pts->parts;p!=NULL;p=p->next) {
		if (p->part<0) continue;
		if (p->part>maxparts) continue;
		gotit[p->part] = TRUE;
	}

	for (i=1; i<=maxparts; i++) if (!gotit[i]) return FALSE;

	return TRUE;
}
#define PART_OLD file_work("part.old")
int part_init(void)
{
	/* Read in index, extend bucket. etc..*/
	time_zero = time(NULL);
	if (!part_file_init()) return FALSE;
	part_idx_read();
	imsg("part: after init, in bucket %d of %d\n",dbbkt,ndbbkt); 
	part_next_set(); /* Jump to next set on startup so we don't need to remember where we were upto in a bucket */
	part_open_all();
#ifdef DOTHREADS
	{ Thread *t;
	imsg("part: Output thread created \n");
	t = th_new(part_thread_complete,NULL);
	if (t==NULL) {
		emsg("Thread part creation failed %s\n",strerror(errno));
		return FALSE;
	}
	th_free(&t);
	imsg("part: Output2 thread created \n");
	}
#endif

	return TRUE;
}

static char *part_dir[100];
static int ndir;
char *part_file(int bkt)
{
        static char bf[BFSZ];
	
	if (strlen(config_get(C_PART_DIR))==0) {
		emsg("part_dir must be defined, and the directory must exist!\n");
		dnews_ex("part_dir not defined");
	}
        lprintf(bf,BFSZ-1,"%s/part%d.dat",part_dir[bkt%ndir],bkt);
        return bf;
}
int part_file_init(void)
{
	/* Create the directory */
	/* If the files dont exist, create them all at the specified size */
	int i;

	dbend = 100000000;
//	dbend = 4000000;
/*	dbend = 1000000;  During test */
	part_disable = FALSE;
	if (strlen(config_get(C_PART_DIR))==0) {
		imsg("part: part processing disabled as part_dir not defined (this is ok)\n");
		part_disable = TRUE;
		return FALSE;
	}
	for (i= 0; i<5; i++) {
		if (strlen(config_get(C_PART_DIR+i))>0) {
			part_dir[i] = mystrdup(config_get(C_PART_DIR+i),SD_PARTS);
			ndir = i+1;
		}
	}

	ndbbkt = atoi(config_get(C_PART_BUCKETS));
	if (ndbbkt==0) {
		ndbbkt = 3;
	}
	if (ndbbkt>=MAXDBBKT-9) ndbbkt = MAXDBBKT-9;
	if ((ndbbkt % ndir) !=0) ndbbkt += ndir - (ndbbkt % ndir);
	imsg("part_buckets %d\n",ndbbkt);
	for (i=0; i<ndbbkt; i++) {
		bktid[i] = 0;
	}

	return TRUE;
}
int part_file_create(int bkt)
{
	FILE *f;
	f = fopen(part_file(bkt),"r");
	if (f!=NULL) {fclosep(f); return TRUE;}
	lib_createdir(part_file(bkt));
	wmsg("part: Creating part bucket, may take some time... %s\n",part_file(bkt));
	f = fopen(part_file(bkt),"w+b");
	if (f==NULL) {filemsg("part: Could not create %s\n",part_file(bkt)); dnews_ex("part_create"); abort();}
	if (fseek(f,dbend,SEEK_SET)!=0) {
		filemsg("part: Could not extend bucket to 100mb %s\n",part_file(bkt));
		fclosep(f);
		remove(part_file(bkt));
		dnews_ex("part_extend_failed");
		abort();
	}
	if (fwrite("1",1,1,f)!=1) {
		filemsg("part: Could not extend bucket to 100mb %s\n",part_file(bkt));
		remove(part_file(bkt));
		dnews_ex("part_extend_failed");
		abort();
	}
	if (fclosep(f)!=0) {
		filemsg("part: 2 Could not extend bucket to 100mb %s\n",part_file(bkt));
		remove(part_file(bkt));
		dnews_ex("part_extend_failed");
		abort();
	}
	imsgf("part: Created bucket successfully \n");
	return TRUE;	
}
static FILE *lastf,*mf;
static int mfat;
static int lastbkt= -1;
FILE *part_open_f(int bkt, int want)
{
	FILE *f;
	Pfile *pf;
	int i;
	time_t start;
	char fname[BFSZ];

	

	for (i=0; i<ndir; i++) {
		pf = &pfiles[i];
		if (pf->bkt==bkt) {
			/* Make sure it is flushed to the point we need it flushed */
			start = time(NULL);
			for  (;pf->flushed < want;) {
				mutex_wait(M_PART_PF);
				pf->flush = TRUE;
				mutex_release(M_PART_PF);
				for (;pf->flush;) {
					lib_msleep(10);
					if (time(NULL)-start>10) emsg("Waiting for %d seconds for flush\n",time(NULL)-start);
				}
				tmsg("flush bucket %d want %d flushed %d\n",bkt,want,pf->flushed);
				if (lastf!=NULL) fclose(lastf); lastf = NULL;
				if (time(NULL)-start > 60) break;
			}
			break;
		}
	}

	if (lastbkt==bkt) if (lastf!=NULL) return lastf;
	if (lastf!=NULL) fclose(lastf); lastf=NULL;
	lastbkt = bkt;
	strcpy(fname,part_file(bkt));
	f = fopen(fname,"rb");
	if (f==NULL) {
		tmsg("Could not open (%s) %s\n",fname,strerror(errno));
		return NULL;
	}
        setvbuf(f,NULL,_IOFBF,32767);
	lastf = f;
	return f;
}
int part_close_f(void)
{
	return TRUE;
}
int part_close_now(void)
{
	if (lastf!=NULL) fclose(lastf);
	lastf = NULL;
	lastbkt = -1;
	return TRUE;
}
int part_next_set(void)
{
	int i;
	static int last;
	part_close_all();
	dbbkt += ndir;
	if (dbbkt>=ndbbkt) dbbkt = 0;
	for (i=dbbkt; i<dbbkt+ndir; i++) {
		part_clear_bkt(i);
		bktid[i]++;
		pfiles[i-dbbkt].upto = 0;
	}
	if (lib_gmtime()-last>600) {
		 last = lib_gmtime();
		 part_index_rewrite();
	}
	part_open_all();
	return TRUE;
}
int part_write_complete(void)
{
	return TRUE;
}
int part_open_all(void)
{
	int i;
	Pfile *pf;
	mutex_wait(M_PART_FILE);
	part_close_all();
	for (i=0; i<ndir; i++) {
		pf = &pfiles[i];
		pf->bkt = dbbkt+i;
		if (pf->f!=NULL) {
			emsg("panic file handle must be null\n");
			dnews_ex("part handle fault");
		}
		pf->flushed = 0;
		pf->f = fopen(part_file(pf->bkt),"r+b");
		if (pf->f==NULL) {
			part_file_create(pf->bkt);
			pf->f = fopen(part_file(pf->bkt),"r+b");
		}
		if (pf->f==NULL) {
			emsg("Unable to open part file (%s) (%s)\n",part_file(pf->bkt),strerror(errno));
			dnews_ex("Unable to open part file");
		}
		fseek(pf->f,0,SEEK_SET);
		pf->upto = 0;
	        setvbuf(pf->f,NULL,_IOFBF,32767);
	}
	mutex_release(M_PART_FILE);
	return TRUE;
}
int part_close_all(void)
{
	int i;
	time_t start=time(NULL);
	Pfile *pf;

	mutex_wait(M_PART_FILE);
	for (i=0; i<ndir; i++) {
		pf = &pfiles[i];
		if (pf->f==NULL) continue;
		mutex_wait(M_PART_PF);
		for (;pf->busy || pf->bfat>0;) {
			mutex_release(M_PART_PF);
			if (!pf->flush) if (!pf->busy) pf->flush = TRUE;
			lib_msleep(10);
			if (time(NULL)-start > 30) tmsg("part_close_all waited %d seconds\n",time(NULL)-start);
			mutex_wait(M_PART_PF);
		}
		pf->flush = FALSE;
		mutex_release(M_PART_PF);
		if (pf->f!=NULL) fclose(pf->f);
		pf->f = NULL;
	}
	mutex_release(M_PART_FILE);
	return TRUE;
}
Pfile *part_findgap(int len)
{
	int i;
	static int next;

	for (i=next; i<ndir; i++) {
		if (pfiles[i].upto+len<dbend) goto foundit;
	}
	for (i=0; i<ndir; i++) {
		if (pfiles[i].upto+len<dbend) goto foundit;
	}
	/* Then move onto next set of 'ndir' buckets */
	part_next_set();
	part_open_all();
	i = 0;
foundit:
	next++;
	if (next>=ndir) next = 0;
	return &pfiles[i];
}
int part_clear_bkt(int bkt)
{
	int h;
	int killit;
	Part *p;
	Parts *pts,*pnext,**last;
	int done=FALSE;
	int nkill=0;

	if (bkt==0) time_zero = time(NULL);

	for (h=0;h<PART_HASH;h++) {
		last = &ptable[h];
		for (pts=ptable[h];pts!=NULL;pts=pnext) {
			pnext = pts->next;
			killit = FALSE;
			done = FALSE;
			for (p=pts->parts;p!=NULL;p=p->next) {
				if (p->bkt==bkt) {
					if (!done) part_lost(pts,p);
					done = TRUE;
					killit = TRUE;
				}
				if (p->len==0) killit = TRUE;
			}
			if (killit) {
				nkill++;
				*last = pts->next;
				stat_dropsize += parts_free_parts(pts,TRUE);
			} else {
				last = &(pts->next);
			}
		}
	}
	return TRUE;
}
int part_lost(Parts *pts, Part *pone)
{
	char gotit[1000];
	int maxparts;
	Part *p = pts->parts;
	int i;
	int have=0;
	static int tot;
	int miss=0;
	int lastmiss=0;
	static FILE *f;

	if (p==NULL) return FALSE;
	maxparts = p->maxparts;

	for (p=pts->parts;p!=NULL;p=p->next) {
		if (maxparts<p->maxparts) maxparts = p->maxparts;
	}

	if (maxparts>900) return TRUE;
	
	for (i=0; i<=maxparts; i++) gotit[i] = FALSE;
	for (p=pts->parts;p!=NULL;p=p->next) {
		if (p->part<0) continue;
		if (p->part>maxparts) continue;
		have++;
		gotit[p->part] = TRUE;
	}
	for (i=1; i<=maxparts; i++) if (!gotit[i]) {
		miss++;
		lastmiss = i;
	}
	
	if (f==NULL) f = fopen(file_log("partlost.log"),"w");
	if (f==NULL) return TRUE;

	if (miss==0) return TRUE; /* Nothing missing */
	fprintf(f,"%d/%d/%d m=%d %s %.300s %.100s\n",miss,have,maxparts,lastmiss,gotit[0] ? "+Z" : "",pts->subj,pts->mid);
	tot+= 100;
	if (tot>1000000) {
		fclose(f); f = NULL;
		tot = 0;
	}
	return TRUE;
}
int part_showindex(int chan)
{
	int h;
	Part *p;
	Parts *pts;
	char bf[BFSZ];
	for (h=0;h<PART_HASH;h++) {
		for (pts=ptable[h];pts!=NULL;pts=pts->next) {
			lprintf(bf,BFSZ-1,"Item %.70s\r\n",pts->subj); chan_write_s(chan,bf);
			for (p=pts->parts;p!=NULL;p=p->next) {
			 lprintf(bf,BFSZ-1,"Part %d/%d at %d len %d bucket %d\r\n"
				 ,p->part,p->maxparts,p->offset,p->len,p->bkt); 
			 chan_write_s(chan,bf);
			}
		}
	}
	return TRUE;
}
static int th_dbbkt = -1;
int part_create_storing_thread(Pfile *pf)
{
	Thread *t;
	t = th_new(part_storing,pf);
	if (t==NULL) {
		emsg("Thread part creation failed %s\n",strerror(errno));
		return FALSE;
	}
	th_free(&t);
	return TRUE;
}
int part_store(Pfile *pf, Part *p, char *item,int len) 
{
	time_t start;
	if (pf->f==NULL) return FALSE;	

	p->offset = pf->upto;
	p->len = len+2;
	p->bkt = pf->bkt;
	if (pf->bf==NULL) {
		pf->bfsz = conf_item_max()+1000000;
		pf->bf = mymalloc(pf->bfsz+10);
		pf->bfout = mymalloc(pf->bfsz+10);
		pf->bfat = 0;
		part_create_storing_thread(pf);
	}
	mutex_wait(M_PART_PF);
	start = time(NULL);
	for (;pf->bfsz-pf->bfat<len+2;) {
		mutex_release(M_PART_PF);
		lib_msleep(10);
		if (time(NULL)-start > 30) tmsg("part_store waited %d seconds\n",time(NULL)-start);
		mutex_wait(M_PART_PF);
	}
	pf->bf[pf->bfat] = '{';
	pf->bf[pf->bfat+len+1] = '}';
	memcpy(pf->bf + pf->bfat + 1,item,len);
	pf->upto += p->len;
	pf->bfat += p->len;
//	tmsg("wrote part bkt=%d at=%d len=%d\n",p->bkt,p->offset,p->len);
	mutex_release(M_PART_PF);
	return FALSE;
}
int part_storing(void *x)
{
	int outlen;
	Pfile *pf = (Pfile *) x;	
	
	for (;TRUE;) {
		outlen = 0;
		mutex_wait(M_PART_PF);
		if (pf->bfat>1000000 || (pf->flush && (pf->bfat>0))) {
			outlen = pf->bfat;
			memcpy(pf->bfout,pf->bf,outlen);
			pf->bfat = 0;
			pf->busy = TRUE;
		}
		mutex_release(M_PART_PF);
		if (outlen>0) {
			fwrite(pf->bfout,1,outlen,pf->f);
			outlen = 0;
		}
		if (pf->flush) {
			fflush(pf->f); 
			mutex_wait(M_PART_PF);
			if (pf->f!=NULL) {
				pf->flushed = ftell(pf->f);
				pf->flush = FALSE;
				pf->busy = FALSE;
			}
			mutex_release(M_PART_PF);
		} else {
			mutex_wait(M_PART_PF);
			pf->busy = FALSE;
			mutex_release(M_PART_PF);
		}
		lib_msleep(20);
	}
}

#define FILE_PART_IDX file_work("part.idx")
#define FILE_PART_IDX_NEW file_work("part.idx_new")
static  FILE *fidx;
typedef struct PIDX {int bkt; int offset; int len; int maxparts; int part; int dbupto; int dbclear; int dbbkt; short subj_len; short mid_len; char ver;  } Pidx;
int part_idx_open(void)
{
	fidx = fopen(FILE_PART_IDX,"ab");
	if (fidx==NULL) {
		emsg("couldn't open %s\n",FILE_PART_IDX);
		return FALSE;
	}
	return TRUE;
}
int part_idx_add(Part *p, char *subj, char *mid)
{
	Pidx idx;
	if (fidx==NULL) if (!part_idx_open()) return FALSE;
	idx.ver = 10; /* Version */
	idx.offset = p->offset;
	idx.len = p->len;
	idx.part = p->part;
	idx.bkt = p->bkt;
	idx.maxparts = p->maxparts;
	idx.dbclear = dbclear;
	idx.dbbkt = dbbkt;
	idx.subj_len = strlen(subj) ;
	idx.mid_len = strlen(mid) ;
	if (idx.subj_len>300) idx.subj_len=300;
	if (idx.mid_len>300) idx.mid_len=300;
	fwrite(&idx,sizeof(Pidx),1,fidx);
	fwrite(subj,1,idx.subj_len,fidx);
	fwrite(mid,1,idx.mid_len,fidx);
	
	return TRUE;
}
int part_idx_add_new(Part *p, char *subj, char *mid)
{
	Pidx idx;
	if (fidx==NULL) if (!part_idx_open()) return FALSE;
	idx.ver = 11; /* Version */
	idx.offset = p->offset;
	idx.len = p->len;
	idx.part = p->part;
	idx.bkt = p->bkt;
	idx.maxparts = p->maxparts;
	idx.dbupto = p->grp;
	idx.dbclear = p->num;
	idx.dbbkt = dbbkt;
	idx.subj_len = strlen(subj) ;
	idx.mid_len = strlen(mid) ;
	if (idx.subj_len>300) idx.subj_len=300;
	if (idx.mid_len>300) idx.mid_len=300;
	fwrite(&idx,sizeof(Pidx),1,fidx);
	fwrite(subj,1,idx.subj_len,fidx);
	fwrite(mid,1,idx.mid_len,fidx);
	
	return TRUE;
}
char fstrdup(FILE *f, int len);
int part_idx_read(void)
{
	Pidx idx;
	FILE *f;
	Parts *pts;
	char subj[BFSZ],mid[BFSZ];
	Part *p;
	int h;
	int nbad = 0;
	int n=0;
	int nold=0;
	int nnew=0;
	double tot=0;
	imsg("part: Reading part index\n");
	f = fopen(FILE_PART_IDX,"rb");
	if (f==NULL) return TRUE;
	
	for (;!feof(f);) {
		if (fread(&idx,sizeof(idx),1,f)<1) break;
		if (idx.subj_len>BFSZ-1) idx.subj_len = BFSZ -1;
		if (idx.mid_len>BFSZ-1) idx.mid_len = BFSZ -1;
		if (idx.subj_len<0) idx.subj_len = 0;
		if (idx.mid_len<0) idx.mid_len = 0;
		if (idx.subj_len>0) if (fread(subj,1,idx.subj_len,f)<1) {
			filemsg("part: subject read failed %d\n",idx.subj_len);
			break;
		}
		if (idx.mid_len>0) if (fread(mid,1,idx.mid_len,f)<1) {
			filemsg("part: mid read failed\n");
			break;
		}
		if (idx.ver==10) {
			subj[idx.subj_len] = 0;
			mid[idx.mid_len] = 0;
			h = str_hash(subj,PART_HASH);
			pts = parts_find_add(h,subj,mid);
			p = mymalloc(sizeof(Part)); memset(p,0,sizeof(Part));
			stat_nparts++; stat_partsize += idx.len;
			p->next = pts->parts;
			pts->parts = p;
			p->offset = idx.offset;
			p->len = idx.len;
			p->part = idx.part;
			p->bkt = idx.bkt;
			p->maxparts = idx.maxparts;
			dbbkt = idx.dbbkt;
			dbclear = idx.dbclear;
		} else 	if (idx.ver==11) { /* Index only */
			{nnew++;  continue;}
			subj[idx.subj_len] = 0;
			mid[idx.mid_len] = 0;
			h = str_hash(subj,PART_HASH);
			pts = parts_find_add(h,subj,mid);
			p = mymalloc(sizeof(Part)); memset(p,0,sizeof(Part));
			stat_nparts++; stat_partsize += idx.len;
			p->next = pts->parts;
			pts->parts = p;
			p->offset = idx.offset;
			p->len = idx.len;
			p->part = idx.part;
			p->bkt = idx.bkt;
			p->maxparts = idx.maxparts;
			p->grp = idx.dbupto;
			p->num = idx.dbclear;
			dbbkt = idx.dbbkt;
			dbclear = idx.dbclear;
		} else nbad++;
		n++;
		tot += idx.len;
	}
	fclosep(f);
	imsg("part: read %d/%d parts %d bad, %d old, %d new, into index total %.2fmb/%dmb\n",stat_nparts,n,nbad,nold,nnew,tot/1000000,ndbbkt*dbend/1000000);
	return TRUE;
}
int part_index_rewrite(void);

int part_index_rewrite(void)
{
	Pidx idx;
	FILE *f;
	int h;
	char fname[BFSZ];
	Parts *pts;
	Part *p;
	int n=0;
	int failed=FALSE;
	int np=0;
	if (fidx!=NULL) fclosep(fidx);
	fidx = NULL;

	f = fopen(FILE_PART_IDX_NEW,"wb");
	if (f==NULL) {	filemsg("part: Could not create (%s)\n",FILE_PART_IDX_NEW); return FALSE; }
	for (h=0;h<PART_HASH;h++) {
	  for (pts=ptable[h];pts!=NULL;pts=pts->next) {
		  for (p=pts->parts; p!=NULL; p = p->next) {
			idx.ver = 10; /* Version */
			idx.offset = p->offset;
			idx.len = p->len;
			idx.part = p->part;
			idx.bkt = p->bkt;
			idx.maxparts = p->maxparts;
			idx.dbbkt = dbbkt;
			idx.dbclear = dbclear;
			idx.subj_len = strlen(pts->subj) ;
			idx.mid_len = strlen(pts->mid) ;
			if (idx.subj_len>300) idx.subj_len=300;
			if (idx.mid_len>300) idx.mid_len=300;
			fwrite(&idx,sizeof(Pidx),1,f);
			fwrite(pts->subj,1,idx.subj_len,f);
			if (fwrite(pts->mid,1,idx.mid_len,f)<1) failed=TRUE;
			n++;
		  }
		  np++;
	  }
	}
	if (fclosep(f)!=0) failed=TRUE;
	if (failed) {
		filemsg("Writing part index failed - is workarea disk full? \n");
		return FALSE;
	}
	strcpy(fname,FILE_PART_IDX);
	if (remove(fname)!=0) filemsg("part: Remove of (%s) failed\n",fname);
	if (rename(FILE_PART_IDX_NEW,fname)!=0) filemsg("part: Rename to (%s) failed\n",fname);
	imsg("part: Re-written file part index with %d items %d parts\n",np,n);
	return TRUE;
}
/* Send in the parts in order just to  be fancy */
static Parts *send_que;
int parts_send_tothread(Parts *pts)
{
	Parts *p;
	int qsz = atoi(config_get(C_PART_QUESIZE));
	if (qsz==0) qsz = 400;
	/* Add to the queue of items to be sent in */
	mutex_wait(M_PART_SEND);
	dmsg("part: got part_send mutex\n");
	if (nsend<qsz) {
		dmsg("part: got dup the parts\n");
		p = pts_dup(pts);
		dmsg("part: got dup the parts-done\n");
		p->next = send_que;
		send_que = p;
		nsend++;
	} else {
		wmsg("Dropping item as part send queue is full\n");
		ndrop++;
	}
	dmsg("part: send_tothread done\n");
	mutex_release(M_PART_SEND);
	return TRUE;
}
int part_thread_complete(void *x)
{
	int sync=FALSE;
	Parts *p,*lastone,**lastp;
	lib_boost_thisthread(); /* So the buffer is always empty with luck .*/
	for (;TRUE;) {
		mutex_wait(M_PART_SEND);
		if (nsend<10) {
			mutex_release(M_PART_SEND);
			lib_msleep(20);
			continue;
		}
		// Peel of the end of the thing
		lastp = &send_que;
		lastone = send_que;
		for (p=send_que;p!=NULL;p=p->next) {
			lastone = p;
			if (p->next!=NULL) lastp = &p->next;
		}
		*lastp = NULL;
		p = lastone;
//		send_que = send_que->next;
		nsend--;
		mutex_release(M_PART_SEND);
		if (!part_thread_send(p)) {
			mutex_wait(M_PART_SEND);
			tmsg("part: failed multi part post \n");
			ndrop++;
			mutex_release(M_PART_SEND);
		}
		mutex_wait(M_MAIN);
		parts_free(p);
		mutex_release(M_MAIN);
	}
	return TRUE;
}
int parts_free(Parts *pts)
{
	Part *p,*pnext;
	int t = 0;

	for (p=pts->parts;p!=NULL;p=pnext) {
		pnext = p->next;
		myfree(p);		
	}
	mystrfree(pts->subj,SD_PARTS);
	mystrfree(pts->mid,SD_PARTS);
	pts->subj = NULL;
	pts->mid = NULL;
	myfree(pts);
	return t;
}
int part_thread_send(Parts *pts)
{
	int max = pts->parts->maxparts;
	int n=0;
	int nok=0;
	int i;
	int donesync = FALSE;
	Part *p;
	int ret= TRUE;
	for (p=pts->parts; p!=NULL; p=p->next) {
		if (p->bkt<0) ret = FALSE;
		if (p->bkt>=ndbbkt) ret = FALSE;
		if (p->bktid != bktid[p->bkt]) {
			tmsg("bktid has changed [%d] %d != %d\n",p->bkt,p->bktid,bktid[p->bkt]);
			return FALSE;
		}
		if (!ret) {
			tmsg("Range check failed\n");
			return FALSE;
		}
	}
	/* 
		Now read each one from disk, in order, then
		submit after grabbing main mutex 
	*/
	for (p=pts->parts;p!=NULL;p=p->next) {
		if (max<p->maxparts) max = p->maxparts;
	}
	
	for (i=0; i<=max; i++) {
		for (p=pts->parts;p!=NULL;p=p->next) if (p->part==i) break;
		if (p!=NULL) {
			if (part_send_one(pts,p,TRUE,&donesync)) nok++; 
			n++;
		}
	}
	part_close_now();
	//tmsg("part: parts_complete, Stored %d parts of %d\n",n,max);
	if ((nok>0) || (n>0)) if (nok != n) {
		stat_nfailed++;
		tmsg("part: Post only partially successful %d/%d/%d %s %s\n",nok,n,max,pts->mid,pts->subj);
		ttmsg("part: Partly posted %d/%d/%d %s %s\n",nok,n,max,pts->mid,pts->subj);
	} else {
        	ttmsg("part: OK %d/%d/%d %s %s\n",nok,n,max,pts->mid,pts->subj);
	}
	return TRUE;

	return ret;
}
Parts *pts_dup(Parts *pts)
{
	Parts *head=NULL;
	Part *p=NULL;
	Part *n;
	Part **last;
	int nparts=0;
	
	head = mymalloc(sizeof(Parts));
	*head = *pts;
	if (pts->subj==NULL) {
		emsg("dup failed %p\n",pts);
		fflush(NULL);
		abort();
	}
	head->subj = mystrdup(pts->subj,SD_PARTS);
	head->mid = mystrdup(pts->mid,SD_PARTS);
	head->next = NULL;
	head->parts = NULL;
	last = &(head->parts);
	for (p=pts->parts; p!=NULL; p=p->next) {
		n = mymalloc(sizeof(Part));
		*n = *p;
		nparts++;
		if (n->bkt<0) n->bkt = 0;
		if (n->bkt>=ndbbkt) n->bkt = 0;
		n->bktid = bktid[n->bkt];
		n->next = NULL;
		*last = n;
		last = &n->next;
	}
	dmsg("part: dup finished, parts (%d)\n",nparts);
	return head;
}
int parts_send(Parts *pts)
{
	return parts_send_tothread(pts);
}
void hist_set_part(int x);
int part_send_one(Parts *pts,Part *p, int isthread, int *donesync)
{
	String *xitem;
	static char *mem;
	static int memsz;
	FILE *f=NULL;
	if (p->bkt<0) return FALSE;
	if (isthread && (p->bktid != bktid[p->bkt])) {
		tmsg("Bucket has been rotated bugger\n");
		return FALSE;
	}
	if (!(*donesync)) if (p->bkt==th_dbbkt) { 
		*donesync = TRUE;
	}
	if (mem==NULL) {
		mutex_wait(M_MAIN);
		memsz = conf_item_max()+102000;
		mem = mymalloc(memsz+100);
		mutex_release(M_MAIN);	
	}

	f = part_open_f(p->bkt,p->offset+p->len-1);
	if (f==NULL) {
		tmsg("part: Cannot send part as bucket file disappeared %s\n",strerror(errno));
		part_close_f();
		return FALSE;
	}
	if (fseek(f,p->offset,SEEK_SET)!=0) {
		tmsg("part: Cannot send part as seek failed\n");
		goto failed;
	}
	if (p->len<0) { tmsg("FATAL ERROR, part length less than zero\n");goto failed;}
	if (p->len+10>=memsz) {tmsg("FATAL ERROR, part length is too large %d\n",p->len); goto failed;}
	if ((int) fread(mem,1,p->len,f)!=(p->len)) {
		tmsg("part: Cannot send part as fread2 failed bkt=%d at=%d\n",p->bkt,p->offset); goto failed;
	}
	if (mem[0]!='{') {tmsg("Header gone bkt=%d at=%d len=%d \n",p->bkt,p->offset,p->len); goto failed;}
	if (mem[p->len-1]!='}') {tmsg("tail gone bkt=%d at=%d len=%d \n",p->bkt,p->offset,p->len); goto failed;}
	part_close_f();
	mutex_wait(M_MAIN);
	part_inprogress = TRUE;
	xitem = str_new();
        str_setlen(xitem,p->len);
	memcpy(str_s(xitem),mem+1,p->len-1);
	str_setlen(xitem,p->len-2);
	hist_set_part(TRUE);
	if (!item_suck(xitem)) {
		hist_set_part(FALSE);
		imsg("part: badpart mid (%s) subj (%s) %d/%d %s\n",pts->mid,pts->subj,p->part,p->maxparts,msg_reason());
		partmsg("part: badpart mid (%s) subj (%s) %d/%d %s\n",pts->mid,pts->subj,p->part,p->maxparts,msg_reason());
		stat_nrej++;
		tmsg("item rejected for some reason (%s)\n",msg_reason());
		goto failed2;
	} else {
		hist_set_part(FALSE);
		dmsg("part: Sent part %d/%d ok\n",p->part,p->maxparts);
		stat_okitems++;
		stat_oksize+= p->len;
	}	
	str_free(xitem); xitem = NULL;
	part_inprogress = FALSE;
	mutex_release(M_MAIN);
	return TRUE;
failed2:
	part_inprogress = FALSE;
	str_free(xitem); xitem = NULL;
	mutex_release(M_MAIN);
failed:
	mutex_wait(M_MAIN);
	partmsg("part: badread mid (%s) subj (%s) %d/%d\n",pts->mid,pts->subj,p->part,p->maxparts);
	stat_readfail++;
	mutex_release(M_MAIN);
	part_close_f();
	return FALSE;
}


void partmsg(char * arg_list, ...)
{
	char str[BFSZ];
        va_list arg_ptr;
        char *format;  /* Shouldn't be necessary */
	char output[9000];
	FILE *f;
	time_t t;
        va_start(arg_ptr, arg_list);
        format = arg_list;
        vsnprintf(output, BFSZ-1, format, arg_ptr);
        output[BFSZ-1] = 0;

	f = fopen(file_log("parterr.log"),"a");
	if (f==NULL) return;
	time(&t);
	fprintf(f,"%.19s: s%d %s",safe_ctime(&t,str),slave_n(),output);
	fclosep(f);
	if (lib_length_file(file_work("parterr.log"))>200000) {
		remove(file_work("parterr.log"));
	}
}
