
/* Copyright (C) 1995 NetWin World, New Zealand, All rights reserved. */
/*   
--------
change xover & xhdr to only ask for items that exist.
---------------------
item_xov() if (isxov, and missing  and db_exists) 
then call_append (but only if it's > xov_last) 
---------------------
After an expire, do an xover expire, for each 'xov' group, 
read in group, read theru .xov data, write out with 
deleted items removed and add any missing items.
---------------------
Allow additional xover headers, include xref by default
Change xhdr to use xover data if it's one of the known fields
*/


#ifdef __MWERKS__ /*RHP we will take this to mean we are making a Mac version*/
#include "mac.h"
#endif

#include <stdlib.h>
#include <stdio.h>
#include <ctype.h>
#include <string.h>
#include <assert.h>
#include <errno.h>
#ifdef VMS
#include <unixio.h>
#define unlink remove
#endif
#ifndef NO_FCNTL
#include <fcntl.h>
#else
/*RHP not for mac */
#ifndef Mac
#include <file.h>
#endif
#endif
#include "emsg.h"
#include "sary.h"
#include "str.h"
#include "lib.h"
#include "group.h"
#include "hist.h"
#include "suck.h"
#include "chan.h"
#include "feed.h"
#include "expire.h"
#include "control.h"
#include "item.h"
#include "db.h"
#include "xhash.h"
#ifdef NETWARE
#define NOCACHED_BODY     file_config("nocached.bod")
#else
#define NOCACHED_BODY     file_config("nocached.body")
#endif



#undef mymalloc
#undef myfree
#define mymalloc(x) mymalloc2(x,M_ITEM)
#define myfree(x) myfree2(x,M_ITEM)

#undef F_READ
#undef F_WRITE
#undef F_APPEND
#ifdef WIN32
#define F_READ "rb"
#else
#define F_READ "r"
#endif
#define F_WRITE "w"
#define F_APPEND "a"

/* #define EOL "\015\012"  */
/* Routines to deal with item text and header. */
int xclose(int x);
int item_expire_xover_idle(int grp);

void item_addcache(String *str,int grp, int num, int artlen);
String *item_getcache(String *str, int grp, int num);
String *item_fixhead(String *item, String *xref, String *xpath);
void item_loading(String *h, int grp);
void fiddle_order(char *grps);
int check_body(String *body);
int check_head(String *head, int grp, int num);
static int maxcache;
void wcache_write_one(int x, int last);
void item_xover_write_cached(int grp, int num, String *xitem, int force);
void wcache_add(int grp, int num, String *str);
void wcache_init(void);
void wcache_shrink(void);
void wcache_write_all(void);
int is_head_exact(String *sitem, char *head);


/* len and hlen are the length in compressed format, not uncompressed.  */
char *file_store(int x);
typedef struct {int pnt, len, hlen;} Prec;
Prec head[101],head2[101];
static int inexpire;
void xover_clear_cache(void);
void debug_store_item(String *item);
void slave_tellmaster_buildxover(int g);
void item_purgedel(int g, int num);
int item_purgedo2(int g, int num, int upto);
static int inpurge;
static int in_expire_xover = FALSE;
void fiddle_first(char *grps,char *wanted);
void xover_copy_file(int grp);
static int total_cancel;

#define MAX_XOVER 20000

int item_findfirst(int group, int first, int last)
{
	return g_list_start(group);
	/*
	for (i=first; i<=last; i++) {
	if (db_exists(group,i)) return i;
	}
	*/
}
int item_findnum(int group, int *xfirst, int *xlast)
{
	return g_list_findnum(group,xfirst,xlast);
}
static int last_len;
static int xover2_nfill,xover2_ncopy,xover2_ngroups,xover2_nmiss,xover2_nmiss2;

String *item_head_cheat(int group, int num, char *gnum)
{
	String *s;
	db_cheat_set(gnum);
	s = item_head(group,num);
	db_cheat_set(NULL);	
	return s;
}
String *item_body_cheat(int group, int num, char *gnum)
{
	String *s;
	db_cheat_set(gnum);
	s = item_body(group,num);
	db_cheat_set(NULL);	
	return s;
}
String *item_head(int group, int num)
{
	static String *tmp,*hstr;
	int start,iscached;
	
	vtrace();
	if (group<0 || group>=group_n()) {emsg("item: Bad group number for head %d %d \n",group,num); return NULL;}
	if (hstr==NULL) hstr = str_new();  str_clearbig(hstr);
	
	
	tmp = item_getcache(hstr,group,num);
	if (tmp!=NULL) return tmp;
	
	start = group_getstart(group);
	iscached = group_iscached(group);
	if (num<start && inexpire) {return NULL;}
	if (!group_nodownload())
		if ((num==start || num+1==start) && !inexpire) {
			if ( (group_inupdate(group) && num==start) ||
				(iscached && (num==start-1)) || 
				(!iscached && (num==start))  ) {
                if (!group_inupdate(group) && !iscached) {
					if (!in_expire_xover) {
						if (is_slave()) group_setcached_slave(group);
						else {
							suck_queue_anyone(group_name(group),0);
							group_setupdate(group,TRUE);
						}
					}
				}
				item_loading(hstr,group);
				last_len = 10000; /* Hopefully a big number won't cause a crash*/
				return hstr;
			}
        }
		
		
        tmp = db_get_head(group,num);
        if (tmp==NULL) {
			if (group_getstart(group)==num) {
				group_setstart(group,num+1); 
				group_setrange(group,-1);
				imsg("Correcting group start {%.200s} %d\n",group_name(group),num+1);
			}
			return NULL;
		}
		last_len = db_last_len();
        str_copy(hstr,tmp);
        if (!check_head(hstr,group,num)) return NULL;
        item_addcache(hstr,group,num,last_len);
        return hstr;
}
int item_last_len(void)
{
	return last_len;
}
String *item_body(int group, int num)
{
	int x;
	return item_body_size(group, num, 0, 7000000,&x);
}
String *item_body_size(int group, int num, int from, int from_len, int *rfrom)
{
	static String *tmp;
	static String *hstr;
	int start,iscached;
	char bf2[BFSZ],bf[BFSZ];
	if (hstr==NULL) hstr = str_new();       
	
	*rfrom = -1;
	str_clearbig(hstr);
	if (group<0 || group>=group_n()) {emsg("item: Bad group number %d %d \n",group,num);return NULL;}
	
	start = group_getstart(group);
	iscached = group_iscached(group);
	if (num<start && inexpire) return NULL;
	if (!group_nodownload()) 
        if (num==start || num+1==start) {
			if ((group_inupdate(group) && num==start) ||
				(iscached && (num==start-1)) || 
				(!iscached && (num==start)) )   {
                if (!group_inupdate(group) && !iscached) {
					suck_queue_anyone(group_name(group),0);
					group_setupdate(group,TRUE);
                }
                str_read(hstr,NOCACHED_BODY);
                dnews_getkey(bf2);
				if (config_true(C_HIDE_REGISTER)) {
					if (strlen(bf2)>0) {
                        strcpy(bf,"     This is a trial copy of DNEWS, we hope you \015\012"); str_adds(hstr,bf,strlen(bf));
                        strcpy(bf,"     decide to register this product, see LICENSE.TXT for \015\012"); str_adds(hstr,bf,strlen(bf));
                        lprintf(bf,BFSZ-1,"    instructions on registering. \015\012"); str_adds(hstr,bf,strlen(bf));
                        lprintf(bf,BFSZ-1,"    %.200s\015\012\015\012",bf2); str_adds(hstr,bf,strlen(bf));
					}
				}
				if (config_true(C_NO_QUEUE)) {
					if (!config_true(C_HIDE_QUEUE)) {
						suck_queue_show(hstr);
					} else {
						suck_queue_hide(hstr,group);
					}
				}
                if (!config_true(C_NOADVERT)) {
					lprintf(bf,BFSZ-1,"DNEWS Home page   http://netwinsite.com \015\012"); str_adds(hstr,bf,strlen(bf));
					lprintf(bf,BFSZ-1,"DNEWS FTP site    ftp://netwinsite.com/dnews\015\012"); str_adds(hstr,bf,strlen(bf));
                }
				*rfrom = -1;
                return hstr;
			}
        }
        tmp = db_get_body(group,num,from,from_len, rfrom);
        if (tmp==NULL) return NULL;
        str_copy(hstr,tmp);
        if (*rfrom<0) if (!check_body(hstr)) return NULL;
        return hstr;
}
char *item_getahead(int group, int num, char *wanted)
{
	String *hdr;
	char *s;
	hdr = item_head(group,num);
	if (hdr==NULL) return NULL; /* Item doesn't exist */    
	s = get_ahead(hdr,wanted);
	return s;
}
static int xhit,xmiss;
String *item_xover_read(int grp, int num, int getnext);
void item_xover_write(int grp, int num, String *xitem, int force);
double xcache_hitrate(void)
{
	if (xmiss==0) xmiss = 1;
	return (double) xhit / (double) (xmiss + xhit);
}
static int xs_tot,xs_file,xs_db;
static int xs_nmem,xs_nfile,xs_ndb;
char *item_xover_status(void)
{
	static char bf[BFSZ];
	if (xs_ndb==0) xs_ndb=1;
	if (xs_nfile==0) xs_nfile=1;
	lprintf(bf,BFSZ-1,"Xover %ds, xov=%ds/%d=%3.3f item=%ds/%d=%3.3f\r\n",
		xs_tot,xs_file,xs_nfile,((double) xs_file)/xs_nfile,
		xs_db,xs_ndb,((double) xs_db)/xs_ndb);
	return bf;
}
String *xitem_xover(int group, int num, int getnext);
String *item_xover(int group, int num, int getnext)
{
	String *r;
	xs_tot -= lib_gmtime();
	r = xitem_xover(group,num,getnext);
	xs_tot += lib_gmtime();
	return r;
}
String *xitem_xover_xhash_raw(int group, int num, int getnext)
{
	char reason[BFSZ];
	int gid;
	int r;
	int len=0;
	char xline[MAX_XOVER];
	static String *x;
	if (x==NULL) x = str_new();
	gid = group_getgroupid(group);
	if (gid<0) return NULL;
	r = xhash_get(gid,num,xline,sizeof(xline)-2,&len,reason);
	if (!r) {
		if (strlen(reason)>0) imsg("hash_get: %s\n",reason);
		return NULL;
	}
	str_copys(x,xline,len);
	return x;
}
String *xitem_xover_xhash(int group, int num, int getnext)
{
	String *hdr;
	static String *x;
	
	wcache_write_one(group,-1); /* Flush the cache if it exists for this group */
	x = xitem_xover_xhash_raw(group,num,getnext);
	if (x!=NULL) xs_nfile++;
	if (x==NULL) {
		xs_db -= lib_gmtime();
		xs_ndb++;
		hdr = item_head(group,num);
		xs_db += lib_gmtime();
		if (hdr==NULL) return NULL;
		x = xover_make(hdr,num);
		/* Write it to the cache */
		item_xover_write(group,num,x,FALSE); 
	}
	xcache_store(group,num,x); /* Store for use next time */
	xmiss++;
	return x;
}

String *xitem_xover(int group, int num, int getnext)
{
	String *hdr;
	static String *x;
	int rang;
	
	if (group<0) return NULL;
	x = xcache_get(group,num);
	if (x!=NULL) {xhit++; xs_nmem++; return x;}
	
	if (config_true(C_XOVER_XHASH)) {
		group_hitxover(group);
		return xitem_xover_xhash(group,num,getnext);
	}
	
	
	if (!config_true(C_XOVER_ONDEMAND)) {
		/*		xs_file -= lib_gmtime();*/
		xs_nfile++;
		wcache_write_one(group,-1); /* Flush the cache if it exists*/
		x = item_xover_read(group,num,getnext);
		/*		xs_file += lib_gmtime();*/
	}
	if (x==NULL) {
		xs_db -= lib_gmtime();
		xs_ndb++;
		hdr = item_head(group,num);
		xs_db += lib_gmtime();
		if (hdr==NULL) {
			/*			imsg("xover: could not get header of item %s:%d\n",group_name(group),num);*/
			return NULL;     /* Item doesn't exist */        
		}
		x = xover_make(hdr,num);
		if (!config_true(C_XOVER_ONDEMAND)) {
			if (maxcache==0) item_xover_write(group,num,x,FALSE); /* Try and append it to .xov file */	
		}
		rang = group_rawrange(group);
		/*imsg("Missing xover item (%.200s) %d/%d/%d r=%d len=%d\n"
		,group_name(group),group_getfrom(group),num,group_getend(group),rang,str_len(x));
		*/
		if (x!=NULL) if (group_isxover(group)) 
			if (group_rawrange(group)>100) 
				if (num<(group_getend(group)-100)) 
					if (num<(group_getend(group)-rang/5)) 
						if (num>(group_getfrom(group)+100)) 
							if (num>(group_getfrom(group)+rang/2)) 
								if (!group_getxsort(group)) {
									if (str_len(x)<(MAX_XOVER-1000)) {		
										if (config_true(C_XOVER_NOIDLE)) { 
											wmsg("NOT Sorting xover for (%.200s) as it seems to have gaps %d/%d/%d r=%d len=%d\n"
												,group_name(group),group_getfrom(group)
												,num,group_getend(group),rang,str_len(x));
										} else {
											wmsg("Sorting xover for (%.200s) as it seems to have gaps %d/%d/%d r=%d len=%d\n"
												,group_name(group),group_getfrom(group)
												,num,group_getend(group),rang,str_len(x));
											xidle_sort(group);
										}
									}
									/*
									check 'slave' is 1
									check not readonly server?
									
									  stat_start(X_XSORT);
									  g_list_save();
									  sort_xover(group);
									  g_list_restore();		
									  stat_end(X_XSORT);
									  wmsg("Sorting xover for (%.200s) as it seems to have gaps, finished\n",group_name(group));
									*/
									group_setxsort(group,TRUE);
								}
	}
	xcache_store(group,num,x); /* Store for use next time */
	xmiss++;
	return x;
}
static int isrebuild;
int set_item_rebuild(int x)
{
	isrebuild = x;
	return TRUE;
}

/*--------------xover file stuff --------------*/
static FILE *fxov;
static int xov_grp = -1;
static int xov_first = -1;
static int xov_last = -1;
static int xov_lastread = -1;
static int xov_write = FALSE;
static int xov_gap1,xov_gap2;

char *file_xov_new(void)
{
	static char bf[BFSZ];
	strcpy(bf,file_xwork("xov.new",-1));
	return bf;
}
char *file_xov_srt(void)
{
	static char bf[BFSZ];
	strcpy(bf,file_xwork("xov.srt",-1));
	return bf;
}
char *file_xov_idle(void)
{
	static char bf[BFSZ];
	strcpy(bf,file_xwork("xov.idl",-1));
	return bf;
}

char *file_xwork(char *fname, int x)
{
	static char bf[BFSZ];
	static char bf2[BFSZ];
	char *s;
	
	s = config_get(C_XOVER_DIR);
	if (x>=0) if (config_true(C_XOVER_HASH)) {
		lprintf(bf2,BFSZ-1,"%d/%.200s",x % 100,fname);
		fname = bf2;
	}
	if (strlen(s)==0) return file_work(fname);
#ifdef VMS
	lprintf(bf,BFSZ-1,"%.200s%.200s",s,fname);
#else
	lprintf(bf,BFSZ-1,"%.200s/%.200s",s,fname);
#endif
	return bf;
}
char *file_xover(int grp)
{
	char bf[BFSZ];
	lprintf(bf,BFSZ-1,"x_%d.xov",group_getgroupid(grp));
	return file_xwork(bf,group_getgroupid(grp));
}
void xover_test_one(int chan, int grp);
void item_test_xover(int chan)
{
	int g;
	int start=lib_gmtime();
	for (g=0; g<group_n() && ((lib_gmtime()-start)<160); g++) {
		if (group_iscached(g)) xover_test_one(chan,g);
	}
}
void xover_test_one(int chan, int grp)
{
	int n;
	int len;
	int i;
	char bf[BFSZ];
	String *fstr;
	String *hdr,*dstr;
	char *s;
	n = g_list_start(grp);
	for (; (n>=0);n = g_list_next())  {
		fstr = item_xover_read(grp,n,FALSE);
		if (fstr==NULL) continue;
		
		hdr = item_head(grp,n);
		if (hdr==NULL) continue;
		dstr = xover_make(hdr,n);
		if (dstr==NULL) continue;
		s = str_s(fstr);
		for (i=0; i<5; i++) {
			if (s!=NULL) s = strchr(s+1,'\t');
		}
		len = 0;
		if (s!=NULL) len = s - str_s(fstr);
		if (strncmp(str_s(fstr),str_s(dstr),len)!=0) {
			lprintf(bf,BFSZ-1,"Xover mismatch %.200s:%d\r\n",group_name(grp),n);
			chan_write(chan,fstr);
			chan_write(chan,dstr);
			emsg("XOVER Mismatch %.200s:%d\n",group_name(grp),n);
			chan_write_s(chan,bf);
		}
	}
}
void item_xover_close(void) /* Call when finish finished or if we want to rewrite them */ 
{
	if (fxov!=NULL) fclosep(fxov);
	fxov = NULL;
	xhash_close();
}
int xover_seek(FILE *f, int grp, int num)
{
	char bf[MAX_XOVER+30];
	int len = file_length(fileno(f));
	int min = group_getfrom(grp);
	int range = group_rawrange(grp);
	int max = group_getend(grp);
	int guess;
	int ilen;
	int i;
	int nr2;
	int nseek=0;
	char *s;
	int n;
	int nr;
	if (range<10) return TRUE;
	ilen = len / range;
	
	
	guess = ilen * (range - (max - num));
	/* Reusing min,max !!! different meaning now  */
	min = 0; max = len;
	if (guess<100) return TRUE;
	for (i=0;i<10;i++) {
		nseek++;
		/*imsg("Seeking [%d] %6d %6d %6d\n",nseek,min,guess,max);*/
		fseek(f,guess,SEEK_SET);
		nr = fread(bf,1,500,f);
		if (nr<=0) goto failed;
		bf[nr] = 0;
		s = strchr(bf,'\n');
		if (s==NULL) {
			nr2 = fread(bf+nr,1,MAX_XOVER+12-500,f);
			if (nr2<=0) goto failed;
			bf[nr+nr2] = 0;
		}
		s = strchr(bf,'\n');
		if (s==NULL) goto failed;
		n = atoi(s+1);
		/*imsg("Number %d, wanted %d\n",n,num);*/
		if (n>num) {
			max = guess;
			guess -= ((n-num+1)*ilen);
			if (guess<=min) break;
			if (max-min<100) break;
		} else if (n<(num-5)) {
			min = guess;
			guess += ((num-n+1)*ilen);
			if (guess>=max) break;
			/* guess = min + (max - guess) / 2; */
			if (max-min<100) break;
		} else {
			if ((guess-2) < 0) guess = 2;
			fseek(f,guess-2,SEEK_SET);
			fgets(bf,MAX_XOVER,f);
			xov_lastread = n;
			if (xov_lastread>num) {
				emsg("xov_seek odd %.200s %d > %d\n",group_name(grp),xov_lastread,num);
			}
			dmsg("xover: Seeks %d\n",nseek);
			return TRUE;
		}
	}
	if (i>7) imsg("Searching failed, thats odd\n");
failed:	fseek(f,0,SEEK_SET);
		xov_lastread = -1;
		return TRUE;
}
String *item_xover_read_old(int grp, int num)
{
	static String *x;
	char bf[MAX_XOVER];
	int n= -1;
	int len;
	if (x==NULL) x = str_new();
	
	if (grp<0) return NULL;
	dmsg("xover: Read xover (%d:%d):%d, last=%d %d\n",grp,xov_grp,num,xov_last,xov_lastread);
	if (!group_hitxover(grp)) {
		return NULL; /* Not enough items to bother */
	}
	if (grp==xov_grp) {
		if ((num > xov_gap1) && (num < xov_gap2)) return NULL;
		if (xov_last>=0) if (num>xov_last) {
			return NULL;	
		}
		if (!xov_write) if (fxov!=NULL) goto isopen;
	}
	if (xov_write) item_xover_close();
	if (fxov!=NULL) fclosep(fxov);
	stat_start(X_XOPEN);
	fxov = myfopen(file_xover(grp),F_READ);
	stat_end(X_XOPEN);
	dmsg("xover: Open read {%.200s}\n",file_xover(grp));
	if (fxov==NULL) {
		fimsg("Couldn't find a {%.200s} file so I will create it\n",file_xover(grp));
		xov_grp = grp;
		xov_write = FALSE;
		xov_last = 0;
		return NULL;
	}
	xov_write = FALSE;
	xov_first = -1;
	xov_last = -1;
	xov_lastread = -1;
	xov_grp = grp;
	xov_gap1 = -1;
	xov_gap2 = -1;
isopen:;
	   if (num<xov_first) return NULL;
	   if (num<=xov_lastread) {
		   dmsg("xover: Seek Start %d %d\n",num,xov_lastread);
		   fseek(fxov,0,SEEK_SET);
		   xov_lastread = -1;
	   }
	   if (xov_lastread<num-20) {
		   if (!config_true(C_XOVER_SEEK_DISABLE)) {
			   xover_seek(fxov,grp,num);
		   }
	   }
	   
	   for (;!feof(fxov);) {
		   if (fgets(bf,MAX_XOVER-3,fxov)==NULL) break;
		   len = strlen(bf);
		   if (bf[len-2]=='\r') {
			   bf[len-2] = '\n';
			   bf[len-1] = 0;
			   len--;
		   }
		   if (bf[len-1] != '\n') {
			   if (is_readonly()) {
				   imsg("Incomplete line, ok as file may not be flushed (item=%d+1) (%.200s)\n",xov_lastread,file_xover(grp));
				   continue;
			   }
			   /* Xover file corrupt, delete it */
			   filemsg("Error reading xover file, deleting it {%.200s} \n",file_xover(grp));
			   item_xover_close();
			   xov_first = -1;
			   xov_lastread = -1;
			   remove(file_xover(grp));
			   return NULL;
		   }
		   if (len>2) if (bf[len-2]=='\r') {
			   bf[len-2] = '\n';
			   bf[len-1] = 0;
		   }
		   n = atoi(bf);
		   if (xov_first<0) xov_first = n;
		   if (xov_lastread>=0) if (xov_lastread > n) {
			   wmsg("xov: Xover file out of order? %.200s:%d %d\n",group_name(grp),n,xov_lastread); 
			   /*			xidle_sort(group);			*/
			   
			   xidle_sort(grp);
			   return NULL;
			   /* Remove because multiple slaves try and sort then all hell breaks loose*/
			   
			   /*			g_list_save();
			   if (slave_n()<2) {
			   sort_xover(grp);
			   }
			   g_list_restore();*/
			   /*			return NULL;  Force one lookup */ 
		   }
		   xov_lastread = n;
		   xov_gap1 = xov_gap2;
		   xov_gap2 = n;
		   
		   if (n>num) return NULL;
		   if (n==num) {
			   strcpy(bf+len-1,EOL);
			   str_copysn(x,bf);
			   return x;
		   }
	   }
	   xov_last = xov_lastread;
	   if (xov_last<0) xov_last = 0;
	   return NULL;
}
String *xov_read_one(int grp, int num);
static char freadbf[32000];
int item_xover_reset(void)
{
	item_xover_close();
	xhash_close();
	xov_grp = -1;
	return TRUE;
}
String *item_xover_read(int grp, int num, int getnext)
{
	if (config_true(C_XOVER_XHASH)) {
		emsg("item_xover_read called by mistake\n");
		dnews_ex("item_xover_read wrong call");
		return NULL;
	}
	
	
	if (grp<0) return NULL;
	if (!group_hitxover(grp)) {
		dmsg("xover: Not reading from index as hitxover false %s\n",group_name(grp));
		return NULL; /* Not enough items to bother */
	}
	if (grp==xov_grp) {
		if ((num > xov_gap1) && (num < xov_gap2)) {
			dmsg("xover: Inside gap so not reading %s %d %d %d\n",group_name(grp),xov_gap1,num,xov_gap2);
			return NULL;
		}
		if (xov_last>=0) if (num>xov_last) {
			dmsg("xover: xov_last failed %d %d %s\n",num,xov_last,group_name(grp));
			return NULL;	
		}
		if (!xov_write) if (fxov!=NULL) goto isopen;
	}
	if (xov_write) item_xover_close();
	if (fxov!=NULL) fclosep(fxov);
	stat_start(X_XOPEN);
	fxov = myfopen(file_xover(grp),F_READ);
	stat_end(X_XOPEN);
	/*	fxov = myfopen(file_xover(grp),"rb");*/
	if (fxov==NULL) {
		fimsg("Couldn't find a {%.200s} file so I will create it\n",file_xover(grp));
		xov_grp = grp;
		xov_write = FALSE;
		xov_last = 0;
		return NULL;
	}
#ifndef VMS
    setvbuf(fxov,freadbf,_IOFBF,sizeof(freadbf));
#endif
	xov_write = FALSE;
	xov_first = -1;
	xov_last = -1;
	xov_lastread = -1;
	xov_grp = grp;
	xov_gap1 = -1;
	xov_gap2 = -1;
isopen:;
	   if (num<xov_first) return NULL;
	   if (num<=xov_lastread) {
		   dmsg("xover: Seek Start %d %d\n",num,xov_lastread);
		   fseek(fxov,0,SEEK_SET);
		   xov_lastread = -1;
	   }
	   if (xov_lastread<num-20) {
		   if (!config_true(C_XOVER_SEEK_DISABLE)) {
			   if (!getnext) xover_seek(fxov,grp,num);
		   }
	   }
	   
	   return xov_read_one(grp,num);
} 
char *xov_fgets( char *string, int n, FILE *stream )
{
	
	return fgets(string,n,stream);
}
void xover_copy_file(int grp)
{
	FILE *fin,*fout;
	char fname[BFSZ];
	char outname[BFSZ];
	int r;
	static int done;
	char bf[BFSZ];
	
	if (done) return; /* Only do this once */
	done = TRUE;
	item_xover_close();
	ncpy(fname,file_xover(grp),300);
	ncpy(outname,file_log("xover2.bug"),300);
	emsg("Copying corrupt xover file to (%.200s)\n",outname);
	
	fin = fopen(fname,"rb");
	if (fin==NULL) {
		filemsg("Could not open %.200s to make copy of it\n",fname);
		return;
	}
	fout = fopen(outname,"wb");
	if (fout==NULL) {
		filemsg("Could not open %.200s\n",outname);
		fclosep(fin);
		return; 
	}
	for (;!feof(fin);) {
		r = fread(bf,1,BFSZ-1,fin);
		if (r<=0) break;
		fwrite(bf,1,r,fout);
	}
	fclosep(fin); fclosep(fout);
	
}

String *xov_read_one(int grp, int num)
{
	static String *x;
	static char bf[MAX_XOVER];
	int n= -1;
	int len;
	if (x==NULL) x = str_new();
	
	/*imsg("xover: read_one %.200s:%d\n",group_name(grp),num);*/
	for (;!feof(fxov);) {
		if (xov_fgets(bf,MAX_XOVER-3,fxov)==NULL) break;
		len = strlen(bf);
		if (bf[len-2]=='\r') {
			bf[len-2] = '\n';
			bf[len-1] = 0;
			len--;
		}
		if (bf[len-1] != '\n') {
			if (is_readonly()) {
				imsg("Incomplete line, ok as file may not be flushed (item=%d+1) (%.200s)\n",xov_lastread,file_xover(grp));
				continue;
			}
			
			/* Also if during expire it's ok */
			if (is_expiring()) {
				if (config_true(C_SPAWN_EXPIRE)) {
					imsg("Incomplete line during expire, ok as file may not be flushed (item=%d+1) (%.200s)\n",xov_lastread,file_xover(grp));
					continue;
				}
			}
			
			/* Xover file corrupt, delete it */
			filemsg("Error reading xover file, deleting it {%.200s} \n",file_xover(grp));
			item_xover_close();
			xover_copy_file(grp);
			xov_first = -1;
			xov_lastread = -1;
			remove(file_xover(grp));
			return NULL;
		}
		if (len>2) if (bf[len-2]=='\r') {
			bf[len-2] = '\n';
			bf[len-1] = 0;
		}
		n = atoi(bf);
		/*imsg("xover: read_one %.200s:%d %d\n",group_name(grp),num,n);*/
		if (xov_first<0) xov_first = n;
		if (xov_lastread>=0) if (xov_lastread > n) {
			wmsg("xov: Xover file out of order (3) ? %.200s:%d %d %d\n",group_name(grp),n,xov_lastread,ftell(fxov)); 
			/*xov_fgets(bf,MAX_XOVER-3,fxov);
			emsg("Next line in file (%.200s)\n",bf);			
			
			  
			*/
			/*xover_copy_file(grp);*/
			xidle_sort(grp);
			return NULL;
			
			/*			g_list_save();
			sort_xover(grp);
			g_list_restore();*/
			
		}
		xov_lastread = n;
		xov_gap1 = xov_gap2;
		xov_gap2 = n;
		
		if (n>num) return NULL;
		if (n==num) {
			strcpy(bf+len-1,EOL);
			str_copysn(x,bf);
			return x;
		}
	}
	xov_last = xov_lastread;
	if (xov_last<0) xov_last = 0;
	return NULL;
}



void xover_sort_block(int grp, int *block);

void sort_xover(int grp)
{
	int block;
	/*	Read in list of valid items
	Then sort it in chunks of 1000 and write it out
	to new one,
	*/
	char bf[BFSZ];
	int start = group_getstart(grp);
	int end = group_getend(grp);
	int max = group_getrange(grp)+1000;
	
	wmsg("Sorting groups xover file into order %.200s %d items %.200s\n",group_name(grp),max-1000,file_xover(grp));
	
	item_xover_close();
	remove(file_xov_srt());
	for (block=start; block<end; ) {
		xover_sort_block(grp,&block);
		if (block<=0) break;
	}
	if (remove(file_xover(grp))!=0) {
		fimsg("Unable to remove (%.200s) \n",file_xover(grp));
	}
	strcpy(bf,file_xov_srt());
	if (rename(bf,file_xover(grp))!=0) fimsg("Unable to rename to (%.200s) \n",file_xover(grp));
	wmsg("Sort completed\n");
}
void xover_sort_block(int grp, int *block)
{
	int nums[5000];
	String *vals[5000];
	int ngot=0;
	int i;
	int len;
	char *s;
	FILE *f;
	int upto=0;
	int n;
	String *hdr,*x;
	char bf[MAX_XOVER];
	
	nums[0] = 0; vals[0] = 0; /* keeps lint happy */
	n = g_list_start(grp);
	for (;(n>0) && (n<*block);) n = g_list_next();	
	if (n<=0) {*block = -1; return;}
	for (;n>0;) {
		nums[ngot++] = n;
		if (ngot>4900) break;
		n = g_list_next();	
	}
	if (n>0) *block = n+1;	else *block = 0;
	imsg("Sorting block of %d items\n",ngot);
	
	for (i=0; i<ngot; i++) vals[i] = str_new();		
	
	/* Scan existing xover file and store all valid ones in nums array */
	stat_start(X_XOPEN);
	f = fopen(file_xover(grp),F_READ);
	stat_end(X_XOPEN);
	if (f==NULL) goto gaps;
	for (;!feof(f);) {	
		if (fgets(bf,MAX_XOVER-3,f)==NULL) break;
		len = strlen(bf);
		if (bf[len-2]=='\r') {
			bf[len-2] = '\n';
			bf[len-1] = 0;
			len--;
		}
		if (bf[len-1] != '\n') continue;
		n = atoi(bf); 
		if (n==0) continue;
		for (;upto<ngot;upto++) if (nums[upto]==n) goto found;
		for (upto=0;upto<ngot;upto++) if (nums[upto]==n) goto found;
		continue;
found:		str_copysn(vals[upto],bf);
	}
	fclosep(f);
	
gaps:
	/* Then fill in gaps by fetching from database */
	for (i=0; i<ngot; i++) {
		if (str_len(vals[i])>0) continue;
		hdr = item_head(grp,nums[i]);
		if (hdr==NULL) continue; 
		x = xover_make(hdr,nums[i]);
		ncpy(bf,str_s(x),MAX_XOVER-2);
		s = strstr(bf,EOL);
		if (s==NULL) continue; 
		imsg("Fill in gap %.200s:%d\n",group_name(grp),nums[i]);
		strcpy(s,"\n");
		str_copysn(vals[i],bf);
	}
	
	
	/* Then write out to new file */
	stat_start(X_XOPEN);
	f = fopen(file_xov_srt(),F_APPEND);
	stat_end(X_XOPEN);
	if (f==NULL) {
		filemsg("Could not write new file %.200s\n",file_xov_srt());
		goto giveup;	
	}
	for (i=0; i<ngot; i++) {
		if (str_len(vals[i])<=5) continue;
		fwrite(str_s(vals[i]),1,str_len(vals[i]),f);
	}
	
	fclosep(f);
giveup:	
	
	for (i=0; i<ngot; i++) str_free(vals[i]);		
}
static int xov_total;
void item_xhash_write(int grp, int num, String *xitem, int force)
{
	char bf[MAX_XOVER];
	int r;
	char reason[BFSZ];
	if (str_len(xitem) > MAX_XOVER-6) {
		dmsg("xhash: don't write if is too long %d bytes\n",str_len(xitem));
		return;
	}
	if (group_getfrom(grp)>num) {
		dmsg("xhash: Don't write downloading message\n");
		return; /* Don't write 'downloading' message to xov file */
	}
	ncpy(bf,str_s(xitem),MAX_XOVER-2);
	r = xhash_add(group_getgroupid(grp),num,bf,strlen(bf),reason);
	if (!r) imsg("xhash_write %s\n",reason);
}
void item_xover_write(int grp, int num, String *xitem, int force)
{
	char bf[MAX_XOVER];
	char *s;
	int xlen;
	if (!group_isxover(grp)) {
		dmsg("xhash: Group is not xover (%d)\n",grp);
		return;
	}
	if (config_true(C_XOVER_XHASH)) {
		item_xhash_write(grp,num,xitem,force);
		return;
	}
	if (is_readonly()) return;
	if (str_len(xitem) > MAX_XOVER-6) return;
	if (group_getfrom(grp)>num) return; /* Don't write 'downloading' message to xov file */
	if (grp==xov_grp) {
		if (!force) {
			if (xov_last>=0) if (num<=xov_last) return;
			if (xov_last<0) return;
		}
		if (!xov_write) item_xover_close();
	} else {
		item_xover_close();
	}
	if (fxov==NULL) {
		stat_start(X_XOPEN);
		fxov = myfopen(file_xover(grp),F_APPEND);
		stat_end(X_XOPEN);
		dmsg("xover: Open write {%.200s} %d\n",file_xover(grp),grp);
		if (fxov==NULL) {filemsg("Cannot append to {%.200s} \n",file_xover(grp)); return;}
		xov_write = TRUE;
		xov_grp = grp;
	}
	
	ncpy(bf,str_s(xitem),MAX_XOVER-2);
	
	s = strstr(bf,EOL);
	if (s==NULL) return;
	strcpy(s,"\n");
	xlen = strlen(bf);
	if ((int) fwrite(bf,1,xlen,fxov)!=xlen) {
		filemsg("Failed writing xover file {%.200s} please delete it\n",file_xover(grp));
	}
	if (!config_true(C_XOVER_MOVED)) {
		diskuse_add(xlen);
	}
	xov_total += xlen;
	
	xov_last = atoi(bf); /* Adjust end */
	return;
}
static int xover_ncopy,xover_nfill;
char *xov_expire_status(void)
{
	static char bf[BFSZ];
	lprintf(bf,BFSZ-1,"fillin %d, copy %d, cancel=%d",xover_nfill,xover_ncopy,total_cancel);
	return bf;
}
void xov_total_clear(void)
{
	xover_ncopy = 0;
	xover_nfill = 0;
	xov_total = 0;
}
void xov_total_find2(int h);
void xov_total_find(void)
{
	static int done_init;
	int i;
	int tot;
	if (done_init) return;
	done_init = TRUE;
	
	if (config_true(C_XOVER_HASH)) {
		tot = 0;
		for (i=0; i<100; i++) {
			lib_createdir(file_xwork("",i));
			xov_total_find2(i);
			tot += xov_total;
		}
		xov_total = tot;
	} else {
		lib_createdir(file_xwork("",-1));
		xov_total_find2(-1);
	}
	imsg("xover: total storage is %d\n",xov_total);
}
void xov_total_find2(int h)
{
	static Sary *ls;
	int i;
	char bf[BFSZ];
	char xdir[BFSZ],xwild[BFSZ];
	char *s;
	int gid,grp;
	
	ncpy(bf,file_xwork("*.xov",h),BFSZ-100);
	ls = s_alloc();
	lib_dir_split(bf,xdir,xwild);
	imsg("xover: Scanning disk for xover files hash=%d\n",h);	
	s_wild(ls,xdir,xwild);
	xov_total = lib_wild_total()*100;
	for (i=0; i<s_len(ls); i++) {
		s = s_line(ls,i);
		if (strlen(s)>3) {
			gid = atoi(s+2);
			grp = gid_to_group(gid);
			if (grp>=0) if (!group_isxover(grp)) {
				if (!is_readonly()) {
					imsg("Removing xover file that is no longer used (%.200s)\n",s_line(ls,i));
					remove(file_xwork(s_line(ls,i),h));
				}
			}
		}
	}
	if (xov_total==0) for (i=0; i<s_len(ls); i++) {
		xov_total += lib_filesize(file_xwork(s_line(ls,i),h));
	}
	s_free(ls);
}
int xov_total_get(void)
{
	if (xov_total==0) {
		xov_total_find();
	}
	if (xov_total<0) return 0;
	return xov_total/1000;
}

/*
New one.

*/
int xidle_sorted_new(int grp)
{
	stat_start(X_XSORT);
	if (item_expire_xover_idle(grp)) {
		stat_end(X_XSORT);
		wmsg("xidle: Sort completed (%s)\n",group_name(grp));
		return TRUE;
	} else {
		stat_end(X_XSORT);
		return FALSE;
	}
}


/*

  static int i;
  static int g;
  static int xtot;
  static int start_time;
  static int inxov;
  int tt;
  start_time = lib_gmtime();
  
	if (inxov) goto xov_again;
	
	  next_one:
	  gupto = g;
	  if (g==0) {
	  t_xover = 0;
	  xtot = xov_total_get();
	  xov_total_clear();
	  xover_a = 0;
	  nntp_setsleep(0); 
	  }
	  if (g<group_n()) for (;(!group_isxover(g)) && (g<group_n());) {
	  g++;
	  if (g>=group_n()) break;
	  }
	  if (g>=group_n()) {
	  xtot = xov_total_get() - xtot;
	  imsg("xover: Xover data now takes %d MB changed by %dK \n",xov_total_get()/1000,xtot);
	  if (!config_true(C_XOVER_MOVED)) {
	  diskuse_add(xtot*1000);		
	  }
	  g = 0;
	  nntp_setsleep(3);               
	  end_time = lib_gmtime();
	  inpurge = FALSE;
	  inxover = FALSE;
	  suck_expire_end();
	  return;
	  }
	  inxov = TRUE;
	  xov_again:
	  tt = lib_gmtime();
	  if (item_expire_xover(g)) {
	  inxov = FALSE;
	  }
	  t_xover += lib_gmtime() - tt;
	  if (inxov) {
	  return;
	  }
	  xover_a++;
	  g++;
	  if ((lib_gmtime()-start_time)<1) goto next_one;
	  return;
	  
		
*/

static int skip_xover_fast;
void item_expire_xover_fast(int grp);
int item_expire_xover_idle(int grp)
{
	int n;
	char *s;
	int i;
	char bf[MAX_XOVER];
	/*	char oldname[BFSZ];*/
	FILE *f;
	int zmax;
	String *x,*hdr;
	char newname[BFSZ];
	int start = lib_gmtime();
	static int isbusy;
	static int upto,upto_grp;
	
	if (isbusy) grp = upto_grp;
	strcpy(newname,file_xov_new());
	strcat(newname,"2");
	if (config_true(C_XOVER_ONDEMAND)) {
		remove(file_xover(grp));
		return TRUE;
	}
	if (config_true(C_XOVER_HASH)) return TRUE;
	in_expire_xover = TRUE;	
	wcache_write_one(grp,-1);
	stat_start(X_XOPEN);
	if (!isbusy) {
		wmsg("Idle Expire xover group %.200s\n",group_name(grp));
		f = fopen(newname,F_WRITE);
	} else {
		f = fopen(newname,F_APPEND);
		imsg("Idle Expire xover group %.200s, upto %d\n",group_name(grp),upto);
	}
	stat_end(X_XOPEN);
	if (f==NULL) { filemsg("Couldn't write to %.200s\n",newname); in_expire_xover = FALSE; return TRUE; }
	
	n = g_list_start(grp);
	if (isbusy) {
		for (;n>=0;) {
			if (n==upto) break;
			n = g_list_next();
		}
	}
	isbusy = FALSE;
	zmax = max_items();
	
	for (i=0; (i<zmax) && (n>=0); i++)  {
		if (i>50) if ((i%20)==19) if (lib_gmtime()-start>expire_quanta()) {
			upto = n;
			upto_grp = grp;
			in_expire_xover = FALSE;
			fclosep(f);
			isbusy = TRUE;
			return FALSE;
		}
		group_xov_nohits(TRUE);
		x = item_xover_read(grp,n,i!=0);	
		group_xov_nohits(FALSE);
		if (x==NULL) {
			xover2_nfill++;
			hdr = item_head(grp,n);
			if (hdr==NULL) {
				xover2_nmiss++; xover2_nfill--;
				dmsg("xov: fill in failed %.200s:%d\n",group_name(grp),n);
				goto cont;     /* Item doesn't exist */        
			}
			x = xover_make(hdr,n);
			if (x==NULL) {
				xover2_nmiss2++; xover2_nfill--;
				goto cont;
			}
			imsg("xov: fill in %.200s:%d %d\n",group_name(grp),n,str_len(hdr));
		} else {
			xover2_ncopy++;
		}
		ncpy(bf,str_s(x),MAX_XOVER-2);
		s = strstr(bf,EOL);
		if (s==NULL) goto cont;
		strcpy(s,"\n");
		xov_total += strlen(bf);
		if (fwrite(bf,1,strlen(bf),f)!=strlen(bf)) {
			filemsg("Failed writing new xover file {%.200s} please delete it\n",newname);
		}
cont:	 	n = g_list_next();
	}
	isbusy = FALSE;
	item_xover_close();
	xov_first = -1;
	xov_lastread = -1;
	fclosep(f);
	if (remove(file_xover(grp))!=0) {
		if (errno!=ENOENT) {
			fimsg("Remove of %.200s failed, not serious, using rename\n",file_xover(grp));	
			
			slave_flush_cmd(TRUE,"onlyclose"); /* Tell it to close all files and wait*/
			if (remove(file_xover(grp))!=0) {
				fimsg("Remove of %.200s failed again\n",file_xover(grp));
			}
			if (rename(newname,file_xover(grp))!=0) fimsg("Rename to %.200s failed\n",file_xover(grp));
			slave_flush(FALSE); /* Tell it to close all files and wait*/		
			
			imsg("Expire_xover returning\n");
			in_expire_xover = FALSE;
			return TRUE;
		}
		
	}
	if (rename(newname,file_xover(grp))!=0) fimsg("Rename to %.200s failed\n",file_xover(grp));
	imsg("Expire_xover returning\n");
	in_expire_xover = FALSE;
	return TRUE;
}
int item_xhash_expire(int grp)
{
	int gid = group_getgroupid(grp);
	xhash_expire(grp,gid,group_getstart(grp),group_getend(grp),FALSE);
	return TRUE;
}
int item_expire_xover(int grp)
{
	int n;
	char *s;
	int i;
	char bf[MAX_XOVER];
	FILE *f;
	String *x,*hdr;
	char newname[BFSZ];
	int start = lib_gmtime();
	static int isbusy;
	static int upto,upto_grp;
	int zmax;
	
	if (config_true(C_XOVER_XHASH)) return item_xhash_expire(grp);
	
	if (isbusy) grp = upto_grp;
	strcpy(newname,file_xov_new());
	if (config_true(C_XOVER_ONDEMAND)) {
		remove(file_xover(grp));
		return TRUE;
	}
	in_expire_xover = TRUE;
	
	wcache_write_one(grp,-1);
	if (!skip_xover_fast) {
		
		if (config_true(C_XOVER_EXPIRE_NORMAL)) goto normal;
		if (config_true(C_XOVER_EXPIRE_FAST) || get_expire_mode() ) {
			item_expire_xover_fast(grp);
			return TRUE;
		}
	}
	
normal:
	stat_start(X_XOPEN);
	if (!isbusy) {
		wmsg("Expire xover group %.200s\n",group_name(grp));
		f = fopen(newname,F_WRITE);
	} else {
		f = fopen(newname,F_APPEND);
		imsg("Expire xover group %.200s, upto %d\n",group_name(grp),upto);
	}
	stat_end(X_XOPEN);
	if (f==NULL) { filemsg("Couldn't write to %.200s\n",newname); in_expire_xover = FALSE; return TRUE; }
	
	n = g_list_start(grp);
	if (isbusy) {
		for (;n>=0;) {
			if (n==upto) break;
			n = g_list_next();
		}
	}
	isbusy = FALSE;
	zmax = max_items();
	
	for (i=0; (i<zmax) && (n>=0); i++)  {
		if (i>50) if ((i%20)==19) if (lib_gmtime()-start>expire_quanta()) {
			if (!get_expire_mode()) { /* No need to return if in expire process */
				upto = n;
				upto_grp = grp;
				in_expire_xover = FALSE;
				fclosep(f);
				isbusy = TRUE;
				return FALSE;
			}
		}
		group_xov_nohits(TRUE);
		x = item_xover_read(grp,n,i!=0);	
		group_xov_nohits(FALSE);
		if (x==NULL) {
			xover_nfill++;
			hdr = item_head(grp,n);
			if (hdr==NULL) {
				dmsg("xov: fill in failed %.200s:%d\n",group_name(grp),n);
				goto cont;     /* Item doesn't exist */        
			}
			x = xover_make(hdr,n);
			if (x==NULL) goto cont;
			imsg("xov: fill in %.200s:%d %d\n",group_name(grp),n,str_len(hdr));
		} else {
			xover_ncopy++;
		}
		ncpy(bf,str_s(x),MAX_XOVER-2);
		s = strstr(bf,EOL);
		if (s==NULL) goto cont;
		strcpy(s,"\n");
		xov_total += strlen(bf);
		if (fwrite(bf,1,strlen(bf),f)!=strlen(bf)) {
			filemsg("Failed writing new xover file {%.200s} please delete it\n",newname);
		}
cont:	 	n = g_list_next();
	}
	isbusy = FALSE;
	item_xover_close();
	xov_first = -1;
	xov_lastread = -1;
	fclosep(f);
	if (remove(file_xover(grp))!=0) fimsg("Remove of %.200s failed, not serious\n",file_xover(grp));
	if (rename(newname,file_xover(grp))!=0) fimsg("Rename to %.200s failed\n",file_xover(grp));
	imsg("Expire_xover returning\n");
	in_expire_xover = FALSE;
	
	if (get_expire_mode()) {
		imsg("Tell master to patch this group\n");
		xtell_master_patch(group_name(grp));
	}
	return TRUE;
}
void item_expire_xover_fast(int grp)
{
	char newname[BFSZ];
	char bf[MAX_XOVER];
	static char finbf[32000];
	static char foutbf[32000];
	FILE *f;
	int nok = 0;
	static int last=0;
	int len,n;
	int min,max,range;
	int ndel=0;
	int skip=FALSE;
	
	strcpy(newname,file_xov_new());	
	min = group_getstart(grp);
	max = group_getend(grp);
	range = group_getrange(grp);
	item_xover_close();
	stat_start(X_XOPEN);
	f = fopen(newname,F_WRITE);
	stat_end(X_XOPEN);
	if (f==NULL) {
		filemsg("Could not write (%.200s)\n",newname);
		return;
	}
#ifndef VMS
	setvbuf(f,foutbf,_IOFBF,sizeof(foutbf));
#endif
	stat_start(X_XOPEN);
	fxov = myfopen(file_xover(grp),F_READ);
	stat_end(X_XOPEN);
	if (fxov==NULL) {
		fclosep(f);
		return ;
	}
#ifndef VMS
	setvbuf(fxov,finbf,_IOFBF,sizeof(finbf));
#endif
	imsg("Expire Xover Fast %d->%d %d (%.200s)\n",min,max,range,group_name(grp));
	last = 0;
	for (;!feof(fxov);) {
		if (fgets(bf,MAX_XOVER-3,fxov)==NULL) break;
		len = strlen(bf);
		if (bf[len-2]=='\r') {
			bf[len-2] = '\n';
			bf[len-1] = 0;
			len--;
		}
		if (bf[len-1] != '\n') {
			if (feof(fxov)) {
				imsg("Corrupt but at end of file so thats ok (%.200s) (%.200s)\n",group_name(grp),file_xover(grp));
				skip = TRUE;
				break;
			}
			imsg("Corrupt line in xover file, group %.200s %.200s\n",group_name(grp),file_xover(grp));
			continue;
		}
		n = atoi(bf);
		if (n<last) {
			continue;
		}
		last = n;
		if (n<(max-range-10)) {
			ndel++;
			continue;
		}
		xov_total += len;
		nok++;
		if ((int) fwrite(bf,1,len,f)!=len) {
			filemsg("Failed writing new xover file {%.200s} please delete it\n",newname);
		}
	}
	fclosep(f);
	fclosep(fxov);
	fxov = NULL;
	xov_first = -1;
	xov_lastread = -1;
	if (!skip) {
		if (remove(file_xover(grp))!=0) fimsg("Remove of %.200s failed, not serious\n",file_xover(grp));
		if (rename(newname,file_xover(grp))!=0) fimsg("Rename to %.200s failed\n",file_xover(grp));
		imsg("Expire_xover returning deleted %d items, left %d\n",ndel,nok);
	}
	in_expire_xover = FALSE;
}


int item_read_check_group(int grp)
{
	int n;
	int i;
	int allok = TRUE;
	String *x;
	int zmax;
	
	imsg("Read check group %.200s\n",group_name(grp));
	n = g_list_start(grp);
	zmax = max_items();
	
	for (i=0; (i<zmax) && (n>=0); i++)  {
		x = item_body(grp,n);	
		if (x==NULL) {
			allok = FALSE;
			emsg("Read failed %.200s:%d\n",group_name(grp),n);
		} 
		x = item_head(grp,n);	
		if (x==NULL) {
			allok = FALSE;
			emsg("Read failed %.200s:%d\n",group_name(grp),n);
		} 
		n = g_list_next();
	}
	return allok;
}
/*--------------END OF  xover stuff --------------*/
int check_body(String *body)
{
	char *s = str_s(body) + str_len(body) - 2;
	if (str_len(body)<2) {
		wmsg("Retrieved item with nothing in it? %d bytes\n",str_len(body));
		return FALSE;
	}
	if (strcmp(s,"\015\012")!=0) {
		emsg("Retrieved item did not end with CRLF, throwing it away {%.200s} \n",s);
		return FALSE;
	}
	
	if (config_true(C_NAKED_REMOVE)) {
		if (!item_isascii(body)) {
			imsg("Dropping item with naked cr or lf characters\n");
			return FALSE;	
		}
	}
	
	return TRUE;
}
int check_head(String *body, int g, int n)
{
	int i;
	char bf[200];
	int j;
	char *s = str_s(body) + str_len(body) - 2;
	if (str_len(body)<2) {
		emsg("Retrieved item with nothing in its head? \n");
		return FALSE;
	}
	if (strcmp(s,"\015\012")!=0) {
		for (j = -1; j<2; j++) {
			imsg("Byte [%d]  %d\n",j,s[j]);
		}
		emsg("FATAL: Retrieved item head did not end with CRLF, throwing it away \n");
		return FALSE;
	}
	s = str_s(body);
	for (i=0; (*s!=0) && (i<80); i++,s++) {
		if (isspace(*s)) {
			ncpy(bf,str_s(body),80);			
			emsg("Retrieved head did not contain any colon, %.200s:%d {%.200s} \n",group_name(g),n,bf);
			show_mrec(g,n);
			return FALSE;
		}
		if (*s==':') return TRUE;
	}
	return TRUE;
}
static int tryagain;
int item_tryagain(void)
{
	return tryagain;
}

#define MAX_GRPS 500 /* MAX cross posts in one item */
#define BFHEAD 10000 /* Maximum size for header line we can process */
/* Increase to 32000 as soon as bugs found */
/*
Things to fix:
copy realgrps, xgrps, should uise bfhead size not bfsz
*/
static int sum_posts,sum_cross,sum_start;
static int xspeed_test;
void item_speed_test(int x)
{
	xspeed_test = x;
	if (x) item_speed_reset(); 	
}
int item_speed_reset(void)
{       
	sum_start = lib_gmtime();
	sum_posts = 0;
	sum_cross = 0;
	return TRUE;
}
char *item_speed(void)
{
	static char bf[200];
	int delay=lib_gmtime()-sum_start;
	if (delay==0) delay = 1;
	lprintf(bf,200-1,"Posts %d, Cross %d, Time %d,   Speed %g  %g /second \n",
		sum_posts,sum_cross,delay,  ((double) sum_posts)/delay
		, ((double) sum_cross)/delay);  
	return bf;
}
static int fileonly = FALSE;
void item_fileonly(void)
{
	fileonly = !fileonly;   
	if (fileonly) emsg("Now only filing incoming articles\n");
	else emsg("Now storing incoming articles properly again\n");
}
static int ihave_chan = -1;
void item_setchan(int c)
{
	ihave_chan = c;
	if (c<0) feed_setipname("local");
	else feed_setipname(chan_getipname(c));
}
static int isuucp;
int item_setuucp(int x)
{
	isuucp = x;
	return TRUE;
}
static int isheader;
static char header_group[BFSZ];
void item_setheader(int x, char *grp)
{
	if (grp==NULL) return;
	isheader = x;
	ncpy(header_group,grp,BFSZ-100);
}
int xitem_suck(String *item);
int item_suck(String *item)
{
	static int lvl;
	int r=TRUE;
	String *xadd,*xnew;
	if (sum_start==0) item_speed_reset();
	sum_posts++;
	xadd = xnew = NULL;
	if (config_true(C_ITEM_ADD)) {
		xadd = str_new();
		xnew = str_new();
		if (str_read(xadd,file_config("item.add"))) {
			str_copy(xnew,item);
			str_add(xnew,xadd);
			item = xnew;
		}
	}
	if (lvl!=0) {
	/*		this occurs normally when part processing is occuring
	emsg("mutex FATAL lock must have failed\n");
	dnews_ex("mutex failed");
		*/
	}
	stat_start(X_STORE);
	lvl++;	
	if (!xspeed_test) r = xitem_suck(item);
	lvl--;
	stat_end(X_STORE);
	if (config_true(C_ITEM_ADD)) {
		str_free(xadd); 
		str_free(xnew);
	}
	
	return r;
}
static char grps[BFHEAD],opath[BFHEAD],distrib[BFHEAD];
static char bf[BFHEAD],xgrps[BFHEAD],realgrps[BFHEAD],inxref[BFHEAD];

static int feed_isoff;
void item_stopfeeds(int t)
{
	feed_isoff = t;
} 
static int isreplic;
void item_set_xreplic(int x)
{
	isreplic = x;
	group_use_xref(x);
}
static int isimport;
void item_setimport(int x) 
{isimport = x;}
static int iscrosspost;
int item_iscrosspost(void)
{
	return iscrosspost;
}
void item_rejectmid(char *mid)
{
	if (!config_true(C_REJECT_ADDHIST)) return;
	if (!hist_exists(mid)) {
		hist_add(mid,"1-Jan-95 1:01:01 GMT","","");
	}
}
int item_hasbody(String *xitem);
int item_hasbody(String *xitem)
{
	char *s;
	if (xitem==NULL) return FALSE;
	s = str_s(xitem);
	if (s==NULL) return FALSE;
	
	s = mystrstr(s,"\r\n\r\n");
	if (s==NULL) return FALSE;
	if (isheader) return TRUE;
	s+=4;
	if (*s==0) {
		if (config_true(C_REJECT_EMPTY)) {
			return FALSE;
		}
		str_addsn(xitem,"\r\n"); /* Because we don't like them to be empty. */
	}
	return TRUE;
}
int item_isascii(String *xitem)
{
	char *s;
	int max = 2600;
	if (xitem==NULL) return FALSE;
	s = str_s(xitem);
	if (s==NULL) return FALSE;
	s++;
	
	for (;*s!=0;s++,max--) {
		if (*s=='\r') if (s[1]!='\n') return FALSE;
		if (*s=='\n') if (s[-1]!='\r') return FALSE;
		if (max<0) break;
	}
	return TRUE;
}
#define PATHS_NEW file_work("inpaths.new")
#define PATHS_DAT file_work("inpaths.dat")
int log_paths(String *opath);
int log_paths(String *xopath)
{
	time_t now;
	struct tm *t; 
	static FILE *f;
	static int lastday;
	char bf[BFSZ];
	struct tm atm;
	
	now = time(NULL);
	t = safe_localtime(&now,&atm);
	
	if (f==NULL) f = myfopen(PATHS_NEW,"a");
	if (f==NULL) return FALSE;
	fprintf(f,"%.200s\n",str_s(xopath));
	if (t->tm_mday!=lastday) {
		lastday = t->tm_mday;
		fclosep(f);
		strcpy(bf,PATHS_DAT);
		rename(PATHS_NEW,bf);
		f = NULL;
	}
	return TRUE;
}
static int ignore_age;
int set_ignoreage(int x)
{
	ignore_age = x;
	return TRUE;
}
static int ignore_control;
int set_ignorecontrol(int x)
{
	ignore_control = x;
	return TRUE;
}
static int ignore_hist;
void set_ignorehist(int x)
{
	ignore_hist = x;
}
static int was_mod;
int item_was_mod(void)
{
	return was_mod;
}

static char last_mid[BFSZ];
char *item_last_mid(void)
{
	return last_mid;	
}
void db_setsplit(int x);
static int gpointid[MAX_GRPS];
int xitem_suck(String *item)
{       
	char *gpoint[MAX_GRPS];
	int ngpoint;
	int r;
	int ni,g,i,j;
	int ngrp,grp[MAX_GRPS],num[MAX_GRPS];
	char gpath[BFSZ],hcontrol[BFSZ];
	char msgid[BFSZ],subject[BFSZ],approved[BFSZ];
	char super[BFSZ],tmp[BFSZ];
	char refs[BFSZ];
	static String *hline,*xlink,*xref,*xpath,*newhead;
	char post_date[BFSZ],expire_date[BFSZ];
	char eoh[10];
	int nosave = FALSE;
	String *xov;
	int ismod = FALSE;

	for (i=0; i<10; i++) gpoint[i] = NULL;
	grp[0] = 0;
	was_mod = FALSE;
	db_setsplit(FALSE);
	set_rej_silent(FALSE);
	msg_show("No reason");
	msg_setlen(str_len(item));
	
	if (fileonly) {
		debug_store_item(item);
		return TRUE;
	}
	
	if (strlen(config_get(C_DEBUG_STORE))>0) debug_store_item(item);
	
	tryagain = FALSE;
	dmsg("item: Item suck starts\n");
	if (str_len(item)==0) {
		msg_reject("Item is completely empty, not even a header");
		msg_show("NULL Item");
		return FALSE;
	}
	if (str_len(item)>1000000) imsg("item: item_suck large item %d bytes\n",str_len(item));
	strcpy(eoh,EOL);
	strcat(eoh,EOL);
	
	if (hline==NULL) {
		hline = str_new();
		xlink = str_new();
		xref = str_new();
		xpath = str_new();
	}
	str_clearbig(hline); 
	str_clearbig(xlink); 
	str_clearbig(xref); 
	str_clearbig(xpath); 
	
	/* Shouldn't do this check here really, but shouldn't hurt */
	ncpy(msgid, get_ahead(item,"Message-ID:"),BFSZ-100);
	ncpy(last_mid,msgid,255);
	msg_id(msgid);
	dmsg("item: Item suck mid (%.200s)\n",xtrunc(msgid));
	if (strlen(msgid)==0) {
		static int n;
		n++;
		msg_reject("No message id in headers"); 
		msg_show("No msgid in headers"); 
		return FALSE;
		if (!str_write(item,file_store(n%10))) {
			emsg("store: Unable to write debug article file %.200s \n",file_store(n%10));
			return FALSE;
		}
		emsg("Stored problem message (%s)\n",file_store(n%10));
		return FALSE;
	}
	if (!is_head_exact(item,"Message-ID: ")) {
		msg_reject("No space after Message-ID:"); 
		msg_show("Message-ID"); 
		return FALSE;
	}
	if (!mid_isok(msgid)) {
		ncpy(tmp,msgid,BFSZ-1);
		wmsg("Rejecting faulty message id (%.200s)\n",tmp);
		msg_reject("Faulty message ID format"); 
		msg_show("Faulty message ID format"); 
		return FALSE;
	}
	iscrosspost = FALSE;
	if (!ignore_hist) {
		stat_start(X_HISTLUP);
		if (hist_gotit(msgid)) {  
			stat_end(X_HISTLUP);
			imsg("item: already got it {%.200s} {%.200s} already exists, ignoring it.\n"
				,msgid,get_ahead(item,"Subject:"));
			msg_reject("Already got message entire item");
			msg_show("Already in history file");
			iscrosspost = TRUE;
			return FALSE;
		}
		stat_end(X_HISTLUP);
	}
	
	if (!item_hasbody(item)) {
		msg_reject("Item body is empty");
		msg_show("Item body is empty");
		return FALSE;
	}
	
	if (!db_is_rerun_filter())
		if (!config_true(C_NAKED_ALLOW)) {
			if (!item_isascii(item)) {
				msg_reject("Item contains naked 'cr' or 'lf' characters, set C_NAKED_ALLOW true");
				msg_show("Item contains naked 'cr' or 'lf' characters, set C_NAKED_ALLOW true");
				return FALSE;
			}
		}
		/* Must add check for items older than REMEMBER  period, don't accept
		them or feed them on */
        ncpy(post_date, get_ahead(item,"Date:"),100 );
        ncpy(refs, get_ahead(item,"References:"),200 );
		
		if (strstr(refs,"<fake@netwin.com.nz>")!=NULL) {
			wmsg("Reject reply to Group download message\n");
			lprintf(bf,BFSZ-1,"Sorry replies not permitted to this message, post a new message. See http://netwinsite.com/suck.htm");
			msg_reject(bf);
			msg_show(bf);
			return FALSE;
		}
		
		if (!isimport) {
			if (!ignore_age) {
				if (expire_toold(post_date)) {
					wmsg("item: Invalid date or too old={%.200s} %.200s\n",post_date,msgid);
					msg_reject("Message is too old");
					msg_show("Message is too old");
					return FALSE;
				}
			}
#ifdef SPAWN_FILTER
			if (strlen(config_get(C_FILTER_CMD))>0) {
				stat_start(X_FILTER);
				r = filter_isok(str_s(item),str_len(item));
				stat_end(X_FILTER);
				if (r>=400)  {
					imsg("filter: code (%d) rejected message %.100s\n",r,msgid);
					if (r==435) if (strlen(msgid)>0) item_rejectmid(msgid);
					set_rej_silent(TRUE);
					msg_reject("External filter rejected message");
					msg_show("External filter rejected message");
					return FALSE;
				}	
				imsg("filter: code (%d) accepted message %.100s\n",r,msgid);
			}
#endif
		}
        ncpy(subject, get_ahead(item,"Subject:"),BFSZ-100 );
        ncpy(super, get_ahead(item,"Supersedes:"),BFSZ-100 );
        ncpy(approved, get_ahead(item,"Approved:"),BFSZ-100 );
        ncpy(grps, get_ahead(item,"Newsgroups:"),BFHEAD-100 );
        strlwr(grps);
        fiddle_order(grps);
		/* if (isheader) fiddle_first(grps,header_group);*/
        ncpy(hcontrol, get_ahead(item,"Control:"),BFSZ-100 );
		
        /* If no control header, then still obey obsolete subject
        based control messages, I don't approve of this I think.  */
        if (strlen(hcontrol)==0) if (strncmp(subject,"cmsg ",5)==0) {
			ncpy(hcontrol,subject+5,BFSZ-100);
        } 
        strcpy(xgrps,"");
        if (strlen(hcontrol)!=0) {
			lprintf(bf,BFSZ-1,"control.%.200s",get_word(hcontrol,0));
			if (group_find(bf)<0) strcpy(bf,"control");
			ncpy(xgrps,bf,BFSZ-100);
			ncpy(realgrps,grps,BFSZ-100);
        }
		
        ncpy(distrib, get_ahead(item,"Distribution:"),BFSZ-100 );
        ncpy(opath, get_ahead(item,"Path:"),BFHEAD-1 );
		
		/*	if (ishdr_old()) {
		group_use_xref(TRUE);
		}
		*/
		if (config_true(C_REPLICATE) || isreplic || ishdr_old()) {
			ncpy(inxref, get_ahead(item,"Xref:"),BFHEAD-1);
			group_xref(inxref);
		}
        /* Split groups into an array */
		
        ngpoint = token_split(grps,gpoint,", \t\015\012",MAX_GRPS-3);
				{
					int k;
					for (k=0; gpoint[k]!=NULL;k++) {
						if (gpoint[k]==1) {
							emsg("Sorry6, groups array, entry %d is corrupt\n",k);
							abort();
						}
					}
				}
		
		/* Check for groups with silly long names */
        for (g=0; g<ngpoint; g++) {
			/**dmsg("post: group (%.200s)\n",gpoint[g]);*/
			if (strlen(gpoint[g])>97) {
				lprintf(bf,BFSZ-1,"Group name too long");
				msg_show(bf);
				msg_reject("%.200s",bf);
				return FALSE;
			}
        }
		
		if (!db_is_rerun_filter()) 
			if (!isimport) {
				if (config_true(C_NOCEM)) {
					dmsg("nocem: NoCem process message now\n");
					nocem_filter(item,gpoint,ngpoint);
					vtrace();
				}
			}
			
			
			/* Check feed is allowed to send us those groups */
			if (!db_is_rerun_filter()) 
				if (ihave_chan>=0) if (config_true(C_IHAVE_ACCESS)) {
					for (g=0; g<ngpoint; g++) {
						if (!chan_canpostgrp(ihave_chan,gpoint[g])) {
							lprintf(bf,BFSZ-1,"access.conf rejected group %.100s",xtrunc(gpoint[g]));
							msg_show(bf);
							msg_reject("%.200s",bf);
							return FALSE;
						}
					}
				} 
				vtrace(); /* Panoia, crash below here */
				if (isuucp) {
					if (strlen(config_get(C_SUCK_UUCP_GROUPS))>0) {
						for (g=0; g<ngpoint; g++) {
							if (!group_match(config_get(C_SUCK_UUCP_GROUPS),gpoint[g])) {
								lprintf(bf,BFSZ-1,"suck_uucp_groups rejected group %.200s from uucp feed",xtrunc(gpoint[g]));
								msg_show(bf);
								msg_reject("%.200s",bf);
								return FALSE;
							}
						}		
					}
				}
				
				/* Added 5.1*/	
				for (i=0; i<ngpoint; i++) {
					gpointid[i] = group_find(gpoint[i]);
				}
				/* Added 5.1*/	
				if (ngpoint==1) if (gpointid[0]==-1) if (config_true(C_NEWGROUP_AUTO)) {
					if (strlen(gpoint[0])<90) {
						wmsg("Auto creating news group (%.200s)\n",gpoint[0]);
						group_new(gpoint[0],"NEWGROUP_AUTO","y","");
						gpointid[0] = group_find(gpoint[0]);
					}
				}
				
				vtrace(); 
				/* Added 5.3 */
				/* Control messages, pretend it's a control group in the header */
				if (strlen(xgrps)!=0) {
					if (db_is_rerun_filter()) return TRUE;
					ngpoint = token_split(xgrps,gpoint,", \t\015\012",MAX_GRPS-3);

				{
					int k;
					for (k=0; gpoint[k]!=NULL;k++) {
						if (gpoint[k]==1) {
							emsg("Sorry7, groups array, entry %d is corrupt\n",k);
							abort();
						}
					}
				}

					for (i=0; i<ngpoint; i++) {
						gpointid[i] = group_find(gpoint[i]);
					}
					/* If it's a cancel message, default to rules_only */ 
					if (!config_true(C_CANCEL_ME)) {
						ncpy(bf,hcontrol,100);
						strlwr(bf);
						if (strstr(bf,"cancel")!=NULL) {
							feed_skipspam(TRUE); 
							goto do_me;
						}
					}
					
					/* This is the default now */
					if (config_true(C_CONTROL_ME)) goto do_me;
					
					if (config_true(C_CONTROL_RULES_ONLY))  {
						feed_skipspam(TRUE); 
						goto do_me;
					}
					goto skip_me;
				}
				
				
do_me:
				vtrace(); 
				{
					int k;
					for (k=0; gpoint[k]!=NULL;k++) {
						if (gpoint[k]==1) {
							emsg("Sorry5, groups array, entry %d is corrupt\n",k);
							abort();
						}
					}
				}
				if (!feed_me(msgid,distrib,gpoint,gpointid,opath,str_len(item),item)) {
					vtrace(); 
					feed_skipspam(FALSE);
					
					if (db_is_rerun_filter()) {
						return FALSE;	
					}
					
					if (strlen(msgid)>0) item_rejectmid(msgid);
					if (config_true(C_ME_CONTINUE) && !feed_wasspam()) nosave = TRUE;
					else {
						lprintf(bf,BFSZ-1,"Feedrule: %.200s",msg_reason());
						msg_reject("%.200s",bf);
						return FALSE; /* We don't want it*/
					}
				}
				feed_skipspam(FALSE);
				if (db_is_rerun_filter()) {
					/*imsg("rerun: feedme was happy\n"); */
					return TRUE;
				}
				
				stat_start(X_PART);
				db_setpart(FALSE);
				if (part_check(subject,msgid,str_s(item),str_len(item))) {
					dmsg("part: Stored part: %.80s\n",subject);
					msg_ok("Item stored in parts buffer");
					stat_end(X_PART);
					return TRUE;
				}
				stat_end(X_PART);
				
skip_me:;
		if (db_is_rerun_filter()) return TRUE;
		vtrace(); 
		if (config_true(C_REBUILD_CANCEL)) {
			if ( (strstr(hcontrol,"cancel")!=NULL)||
				(strstr(hcontrol,"Cancel")!=NULL)||
				(strstr(hcontrol,"CANCEL")!=NULL)) goto always_cancel;
		}
        if (!ignore_control) if (strlen(hcontrol)!=0) {
always_cancel:	vtrace();    /* aNOTHER CRASH AFTER HERE*/
                if (!ctl_process(hcontrol,item)) {
					msg_show("Unwanted control item");
					msg_reject("Control item rejected {%.100s}",hcontrol);
					if ( (strstr(hcontrol,"cancel")!=NULL)||
						(strstr(hcontrol,"Cancel")!=NULL)||
						(strstr(hcontrol,"CANCEL")!=NULL)) {
						if (!hist_exists(msgid)) {
							hist_add(msgid,"1-Jan-95 1:01:01 GMT","","");
						}
						if (feed_ispost()) return FALSE;
						return TRUE;
					}
                }
		}        
		
		
		vtrace();  /* and BEFORE HERE */
		lprintf(bf,BFHEAD-1,"Path: %.200s",lib_hostnamex());
        str_copys(xpath,bf,strlen(bf));
		if (strlen(opath)>0) str_addsn(xpath,"!");
		else str_addsn(xpath,"!not-for-mail");
		
		if (isrebuild) {
			lprintf(bf,BFHEAD-1,"Path: ");
			str_copys(xpath,bf,strlen(bf));
		}
		
		if (!config_true(C_STRIP_ALLPATHS)) {
			str_adds(xpath,opath,strlen(opath));
		}
		if (config_true(C_INPATHS)) {
			log_paths(xpath);
		}
		
        str_adds(xpath,EOL,strlen(EOL));
		
		vtrace(); 
		vtrace(); 
        if (strlen(grps)==0) {
			msg_show("No newsgroups header");
			emsg("item: No Newsgroups in header "); 
			return FALSE;
        }       
        if (strlen(xgrps)!=0) {/* control messages don't get checked for ME */
			ngpoint = token_split(xgrps,gpoint,", \t\015\012",MAX_GRPS-3);
			for (i=0; i<ngpoint; i++) {
				gpointid[i] = group_find(gpoint[i]);
			}
        }
        ngrp = 0;
		
		db_setsplit(FALSE);
        ngrp = 0;
        for (i=0; i<ngpoint; i++) {
			if (i==0) msg_group(gpoint[i]);
			g = gpointid[i] ;
			if (g<0) continue;
			if (group_ismod(g)) ismod = TRUE;
			if (ismod) {
				if (!config_true(C_MODERATED_NOCHECK)) {
					if (ismod) {
						if (strlen(approved)==0) {
							was_mod = TRUE;
							msg_show("Not approved by moderator");
							msg_reject("Not approved by moderator");
							group_addmrej(g);
							post_setmodgrp(g);
							return FALSE; /* We don't want it*/
						}
					}
				}	
			}
		}
        for (i=0; i<ngpoint; i++) {
			g = gpointid[i];
			if (g<0) {
				if (empty_isempty(gpoint[i])) {
					empty_unempty(gpoint[i]);
					g = group_find(gpoint[i]);
				}
			} else {
			}
			if (g<0) dmsg("item: No such group here {%.200s}\n",xtrunc(gpoint[i])); 
			else if (group_isdeleted(g)) imsg("item: ignore message to deleted group %.200s\n",xtrunc(gpoint[i]));
			else {
				if (!group_iscached3(g)) {
					if (config_true(C_CACHEDONLY)) {
						if (isuucp) if (config_true(C_UUCP_UNCACHED)) goto keepit;
						goto next_grp;
					}
				}
				
				if (!db_isregid()) if (!isheader) if (ishdr_old()) if (hdr_matchgroup(group_name(g))) {
					imsg("Skipping cross post to header cached groups from ihave feed %.200s\n",group_name(g));
					goto next_grp;
				}
				
keepit:;
	   /* if (!group_wanted(group_name(g))) goto next_grp; */
	   for (j=0; j<ngrp; j++) if (grp[j]==g) goto next_grp;
	   if (split_check(g)) db_setsplit(TRUE);
	   ni = group_getnext(g);
	   if (ni>=0) {
		   grp[ngrp] = g;
		   num[ngrp++] = ni;
		   hdr_addmap(g,ni);
	   } else {
		   /*imsg("Xref: %.160s\n",get_ahead(item,"Xref:"));*/
	   }
	   if (ngrp+20>MAX_GRPS) {
		   emsg("Item cross posted to too many groups %.200s \n",msgid);
		   ngrp--;
	   }
			}
next_grp:;
        }
		
		str_clear(hline);
        str_clearbig(xref);
        lprintf(bf,BFHEAD-1,"Xref: %.200s",dom_ini_get(C_MYNAME));
        str_copys(xref,bf,strlen(bf));
		/* Create history line and xref line */
        for (g=0; g<ngrp; g++) {
			if (g!=0) str_adds(hline," ",1);
			lprintf(bf,BFSZ-1," %.200s:%d",group_name(grp[g]),num[g]);
			if (g==0) ncpy(gpath,bf+1,BFSZ-100);
			str_adds(xref,bf,strlen(bf));
			lprintf(bf,BFSZ-1,"%.200s/%d",group_name(grp[g]),num[g]);
			str_adds(hline,bf,strlen(bf));
        }
        str_adds(xref,EOL,strlen(EOL));
        if (!config_true(C_XREF_ALWAYS)) if (ngrp<2) str_copys(xref,"",0);
		
		/* Choose which pile this item should be in, if expire_any true then
		choose shortest pile, otherwise choose longest pile */
		gage_choose(grp,ngrp);
		vtrace();
		
        if (ngrp==0) {
			if ((ngpoint>0) && (gpoint[0]!=NULL)) lprintf(bf,BFSZ-1,"No newsgroups in active.dat matched {%.200s}",xtrunc(gpoint[0]));
			else lprintf(bf,BFSZ-1,"No newsgroups match");
			/*imsg("item: %.200s\n",bf);*/
			if (group_n()<10) emsg("item: YOU MAY NEED TO GET ACTIVE.DAT FROM YOUR FEED SITE\n");
			msg_reject("%.200s",bf);
			msg_show(bf);
			if (strlen(msgid)>0) item_rejectmid(msgid);
			return FALSE; /* No valid groups for that item*/
        }
		
        
		
        if (strlen(xgrps)!=0) {
			int gotone=FALSE;
			/* Switch back to real names for feeding but add control group*/
			ngpoint = token_split(realgrps,gpoint,", \t\015\012",MAX_GRPS-3);
			for (i=0; i<ngpoint; i++) {
				gpointid[i] = group_find(gpoint[i]);
				if (gpointid[i]>=0) gotone = TRUE;
			}
			if (!gotone) {
				gpoint[ngpoint] = "control";	         
				gpointid[ngpoint++] = group_find("control");				
				printf("added control back in\n");
			}
			
		}
		
		/* Insert xref,path,Lines, remove old xref path and lines */
		
        newhead = item_fixhead(item,xref,xpath);
        if (!head_replace(item,newhead)) {
			msg_reject("Head replace failed");
			msg_show("Header bad or no body");
			return FALSE;
        }
		/* Write xover data if group is in xover cache */
		stat_start(X_WRITEXOV);
		if (!config_true(C_XOVER_ONDEMAND)) {
			for (g=0; g<ngrp; g++) {
				if (group_isxover(grp[g])) {
					last_len = str_len(item);
					xov = xover_make(item,num[g]);
					if (xov!=NULL) item_xover_write_cached(grp[g],num[g],xov,TRUE);
				}
			}
		}
		stat_end(X_WRITEXOV);
		if (nosave) {
			str_clear(hline);
			str_copysn(hline,"");
			goto skip_store; /* Me didn't match */
		}
		/* Store item and symbolic links */
        lprintf(bf,BFSZ-1,"@%.200s:%d%.200s%.200s@%.200s:%d",group_name(grp[0]),num[0],EOL,EOL
			,group_name(grp[0]),num[0]);
        str_copys(xlink,bf,strlen(bf));
		
		stat_start(X_DBADD);
		/*#ifndef dothisbit*/
        for (g=0; g<ngrp; g++) {
			sum_cross++;
			dmsg("itemxx: Saving to group:%d {%.200s} \n",g,str_s(xlink));
			if (g==0) r = db_add(grp[g],num[g],item); 
			else r = db_add_link(grp[g],num[g]); 
			if (!r) {
				tryagain = TRUE;
				msg_reject("Item save failed, possible disk problem");
				msg_show("Save failed");
				db_setpart(FALSE);
				return FALSE;
			}
			dmsg("item: Item added {%.200s} {%.200s} %d \n",msgid,group_name(grp[g]),num[g]);
        }
		db_setpart(FALSE);
		part_write_complete();
		stat_end(X_DBADD);
		
skip_store:
		/* Feed to anyone else who wants this item*/
		if (isheader) goto nofeed;
		stat_start(X_DBFEED);
		if (!feed_isoff) {
			strcat(gpath,db_cheat_get());
			if (!feed_link(msgid,distrib,gpoint,gpointid,opath,str_len(item),gpath,item)) {
                dmsg("item: No one else wanted it \n"); /* NO one else wants it*/
			}
			if (config_true(C_TOMAIL_ROUTE) || (strlen(get_ahead(item,"X-Gateway-From:"))==0)) {
				tomail_feed(subject,refs,gpointid,ngpoint,gpath,str_len(item));
			}
		}
		stat_end(X_DBFEED);
		
nofeed:
		/* Add line to history file */
        ncpy(post_date, get_ahead(item,"Date:"),100 );
        ncpy(expire_date, get_ahead(item,"Expires:"),100 );
		
		str_lock(hline);
		stat_start(X_WRITEHIST);
        if (!ignore_hist) hist_add(msgid,post_date,expire_date,str_s(hline));
		stat_end(X_WRITEHIST);
		str_unlock(hline);
		if (strlen(super)>0) {
			if (!config_true(C_SUPER_IGNORE)) item_super(super);	
		}
        /* if (strlen(str_s(hline))<8) emsg("item: ***** hline blank \n");*/
        dmsg("item: ------ ITEM added successfully\n");
        msg_ok("Item stored on local system");
		vtrace();
        return TRUE;
}
static int mode_addlost;
void item_setaddlost(int x)
{
	mode_addlost = x;
}
char *item_addlost(String *shead, char *msgid, int maingrp, int mainnum)
{
	static String *hline;
	char *xref;
	int i;
	char *s;
	int g,ngrp,num[1000];
	char *grp[1000];
	char post_date[BFSZ];
	char expire_date[BFSZ],bf[BFSZ];
	
	if (!mode_addlost) return NULL;
	if (hline==NULL) hline = str_new();     str_clear(hline);
	ngrp = 0;
	/* Get xref line to figure out list of groups and numbers */
	/* xref: here comp.os:30 comp.sex:40  */
	xref = get_ahead(shead,"Xref:"); if (xref==NULL) return NULL;
	if (strlen(xref)==0) {
		grp[ngrp] = mystrdup(group_name(maingrp),SD_ITEM);
		num[ngrp++] = mainnum;
	} else {
		s = strtok(xref," \n\r");
		for (;s!=NULL;) {
			s = strtok(NULL,": \n\r"); if (s==NULL) break;
			grp[ngrp] = mystrdup(s,SD_ITEM);
			s = strtok(NULL,": \n\r"); if (s==NULL) break;
			num[ngrp++] = atoi(s);
			if (atoi(s)==0) imsg("addlost warning, zero {%.200s} \n",s);                        
		}               
	} 
	/* Create history line */
	
	for (g=0; g<ngrp; g++) {
		if (g!=0) str_adds(hline," ",1);
		lprintf(bf,BFSZ-1,"%.200s/%d",grp[g],num[g]);
		str_adds(hline,bf,strlen(bf));
	}
	
	ncpy(post_date, get_ahead(shead,"Date:"),100 );
	ncpy(expire_date, get_ahead(shead,"Expires:"),100 );
	str_lock(hline);
	hist_add_fix(msgid,post_date,expire_date,str_s(hline));
	str_unlock(hline);
	for (i=0; i<ngrp; i++)     mystrfree(grp[i],SD_ITEM);
	return str_s(hline); /* Not used ,just a signal */
}

void item_flush(void)
{
	item_xover_close();
	diskuse_flush();
}
/* Used to purge expired items out of the file */
/* Calling sequence: must stay within same file the whole time.
item_purgestart(group,num);
item_purgewrite(group,num,head,body);
...
item_purgeclose();      Close files, write index, rename files
*/
/*
Extract the headers, remove tab and nl's, the headers we want are:
artid|subject|From|Date|Message-Id|References|bytecount|linecount|\
Optional-Header: Stuff

*/

/* MAYBE THIS SHOULD BE 012, OR 015/012   ???? */
int head_lines(String *item);
int head_lines(String *item)
{
	register char *s = str_s(item);
	int n=0;
	if (s==NULL) return n;
	
	for (;*s!=0;s++) {
		if (*s=='\n') n++;
	}
	return n;
}

static char *got[100];
static int nwanted;
static char *wanted[20]={
	"Subject:", "From:", "Date:", "Message-ID:", "References:"
        ,"Bytes:", "Lines:", NULL};
	void init_xover(void)
	{
		char *s;
		int i;
		nwanted = 7;
		s = config_get(C_XOVER_EXTRA1);
		if (strlen(s)==0) s = "Xref:"; 
		if (strstr(s,":")!=NULL) {
			wanted[nwanted++] = mystrdup(s,SD_ITEM);
			wanted[nwanted] = NULL;
		}
		s = config_get(C_XOVER_EXTRA2);
		if (strstr(s,":")!=NULL) {
			wanted[nwanted++] = mystrdup(s,SD_ITEM);
			wanted[nwanted] = NULL;
		}
		for (i=7; i<nwanted; i++) {
			imsg("Extra xover field {%.200s} \n",wanted[i]);
		}
		
	}
	void item_xover_fmt(int chan)
	{
		int i;
		char bf[BFSZ];
		for (i=0; i<7; i++) {
			lprintf(bf,BFSZ-1,"%.200s\r\n",wanted[i]);
			chan_write_s(chan,bf);
		}
		for (i=7; i<nwanted; i++) {
			lprintf(bf,BFSZ-1,"%.200sfull\r\n",wanted[i]);
			chan_write_s(chan,bf);
		}
		chan_write_s(chan,".\r\n");
	}
	int item_isxover(char *hdr)
	{
		int i;
		int len;
		char bf[BFSZ];
		ncpy(bf,hdr,BFSZ-100);
		if (strchr(bf,':')==NULL) strcat(bf,":");
		
		len = strlen(bf);
		for (i=0; i<nwanted; i++) {
			if (strncmpnc(bf,wanted[i],len)==0) if (len==(int) strlen(wanted[i])) {
				return i;
			}
		}
		imsg("item: Not in xover database {%.200s} \n",hdr);
		return -1;
	}
	char *item_getahead_x(int grp, int n, int xfield, int getnext)
	{
		char *p[100];
		static char bf[MAX_XOVER];
		int np;
		char *s,*b;
		String *xov;
		xfield++;
		/*	imsg("get xover %d:%d, field %d %.200s\n",grp,n,xfield,group_name(grp));*/
		xov = item_xover(grp,n,getnext);
		if (xov==NULL) {
			imsg("xhdr: Xover record not found\n");
			return "";
		}
		ncpy(bf,str_s(xov),MAX_XOVER-3);
		np = token_split2(bf,p,"\t",90);
		/*	imsg("Token split {%.200s} {%.200s} {%.200s} %d field=%d\n",p[0],p[1],p[2],np,xfield);*/
		if (np<nwanted) return "";
		if (xfield<=7) return p[xfield];
		else s = p[xfield];
		if (s==NULL) return "";
		s = strchr(s,' ');
		if (s==NULL) return "";
		for(b=s+1;*b!=0;b++) if (*b=='\r') {*b=0; break;}
		return s+1;
	}
	char *detabcr(char *s)
	{
		static char bf[MAX_XOVER+10];
		char *out;
		char c;
		for (out=bf;*s!=0;s++) {
			c = *s;
			if (c=='\r') c = ' ';
			if (c=='\n') c = ' ';
			if (c=='\t') c = ' ';
			*out++ = c;
			if ((out-bf) > MAX_XOVER-1) break;
		}
		*out++ = 0;
		return bf;
	}
	String *xover_make(String *sitem, int num)
	{
        static String *xov;
        char bf[200];
        char *s = str_s(sitem);
        int i;
		int bytes;
        for (i=0;wanted[i]!=NULL;i++) {
			if (got[i] == NULL) got[i] = mymalloc(MAX_XOVER); 
			*got[i] = 0;
        }
		
        if (xov==NULL) xov = str_new();
        lprintf(bf,200-1,"%d\t",num);
        str_clearbig(xov);
        str_copys(xov,bf,strlen(bf));
        s = str_s(sitem);
        for (;s!=NULL && *s!=0;) {
			if (*got[0]!=0) if (strncmp(s,EOL,2)==0) break;
			for (i=0;wanted[i]!=NULL;i++) {
				if (strncmpnc(s,wanted[i],strlen(wanted[i]))==0) {
					ncpy(got[i],get_head(s),MAX_XOVER-1);
					break;
				}       
			}
			if (strncmp(s,"Message-Id:",11)==0) {
				ncpy(got[3],get_head(s),MAX_XOVER-1);
			}       
			for (;*s!=0;s++) {
				if (*s=='\n') break;
			}
			if (*s!=0) s += 1;
        }
		/* Generate byte count as it isn't a real header, must do during store 
		as the item is not loaded at this time. 
		But is illegal header, so should not be done during store (as it may
		change)
		
		  Byte count =  bytes - lines - header_lines + fiddle_factor
		*/
		/*imsg("item: xover bytes %d %d %d \n",item_last_len(),atoi(got[6]),head_lines(sitem));*/
		/*
       	bytes = item_last_len()-atoi(got[6])-rough_head_lines(sitem)-1+1;
		if (bytes<500) bytes = head_lines(sitem)+item_last_len() * 70;
		removing head_lines as it seems pointless and is very slow.
		
		*/
       	bytes = item_last_len()-atoi(got[6])-1+1;
		/*if (bytes<500) bytes = item_last_len() * 70;*/
		if (ishdr()) {
			if (bytes<50) {
				bytes = 10000;
			}
		}
		if (bytes<0) bytes = 1000;
		if (bytes>10000000) bytes = 4000000;
       	lprintf(bf,200-1,"%d",bytes);
       	ncpy(got[5],bf,MAX_XOVER-1);
        for (i=0;(wanted[i]!=NULL) && (i<7);i++) {
			if (i!=0) str_adds(xov,"\t",1);
			str_adds(xov,detabcr(got[i]),strlen(got[i]));
        }
        for (i=7;(wanted[i]!=NULL);i++) {
			if (*wanted[i] != 0) {
				str_adds(xov,"\t",1);
				str_adds(xov,wanted[i],strlen(wanted[i]));
				str_addsn(xov," ");
				str_adds(xov,detabcr(got[i]),strlen(got[i]));
			}
		}
		
		
        str_adds(xov,EOL,strlen(EOL));
        return xov;
		
	}
	char *get_ahead(String *sitem, char *head)
	{
        char tmp[50],tmp2[BFSZ];
        char *s = str_s(sitem);
        int stripit=FALSE;
		
        if (strstr(head,"Message-ID")!=NULL) stripit=TRUE;
        if (strchr(head,':')==NULL) {
			ncpy(tmp2,head,BFSZ-2); head = tmp2;
			strcat(head,":");
        }
        s = str_s(sitem); 
        if (s!=NULL) for (;s!=NULL && *s!=0;) {
			strncpy(tmp,s,40); tmp[40] = 0; 
			if (strncmp(s,EOL,strlen(EOL))==0) break;
			if (strncmpnc(s,head,strlen(head))==0) {
				if (stripit) return lib_trim_msg(get_head(s));
				else return get_head(s);
			}
			for (s=s+1;*s!=0;s++) if (*s=='\n') break;
			if (*s==0) break;
			s += 1;
        }
        return "";
	}
	int is_head_exact(String *sitem, char *head)
	{
        char tmp[50];
        char *s = str_s(sitem);
      	
        s = str_s(sitem); 
        if (s!=NULL) for (;s!=NULL && *s!=0;) {
			strncpy(tmp,s,40); tmp[40] = 0; 
			if (strncmp(s,EOL,strlen(EOL))==0) break;
			if (strncmpnc(s,head,strlen(head))==0) {
				return TRUE;
			}
			for (s=s+1;*s!=0;s++) if (*s=='\n') break;
			if (*s==0) break;
			s += 1;
        }
        return FALSE;
	}
	char *get_aheadraw(String *sitem, char *head)
	{
        char tmp[50],tmp2[BFSZ];
        char *s = str_s(sitem);
        int stripit=FALSE;
		
        if (strstr(head,"Message-ID")!=NULL) stripit=TRUE;
        if (strchr(head,':')==NULL) {
			ncpy(tmp2,head,BFSZ-2); head = tmp2;
			strcat(head,":");
        }
        s = str_s(sitem);
		if (*s=='~') {
			s = strchr(s,':');
			if (s!=NULL) s = strchr(s+1,':');
			if (s!=NULL) s++;
			if (s!=NULL) if (isdigit(*s)) {
				s = strchr(s+1,':');
				if (s!=NULL) s++;
			}
		}
		if (s==NULL) {
			imsg("item: getaheadraw failed to find colons\n");
			return "";
		}
        if (s!=NULL) for (;s!=NULL && *s!=0;) {
			strncpy(tmp,s,40); tmp[40] = 0; 
			if (strncmp(s,EOL,strlen(EOL))==0) break;
			if (strncmpnc(s,head,strlen(head))==0) {
				if (stripit) return lib_trim_msg(get_head(s));
				else return get_head(s);
			}
			s = strstr(s+1,EOL);
			if (s!=NULL) s+=strlen(EOL);            
			if (s==NULL) break;
        }
        return "";
	}
	char *get_head(char *s)
	{
        static char tmp[11000];    /* No header should excced this !!! */
        char *o = tmp;
		int skip_space=TRUE;
		if (strncmp(s,"Subject:",8)==0) skip_space = FALSE;
		if (strncmp(s,"Subject",7)==0) skip_space = FALSE;
		
        s = strchr(s,':');
        if (s==NULL) return "FSRV-ERROR";
        s++;
        if (*s==0) return "";
        for (;(*s==' ' || *s=='\t') && (*s!=0);s++) ;  /*Skip over leading spaces*/
        for (; ((o-tmp)<10000) && (*s!=0);) {
			if (*s=='\015') {
				if ( s[2]==' ' || s[2]=='\t' ) {
					s+=2;  /* Eat eol */
					if (skip_space) {
						s++;
						for (;*s!=0 && isspace(*s) ;) s++ ;
					}
					if (*s=='<') *o++ = ' ';
				} else break; 
			}
			if (*s==0) break;
			if (*s=='\t') {*o++ = ' '; s++;}  /* Convert tabs to spaces */
			else *o++ = *s++; 
        }       
        *o++ = 0;
        return tmp;
	}
	
	
#define MAX_CACHE 11000
	static int max_cache=200,bigest_cache=3000;
	typedef struct {int grp; int num; String *str; int len; char *mid; int artlen; } Cache;
	static Cache cache[MAX_CACHE];
	int item_clearcache();
	int item_clearcache()
	{
        int i;
        for (i=0; i<max_cache; i++) {
			if (cache[i].str!=NULL) {
				mem_log(M_HEAD,-str_len(cache[i].str));
				str_free(cache[i].str);
				cache[i].str = NULL;
			}
			cache[i].grp = -2;
        }
        i = atoi(config_get(C_HEAD_CACHE));
        if (i>0) max_cache = i;
        if (max_cache>=MAX_CACHE) max_cache = MAX_CACHE-2;
        i = atoi(config_get(C_HEAD_LIMIT));
        if (i>0) bigest_cache = i;
		
		
        return TRUE;
	}
	int item_hash(int grp, int num)
	{
        return (grp+num+500) % max_cache;
	} 
#define MID_LEN 80
	void item_addcache(String *str,int grp, int num, int artlen)
	{
        int h;
        String *tmp;
		
        h = item_hash(grp,num);
		/*      dmsg("item: head_cache[%d] Add %d:%d, size %d max=%d  \n",h,grp,num,str_len(str),bigest_cache);
        dmsg("item: cache subject %d:%d {%.200s} %p %p\n",grp,num,get_ahead(str,"Subject:"),str,str_s(str));
		*/
        if (str_len(str)>bigest_cache) return; /* Don't cache whoppers */
		
        if (cache[h].str==NULL) tmp = str_new(); else tmp=cache[h].str;
		if (cache[h].str!=NULL) mem_log(M_HEAD,-str_len(cache[h].str));
		
        str_copy(tmp,str);
        cache[h].str = tmp;
        cache[h].grp = grp;
        cache[h].num = num;
        cache[h].artlen = artlen;
        if (cache[h].mid==NULL) cache[h].mid = mymalloc(MID_LEN);
        ncpy(cache[h].mid,get_ahead(str,"Message-ID:"),MID_LEN-1);
        cache[h].len = str_len(str); /* To confirm that it is the right one */
		
		mem_log(M_HEAD,str_len(str));
		
	}
	String *item_getcache(String *str, int grp, int num)
	{
        char mid[MID_LEN];
        String *ss;
		static String *xreturn;
        int h=item_hash(grp,num);
        if ((cache[h].grp==grp) && (cache[h].num==num) && (num!=0)) {
			ncpy(mid,get_ahead(cache[h].str,"Message-ID:"),MID_LEN-1);
			ss = cache[h].str;
			if (cache[h].len != str_len(cache[h].str)) {
				emsg("item: cache len %d, %d\n",cache[h].len,str_len(cache[h].str)) ;
				emsg("item: cache[%d] read %d:%d subject {%.200s} %p %p\n",h,grp,num,get_ahead(ss,"Subject:"),ss,str_s(ss));
				emsg("item: CACHE FAILED, RETURNING NULL, BUG IN STR ROUTINES\n");
				emsg("item: Check mid's {%.200s} {%.200s} \n",cache[h].mid,mid);
				str_write(ss,file_work("string.bad"));
				return NULL;
			}
			if (strcmp(cache[h].mid,mid)!=0) {
				emsg("item: CACHE FAILED, MID didn't match {%.200s} {%.200s} \n",cache[h].mid,mid);
				return NULL;
			}
			ss = cache[h].str;
			last_len = cache[h].artlen;
			/* dmsg("item: cache[%d] read %d:%d subject {%.200s} %p %p\n",h,grp,num,get_ahead(ss,"Subject:"),ss,str_s(ss));*/
			if (xreturn==NULL) xreturn = str_new();
			str_copy(xreturn,cache[h].str);
			return xreturn;
        } else return NULL;
	}
	void item_freecache(int grp, int num)
	{
        int h=item_hash(grp,num);
        if ((cache[h].grp==grp) && (cache[h].num==num) && (num!=0)) {
			if (cache[h].str!=NULL) {
				mem_log(M_HEAD,-str_len(cache[h].str));
				str_free(cache[h].str);
				cache[h].str = NULL;
				cache[h].grp = -2;
			}
        } else return ;
	}
	void xover_clear_cache(void)
	{
		int h;
		for (h=0; h<MAX_CACHE; h++) {
			if (cache[h].str!=NULL) {
				mem_log(M_HEAD,-str_len(cache[h].str));
				str_free(cache[h].str);
				cache[h].str = NULL;
				cache[h].grp = -2;
			}
		}
	}
	int head_replace(String *item,String *newhead)
	{
        char *s;
        int hlen,nlen,diff,ilen;
        char eoh[10]; 
		
        strcpy(eoh,EOL);
        strcat(eoh,EOL);
		
        s = strstr(str_s(item),eoh);
        if (s==NULL) {
			emsg("item: No body for item, junking it\n"); return FALSE;
        }
		
        hlen = s-str_s(item);
        if (hlen<0) {emsg("FATAL: PANIC, head replace strstr failed \n"); return FALSE;}
        if (hlen > 30000) emsg("HUGE: Huge total header found, warning %d\n",hlen); 
        s = strstr(str_s(newhead),eoh);
        if (s==NULL) {emsg("item: no new body for item, junking it\n"); return FALSE;}
		
        nlen =  s-str_s(newhead);
        diff = nlen-hlen;
        ilen = str_len(item);
		
        dmsg("item: nlen hlen ilen diff %d %d %d %d \n",nlen,hlen,ilen,diff);
		
		if ((ilen+diff)<0) {emsg("item: head_replace failed %d %d\n",ilen,diff); return FALSE;}
		if ((ilen-hlen)<0) {emsg("item: head_replace failed %d %d\n",ilen,diff); return FALSE;}
		if ((nlen)<0) {emsg("item: head_replace failed %d %d\n",ilen,diff); return FALSE;}
        str_setsize(item,ilen+diff+1);
        memmove(str_s(item)+nlen,str_s(item)+hlen,ilen-hlen);
        str_setlen(item,ilen+diff);
        memmove(str_s(item),str_s(newhead),nlen);
        *(str_s(item)+ilen+diff) = 0;
        return TRUE;
	}
	int body_lines(String *item);
	String *item_fixhead(String *item, String *xref, String *xpath)
	{
        static String *head;
        char *e,*s = str_s(item);
        char h1[30],h2[30],h3[30];
        int skip;
		
        str_lock(item);
        if (head==NULL) head= str_new();
        str_clearbig(head);
        strcpy(h1,"xref:");
        strcpy(h2,"path:");
        strcpy(h3,"Relay-Version");  /* Always strip off.*/
		
        skip = FALSE;
        for (;(s!=NULL) && (*s!=0);) {
			if (strncmp(s,EOL,strlen(EOL))==0) break;
			if (strncmpnc(s,"Message-ID:",11)==0) {
				if (strncmp(s,"Message-ID:",11)!=0) {
					strncpy(s,"Message-ID:",11);
				}
			}                
			if (strncmpnc(s,h1,5)==0) {
				skip = TRUE; goto next_line;
			} else if (strncmpnc(s,h2,5)==0) {
				skip = TRUE; goto next_line;
			} else if (strncmpnc(s,h3,strlen(h3))==0) {
				skip = TRUE; goto next_line;
			}
			e = strstr(s+1,EOL); if (e==NULL) break;
			if (e<(s+1)) break;
			str_adds(head,s,e-s+2);
next_line:
			s = strstr(s+1,EOL);
			if (s!=NULL) {
				s+=strlen(EOL);            
				if (skip) {
					if ((*s==' ') || (*s=='\t')) goto next_line;
				}
			}
			skip = FALSE;
        }
        str_unlock(item);
        str_add(head,xpath);                    
        if (str_len(xref)>0) str_add(head,xref);                        
		
		if (config_true(C_LINES_ADD)) {
			if (strlen(get_ahead(item,"Lines:"))==0) {
				lprintf(bf,MAX_XOVER+10-1,"Lines %d\r\n",body_lines(item));
				str_addsn(head,bf);
			}
		}
		
        str_adds(head,EOL,2);
        return head;
	}
	static int total_megs,total_bytes,last_wrote;
#define DNEWS_DISK_USE file_work("diskuse.dat")
	int diskuse_get_x(void );
	static int du_done;
	int diskuse_get(void )
	{
		int r;
		static int last;
		if (is_readonly()) {
			if (lib_gmtime()>(last+100)) {
				last = lib_gmtime();
				du_done = FALSE;
			}
		}
		r = diskuse_get_x();
		return r;
	}
	int diskuse_get_x(void )
	{
        FILE *f;
        char bf[101];
        char *s;
        
        if (du_done) return total_megs * 100 + total_bytes/10000;
        du_done = TRUE;
        f = fopen_log(DNEWS_DISK_USE,"r");
        if (f!=NULL) {
			fgets(bf,100,f);
			strlwr(bf);
			if (strncmp(bf,"megs",4)==0) {
				total_megs = atoi(bf+5);
				s = strstr(bf,"bytes");
				if (s!=0) total_bytes = atoi(s+6);
			} else {
				total_megs = atoi(bf) / 10000;
				total_bytes = atoi(bf) % 10000;
			}
			last_wrote = total_megs;
			xfclose(f); f = NULL;
        }
        return total_megs * 100 + total_bytes/10000;
	}
	void diskuse_add(int x)
	{
		static int xnow,xlast;
        total_bytes += x;
        for (;total_bytes > 1000000;) {
			total_megs++;
			total_bytes -= 1000000;
        }
        for (;total_bytes < 0;) {
			total_megs--;
			total_bytes += 1000000;
        }
        if (total_megs<0) {
			emsg("diskuse: resetting megs to zero %d\n",total_megs);
			total_megs = 0; total_bytes = 0;
        }
		xnow = lib_gmtime();
		if (xnow==xlast) return;
		xlast = xnow;
        diskuse_write();
	}
	void diskuse_set(int x)
	{
        total_megs = x / 10000;
        total_bytes = (x % 10000) * 100;
        diskuse_write();
	}
	void diskuse_flush(void)
	{
        diskuse_get(); /* Make sure we've read it in first */
        last_wrote = -1;
        diskuse_write();
	}
	void diskuse_write(void)
	{
        FILE *f;
		if (is_readonly()) return;        
        if (total_megs!=last_wrote) { /* Only log 1/megabyte */
			remove(DNEWS_DISK_USE);
			f = fopen_log(DNEWS_DISK_USE,"w");
			if (f!=NULL) {
				fprintf(f,"megs %d bytes %d\n",total_megs,total_bytes);
				last_wrote = total_megs;
				xfclose(f); f = NULL;
			}
        }  
	}
	
	int kill_max(void)
	{
		static int d;
		if (d!=0) return d;
		d = atoi(config_get(C_EXPIRE_CHUNK));
		
		if (d!=0) return d;
		d = db_total_space();
		if (d<1000) d = d * 200;
		else d = d * 30;
		if (d<50000) d = 50000;
		if (d>500000) d = 500000; 
		return d;
	}
	void dalloc(void **x, int n)
	{
        if (*x != NULL) return;
        *x = mymalloc(n);
		
        if (*x == NULL) {
			emsg("fatal: dalloc failed, cannot continue\n");
			exit(3);
        }
	}
	int item_cancel(int c,int g, int num) {
		char bf[BFSZ];
		String *headers;
		String *cancelmsg;
		char h_msgid[BFSZ];
		char h_from[BFSZ];
		char h_sender[BFSZ];
		char h_subject[BFSZ];
		char h_newsgroups[BFSZ];
		int modgroup = FALSE;
		int ret = FALSE;
		
		total_cancel++;
		modgroup = group_ismod(g);
		headers = item_head(g,num);
		if (headers==NULL) return FALSE;
		ncpy(h_msgid,get_ahead(headers,"Message-id"),BFSZ-100);
		ncpy(h_from,get_ahead(headers,"From"),BFSZ-100);
		ncpy(h_sender,get_ahead(headers,"Sender"),BFSZ-100);
		ncpy(h_subject,get_ahead(headers,"Subject"),BFSZ-100);
		ncpy(h_newsgroups,get_ahead(headers,"Newsgroups"),BFSZ-100);
		
		lprintf(bf,BFSZ-1,"\t\tFrom: %.200s\r\n",h_from);
		chan_write_s(c,bf);
		/*sprintf(bf,"Sender: %.200s\r\n",h_sender);
		chan_write_s(c,bf);*/
		lprintf(bf,BFSZ-1,"\t\tSubject: %.200s\r\n",h_subject);
		chan_write_s(c,bf);
		lprintf(bf,BFSZ-1,"\t\tNewsgroups: %.200s\r\n",h_newsgroups);
		chan_write_s(c,bf);
		lprintf(bf,BFSZ-1,"\t\tMessage-ID: %.200s\r\n",h_msgid);
		chan_write_s(c,bf);
		/*if (modgroup) chan_write_s(c,"Moderated newsgroup\r\n");
		else chan_write_s(c,"Unmoderated newsgroup\r\n");*/
		cancelmsg = str_new();
		lprintf(bf,BFSZ-1,"From: %.200s\r\n",h_from);
		str_addsn(cancelmsg,bf);
		if (strcmp(h_sender,"")!=0) {
			lprintf(bf,BFSZ-1,"Sender: %.200s\r\n",h_sender);
			str_addsn(cancelmsg,bf);
		}
		lprintf(bf,BFSZ-1,"Subject: Re: %.200s\r\n",h_subject);
		str_addsn(cancelmsg,bf);
		lprintf(bf,BFSZ-1,"Newsgroups: %.200s\r\n",h_newsgroups);
		str_addsn(cancelmsg,bf);
		lprintf(bf,BFSZ-1,"Control: cancel %.200s\r\n",h_msgid);
		str_addsn(cancelmsg,bf);
		if (modgroup) {
			lprintf(bf,BFSZ-1,"Approved: %.200s\r\n",h_from);
			str_addsn(cancelmsg,bf);
		}
		str_addsn(cancelmsg,"\r\n");
		lprintf(bf,BFSZ-1,"Cancelling message %.200s\r\n",h_msgid);
		str_addsn(cancelmsg,bf);
		ret=post_article(c,cancelmsg,"");
		str_free(cancelmsg);
		if (ret) lprintf(bf,BFSZ-1,"\tArticle %.200s cancelled\r\n",h_msgid);
		else lprintf(bf,BFSZ-1,"Unable to cancel article %.200s\r\n",h_msgid);
		chan_write_s(c,bf);
		
		return ret;
	}
	
	int item_delete_range(int chan, int g, int from, int to)
	{
		int ret = FALSE;
		int i;
		int start,end;
		/* validate the range */
		start = group_getstart(g);
		end = group_getend(g);
		if (from>to) return FALSE;
		if (from<start) {
			imsg("item_delete_range : trimmed from %d->%d\n",from,start);
			from=start;
		}  
		if (to>end) {
			imsg("item_delete_range : trimmed to %d->%d\n",to,end);
			to=end;
		}
		/* delete the items */
		for(i=from;i<=to;i++)
			/*ret |= item_delete_all(chan,g,i);*/
			ret |= item_delete(g,i);
		return ret;
	}
	
	static int nkill;
	int kill_total(void)
	{
		return nkill;
	}
	int item_delete_all(int chan, int g, int num)
	{
		char xref[10000];
		String *xhead;
		char bf[BFSZ];
		char *c;
		char *s;
		int xg,xn;
		
		nkill++;	
		imsg("delete_all %d %.200s:%d\n",g,group_name(g),num);
		xhead = item_head(g,num);
		if (xhead==NULL) return FALSE;
		ncpy(xref, get_ahead(xhead,"Xref:"),9000);
		s = strtok(xref," ");
		for (;s!=NULL;s=strtok(NULL," ")) {
			c = strchr(s,':');
			if (c==NULL) continue;
			*c = 0;
			xg = group_find(s);
			if (xg<0) continue;
			xn = atoi(c+1);
			lprintf(bf,BFSZ-1,"Kill cross post %.200s:%d\r\n",s,xn);
			if (chan>=0) chan_write_s(chan,bf);
			else 	imsg("Kill cross post %.200s:%d\r\n",s,xn);
			
			item_delete(xg,xn);
		}
		item_delete(g,num);
		return TRUE;
	}
	static int nitemdel;
	int item_ndel(void)
	{
		return nitemdel;
	}
	int item_delete(int g, int num)
	{
		if (inpurge) {
			imsg("Sorry, cannot process cancel during expire_bits %d %d \n",g,num);
			return TRUE;
		}
		nitemdel++;
		xcache_forget(g,num);
		if (config_true(C_HANGING_FAST)) {
			db_cancel_fast(g,num,"item_delete",FALSE);
		} else {
			db_cancel(g,num); 
		}
        /*
        item_kill(g,num);
        for (;item_purge()>=0;) ;
        */
        item_clearcache();
        return TRUE;
	}
	void item_init(void)
	{
		init_xover();
        item_clearcache();
	}
	void item_loading(String *h, int grp)
	{
        char bf[500];
		
        lprintf(bf,500-1,"From: %.200s\015\012",config_get(C_FROM)); str_copysn(h,bf);
		if (strlen(config_get(C_NOCACHED_SUBJECT))>0) {
			lprintf(bf,500-1,"Subject: %.200s\015\012",config_get(C_NOCACHED_SUBJECT)); str_addsn(h,bf);
		} else {
			str_addsn(h,"Subject: Group download in progress...\015\012");
		}
        lprintf(bf,500-1,"Date: %.200s\015\012",lib_strdate2(-60*60*24*60)); str_addsn(h,bf);
        lprintf(bf,500-1,"Path: newsfeed.internetmci.com!decwrl!news.demon.co.uk!enews.sgi.com!uunet!in1.uu.net!waikato!dispatch.news.demon.net!demon!sunsite.dco.ic.ac.uk\015\012"); str_addsn(h,bf);
        lprintf(bf,500-1,"Expires: %.200s\015\012",lib_strdate()); str_addsn(h,bf);
		/*        lprintf(bf,500-1,"Message-ID: <fake@netwin.com.nz>\015\012",lib_messageid());*/
        lprintf(bf,500-1,"Message-ID: <fake@netwin.com.nz>\015\012"); str_addsn(h,bf);
        lprintf(bf,500-1,"Distribution: local\015\012"); str_addsn(h,bf);
        lprintf(bf,500-1,"Followup-To: local.news\015\012"); str_addsn(h,bf);
        if (grp>=0) {lprintf(bf,200,"Newsgroups: %.200s\015\012",group_name(grp)); str_addsn(h,bf);}
	}
	void fiddle_order(char *grps)
	{
        char *n;
        char *s;
        s = strstr(grps,",news.answers");
        if (s==NULL) return;
        n = mymalloc(strlen(grps)+25); if (n==NULL) {emsg("item: mem fail\n"); return;}
        strcpy(n,"news.answers,");
        ncpy(n+strlen(n),grps,s-grps);
        strcat(n,s+13); 
        strcpy(grps,n);
        myfree(n);
        return;
	}
	void fiddle_first(char *grps,char *wanted)
	{
		char bf[BFHEAD];
		char bf2[BFHEAD];
		int len;
		char *s,*out;
		len = strlen(grps);
		out = bf2;
        
		ncpy(bf,grps,BFHEAD-100);
		strcpy(bf2,wanted); out+= strlen(bf2);
		s = strtok(bf," ,\r\n\t");
		for (;s!=NULL;) {
			if (strcmpnc(s,wanted)!=0) {
				lprintf(out,BFSZ-1,",%.200s",s);
				out += strlen(s);
			}
			s = strtok(NULL," ,\r\n\t");
		}
		ncpy(grps,bf2,len);
	}
	void item_purgelost(int g, int range, int *ngood, int *nbad)
	{
	/*        String *shead,*body,*tmp;
	int fg,fi,i,ffi,ffg,islink,islink2;
	int start,newstart=0,end,newend = -1;
	char *msgid;
	char *h;
	char *gnam;
	int gnum;
	
	  shead = str_new();
	  
        start = group_getstart(g);
        end = group_getend(g);
        cvt_numfile(range,&fg, &fi);
        inexpire = TRUE;
        item_purgestart(g,range);
        for (i=fg*100;i<fg*100+100;i++) {
		cvt_numfile(i,&ffg, &ffi);
		tmp = db_get_raw(g,i,TRUE,&islink);
		if (tmp==NULL) goto junk_null;
		str_copy(shead,tmp);
		body = db_get_raw(g,i,FALSE,&islink2);
		
		  if ((tmp==NULL) || (body==NULL)) goto junk_lost;
		  if (islink) goto write_it; 
		  h = NULL;
		  msgid = get_ahead(shead,"Message-ID:");
		  if (msgid!=NULL) {
		  if (strlen(msgid)==0) goto junk_lost;
		  h = hist_exists(msgid);
		  if (h==NULL) {
		  h = item_addlost(shead,msgid,g,i);
		  if (h!=NULL) goto write_it;
		  }
		  }
		  if (h==NULL) goto junk_lost;
		  gnam = hist_groupfirst(h);
		  for (;gnam!=NULL;) {
		  gnum = hist_groupnextnum();
		  if (strcmp(gnam,group_name(g))==0) {
		  if (gnum!=i) goto junk_duplicate;
		  else goto write_it;
		  }
		  gnam = hist_groupnext();
		  }
		  goto junk_duplicate;
		  write_it:
		  if (newstart==0) newstart = i;
		  newend = i;
		  item_purgewrite(g,i,shead,body,islink); (*ngood)++;
		  goto end_loop;
		  junk_duplicate: wmsg("item: Removing duplicate message {%.200s} \n",msgid); (*nbad)++;
		  goto end_loop;
		  junk_lost:      wmsg("item: Removing lost item {%.200s:%d}\n",group_name(g),i); (*nbad)++;
		  junk_null:
		  end_loop:       ;
		  }
		  item_purgeclose(g,range);
		  inexpire = FALSE;       
		  if (newstart!=0) if (newstart<start) {
		  group_setstart(g,newstart);
		  group_setrange(g,-1);
		  } 
		  if (newend>=0) if (newend>end) {
		  group_setend(g,newend);
		  group_setrange(g,-1);
		  }
		  str_free(shead);
		*/
	}
	
	char *file_store(int x)
	{
		static char bf[300];
		lprintf(bf,300-1,"%.200sstore_%d.itm",config_getu(C_SPOOL),x);
		return bf;
	}
	
	void debug_store_item(String *xitem)
	{
        static int fn;
        fn++;
        if (!fileonly) {        
			if (fn>=atoi(config_get(C_DEBUG_STORE))) fn = 1;
        }
		
        dmsg("debug: store item %d \n",fn);
        if (!str_write(xitem,file_store(fn))) {
			emsg("store: Unable to write debug article file %.200s \n",file_store(fn));
			return;
        }
	}
	
	
	
	int item_purgestart(int group, int num)
	{
		/*        return db_open_new(group,num);*/
		return TRUE;
	}
	int item_purgewrite(int group, int num, String *shead, String *sbody,int islink) /* Saves item */
	{
	/*        int r;
	str_adds(shead,"\015\012",2); 
	str_add(shead,sbody);
	str_clearbig(sbody);
	r = db_add(group,num,shead,islink);     
	str_clearbig(shead);
		*/  return TRUE;
	}
	int item_purgeclose(int group, int num)
	{
        return TRUE; /*db_close_new(group,num);*/
	}
	
	static int purgedel[200];
	void item_purgedel(int g, int num)
	{
        int fg,fi;
        cvt_numfile(num,&fg, &fi);
        purgedel[fi] = TRUE;
	}
	void item_purgedo(int g, int num)
	{
	}
	int item_purgedo2(int g, int num, int upto)
	{
		return TRUE;
	}
	
	static int xcache_inited;
	static int xcache_size=0;
	static int *xcache_gid,*xcache_num;
	static String **xcache_str;
	int xcache_init(void)
	{
		int i;
		for (i=0; i<xcache_size; i++) {
			if (xcache_str[i] != NULL) str_free(xcache_str[i]);
		}
		xcache_size = atoi(config_get(C_XOVER_CACHE));
		if (xcache_size<=0) xcache_size = 3500;
		xcache_gid = mymalloc(xcache_size * sizeof(int));
		xcache_num = mymalloc(xcache_size * sizeof(int));
		xcache_str = mymalloc(xcache_size * sizeof(String *));
		for (i=0; i<xcache_size; i++) {
			xcache_gid[i] = -1;
			xcache_str[i] = NULL;
		}
		xcache_inited = TRUE;
		return TRUE;
	}
	int xcache_hash(int g, int n)
	{
		int r;
		if (xcache_size==0) xcache_init();
		r = ((((g % 20003) * 2000) % 30001) + (n % 20000)) % xcache_size;
		if (r<0) return -r;
		return r;
	}
	
	String *xcache_get(int g, int n)
	{
		int h = xcache_hash(g,n);
		if (!xcache_inited) xcache_init();
		if ((xcache_gid[h]==g) && (xcache_num[h]==n)) {
			return xcache_str[h];
		}
		return NULL;
	}
	void xcache_forget(int g, int n)
	{
		int h = xcache_hash(g,n);
		
		if (!xcache_inited) xcache_init();
		if ((xcache_gid[h]==g) && (xcache_num[h]==n)) {
			xcache_gid[h] = -1;
			if (xcache_str[h] != NULL) {
				mem_log(M_XCACHE,-str_len(xcache_str[h]));
				str_free(xcache_str[h]);
			}
			xcache_str[h] = NULL;
		}
	}
	void xcache_store_old(int g, int n, String *x)
	{
		int h = xcache_hash(g,n);
		if (!xcache_inited) xcache_init();
		if (x==NULL) return;
		if (str_len(x)>400) return; /* Limit size of items */
		xcache_gid[h] = -1;
		if (xcache_str[h] != NULL) {
			mem_log(M_XCACHE,-str_len(xcache_str[h]));
			str_free(xcache_str[h]);
		}
		xcache_str[h] = NULL;
		xcache_gid[h] = g;
		xcache_num[h] = n;
		xcache_str[h] = str_new();
		str_copy(xcache_str[h],x);
		mem_log(M_XCACHE,str_len(xcache_str[h]));
	}
	void xcache_store(int g, int n, String *x)
	{
		int h = xcache_hash(g,n);
		if (!xcache_inited) xcache_init();
		if (x==NULL) return;
		
		if (config_true(C_XOVER_CACHE_DISABLE)) return;
		if (str_len(x)>400) return; /* Limit size of items cached */
		xcache_gid[h] = -1;
		if (xcache_str[h] == NULL) {
			xcache_str[h] = str_new();
		} else {
			mem_log(M_XCACHE,-str_len(xcache_str[h]));
		}
		xcache_gid[h] = g;
		xcache_num[h] = n;
		str_copy(xcache_str[h],x);
		mem_log(M_XCACHE,str_len(xcache_str[h]));
	}
	
	typedef struct SLIST {String *str; int grp, num; struct SLIST *next; } Slist;
	
	static Slist **wlist;
	static int nwcache;
	
	void item_xover_write_cached(int grp, int num, String *xitem, int force)
	{
		if (maxcache==0) {
			item_xover_write(grp,num,xitem,force);
			return;
		}
		wcache_add(grp,num,xitem);
		if (nwcache>maxcache) wcache_shrink();
	}
	void wcache_add(int grp, int num, String *str)
	{
		Slist *m;
		if (grp<0) return;
		if (grp>=max_groups) return;
		m = mymalloc(sizeof(Slist));
		m->grp = grp;
		m->num = num;
		m->str = str_new();
		mem_log(M_XWRITE,str_len(str));
		str_copy(m->str,str);
		m->next = wlist[grp];
		wlist[grp] = m;
		nwcache++;
	}
	void wcache_init(void)
	{
		maxcache = atoi(config_get(C_XOVER_WRITE_CACHE));
		if (maxcache==0) maxcache = 1000;
		if (maxcache>0) {
			wlist = mymalloc(max_groups * sizeof(Slist *));
			memset(wlist,0,max_groups * sizeof(Slist *));
		}
	}
	void wcache_shrink(void)
	{
		static int upto;
		int i;
		if (maxcache==0) return;
		if (nwcache==0) return;
		for (i=upto; i<max_groups; i++) {
			if (wlist[i]!=NULL) goto write_one;		
		}
		for (i=0; i<=upto; i++) {
			if (wlist[i]!=NULL) goto write_one;		
		}
write_one:
		upto = i+1;
		if (wlist[i]==NULL) return;
		wcache_write_one(i,-1);
	}
	void wcache_write_all(void)
	{
		int i;
		if (maxcache==0) return;
		for (i=0; i<max_groups; i++) {
			if (wlist[i]!=NULL) wcache_write_one(i,-1);
		}
	}
	void wcache_write_one(int x, int last)
	{
		Slist *m,*mnext,*start=NULL;
		int n=0;
		if (maxcache==0) return;
		
		if (x>=max_groups) {
			emsg("wcache_write_one, invalid group %d\n",x);
			return;
		}
		if (x<0) return ;
		if (wlist[x]==NULL) return; 
		for (m=wlist[x]; m!=NULL; m=mnext) {
			mnext = m->next;
			m->next = start;
			start = m;
		}
		
		
		for (m=start; m!=NULL; m=mnext) {
			mnext = m->next;
			if (m->num>last) item_xover_write(m->grp,m->num,m->str,TRUE);
			mem_log(M_XWRITE,-str_len(m->str));
			str_free(m->str);
			myfree(m);
			nwcache--;
			n++;
		}
		dmsg("item: Wrote xover cache for group %.200s, %d items\n",group_name(x),n);
		wlist[x] = NULL;
	}
	int wcache_find_first(int x)
	{
		Slist *m,*start=NULL;
		int f= -1;
		if (maxcache==0) return -1;	
		if (x<0) return -1;
		if (x>=max_groups) {
			emsg("2wcache_write_one, invalid group %d\n",x);
			return -1;
		}
		
		for (m=wlist[x]; m!=NULL; m=m->next) {
			if (f>m->num) f = m->num;
			if (f<0) f = m->num;
		}
		return f;
	}
	/* Xover background rebuilding - may happen on slaves or master */
	static char *xidle_mask;
	static int nxidle;
#define FILE_XSORT file_work("xover_sort.do")
	void xidle_sort(int grp)
	{
		FILE *f;
		if (xidle_mask==NULL) return;
		if (grp<0) return;
		if (grp>=max_groups) return;
		if (xidle_mask[grp]) return;
		if (group_getxsort(grp)) {
			imsg("xidle: Not sorting group again %.200s\n",group_name(grp));
			return;
		}
		group_setxsort(grp,TRUE); /* Remember not to do this one again*/
		if (config_true(C_XOVER_NOIDLE)) return;
		xidle_mask[grp] = TRUE;
		if (slave_n()>0) {
			slave_tellmaster_buildxover(grp);
			return; /* Only do this on slave1 or master */
		}
		if (is_readonly()) { /* Readonly master if this is true and wasn't slave */
			f = fopen(FILE_XSORT,"a");
			if (f!=NULL) {	
				imsg("Writing to %s %s\n",FILE_XSORT,group_name(grp));
				fprintf(f,"%s\n",group_name(grp));
				fclose(f);
			} else imsg("Could not write to file %s\n",FILE_XSORT);
			return;
		}
		imsg("xidle: Scheduling sort of (%s)\n",group_name(grp));
		nxidle++;
	}
	int xidle_sorted_new(int grp);
	static int xcur_grp,xbusy,xupto;
	void xidle_do_file(void)
	{
		FILE *f;
		char *s;
		int g;
		char bf[BFSZ];
		if (is_readonly()) return;
		f = fopen(FILE_XSORT,"r");
		if (f==NULL) return;
		for (;!feof(f);) {
			if (fgets(bf,BFSZ-1,f)==NULL) break;
			s = strchr(bf,'\n'); if (s!=NULL) *s = 0;
			g = group_find(bf);
			if (g<0) continue;
			imsg("Readonly slave asked us to sort xover file for (%s)\n",group_name(g));
			xidle_sort(g);
		}
		fclose(f);
		remove(FILE_XSORT);
	}
	void xidle_do(void)
	{
		int i;
		
		if (xidle_mask==NULL) {
			xidle_mask = mymalloc(max_groups);
			memset(xidle_mask,0,max_groups);
		}
		if (is_readonly()) return;
		if (config_true(C_XOVER_XHASH)) return;
		
		if (!xbusy) {
			if (nxidle==0) return;
			for (i=0; i<max_groups; i++) if (xidle_mask[i]) {
				imsg("xidle: Sorting (%s)\n",group_name(i));
				xcur_grp = i;
				xupto = 0;
				xbusy = TRUE;
				xidle_mask[i] = FALSE;
				xover2_ngroups++;
				nxidle--;
				return;
			}
			nxidle = 0;
			return;
		}
		
		if (xidle_sorted_new(xcur_grp)) {
			xbusy = FALSE;
		}
	}
	static char xidle_xstatus[BFSZ];
	int xidle_sorted_old(int grp, int *upto)
	{
		char bf[BFSZ];
		int start = group_getstart(grp);
		int end = group_getend(grp);
		int max = group_getrange(grp)+1000;
		
		if (*upto==0) {
			*upto = start;
			wmsg("xidle: Sorting groups xover file into order %.200s %d items %.200s\n",group_name(grp),max-1000,file_xover(grp));
			remove(file_xov_idle());
		}
		
		item_xover_close();
		stat_start(X_XSORT);
		if (xidle_sort_block(grp,upto)) {
			stat_end(X_XSORT);
			if (remove(file_xover(grp))!=0) fimsg("Unable to remove (%.200s) \n",file_xover(grp));
			strcpy(bf,file_xov_idle());
			if (rename(bf,file_xover(grp))!=0) fimsg("Unable to rename to (%.200s) \n",file_xover(grp));
			wmsg("xidle: Sort completed\n");
			return TRUE;
		} else {
			stat_end(X_XSORT);
			return FALSE;
		}
	}
	int xidle_sort_block(int grp, int *block)
	{
		int nums[5000];
		String *vals[5000];
		int ngot=0;
		int i;
		int len;
		int nread=0;
		char *s;
		int finished=FALSE;
		FILE *f;
		int last;
		int upto=0;
		int n;
		String *hdr,*x;
		int start;
		char bf[MAX_XOVER];
		start = lib_gmtime();
		n = g_list_start(grp);
		for (;(n>0) && (n<*block);) n = g_list_next();	
		if (n<=0) {*block = -1; return TRUE; }
		for (;n>0;) {
			nums[ngot++] = n;
			last = n;
			if (ngot>4900) break;
			n = g_list_next();	
		}
		if (n<=0) finished = TRUE;
		imsg("Sorting block of %d items, from %d\n",ngot,*block);
		
		for (i=0; i<ngot; i++) vals[i] = str_new();		
		
		/* Scan existing xover file and store all valid ones in nums array */
		
		stat_start(X_XOPEN);
		f = fopen(file_xover(grp),F_READ);
		stat_end(X_XOPEN);
		if (f==NULL) goto gaps;
		for (;!feof(f);) {	
			if (fgets(bf,MAX_XOVER-3,f)==NULL) break;
			len = strlen(bf);
			if (bf[len-2]=='\r') {
				bf[len-2] = '\n';
				bf[len-1] = 0;
				len--;
			}
			if (bf[len-1] != '\n') continue;
			n = atoi(bf); 
			if (n==0) continue;
			for (;upto<ngot;upto++) if (nums[upto]==n) goto found;
			for (upto=0;upto<ngot;upto++) if (nums[upto]==n) goto found;
			continue;
found:		str_copysn(vals[upto],bf);
		}
		fclosep(f);
		
gaps:
		start = lib_gmtime();
		/* Then fill in gaps by fetching from database */
		for (i=0; i<ngot; i++) {
			*block = nums[i]+1;
			if (str_len(vals[i])>0) continue;
			hdr = item_head(grp,nums[i]);
			if (hdr==NULL) continue; 
			nread++;
			x = xover_make(hdr,nums[i]);
			ncpy(bf,str_s(x),MAX_XOVER-2);
			s = strstr(bf,EOL);
			if (s==NULL) continue; 
			imsg("xidle: Fill in gap %.200s:%d %d-%d len=%d\n",group_name(grp),nums[i],group_getstart(grp),group_getend(grp),str_len(x));
			strcpy(s,"\n");
			str_copysn(vals[i],bf);
			if (nread>20) if ((i % 10)==9) if ((lib_gmtime()-start)>10) {
				imsg("xidle: Stopping at %d after %d seconds\n",*block,lib_gmtime()-start);
				lprintf(xidle_xstatus,BFSZ-1,"xidle: Sorting %.200s at %d after %d seconds\r\n",group_name(xcur_grp),*block,lib_gmtime()-start);
				break;
			}
		}
		
		/* Then write out to new file */
		stat_start(X_XOPEN);
		f = fopen(file_xov_idle(),F_APPEND);
		stat_end(X_XOPEN);
		if (f==NULL) {
			filemsg("Could not write new file %.200s\n",file_xov_idle());
			goto giveup;	
		}
		for (i=0; i<ngot; i++) {
			if (nums[i]> *block) break;
			if (str_len(vals[i])<=5) continue;
			fwrite(str_s(vals[i]),1,str_len(vals[i]),f);
		}
		
		fclosep(f);
giveup:	
		
		for (i=0; i<ngot; i++) str_free(vals[i]);		
		lprintf(xidle_xstatus,BFSZ-1,"xidle: Sorting %.200s at %d after %d seconds\r\n",group_name(xcur_grp),*block,lib_gmtime()-start);
		
		if (finished) if (*block>last) {
			lprintf(xidle_xstatus,BFSZ-1,"xidle: q=%d Sorted %.200s at %d after %d seconds\r\n",nxidle,group_name(xcur_grp),*block,lib_gmtime()-start);
			return TRUE;
		}
		return FALSE;
}
char *xidle_status(void)
{
	if (xover2_ngroups>0) {
		lprintf(xidle_xstatus,BFSZ-1,"xbuild: q=%d sorted=%d, fillin=%d copy=%d miss=%d %d\r\n",
			nxidle,xover2_ngroups,xover2_nfill,xover2_ncopy,xover2_nmiss,xover2_nmiss2);
	}
	return xidle_xstatus;
}


/* New xover stuff  

  First a function to clean up the gap at the end of a group due to index 
  caching.
  
*/

int item_xover_test(int chan, int grp)
{
	int first=group_getfrom(grp);
	int last=group_getend(grp);
	int n;
	String *x;
	int i;
	int nlost=0;
	char bf[BFSZ];
	int nok=0;
	int tot=0;
	double cpu;
	int msec;
	int start_msec;
	int start_clock;
	int len;
	wcache_write_one(grp,-1); /* Flush the cache if it exists*/
	
	/* Fetch all items in order */
	
	n = g_list_start(grp);
	start_clock = my_clock();
	start_msec = lib_msec();
	len = nok = tot = 0;
	xcache_init();
	for (i=0; (i<200000) && (n>=0); i++)  {
		x = item_xover(grp,n,i!=0);	
		if (x!=NULL) {
			nok++;
			len += str_len(x);
		}
		tot++;
		n = g_list_next();
	}
	cpu = my_clock() - start_clock;
	cpu = cpu / CLOCKS_PER_SEC;
	msec = lib_msec() -  start_msec;
	lprintf(bf,200,"Read %d of %d items ok, cpu=%3.6g seconds, time=%dmsec %d bytes %dk/sec\n",nok,tot,cpu,msec,len,(len/msec)); chan_write_s(chan,bf);
	/* ----------- Now fetch in reverse-------- */
	
	n = g_list_start(grp);
	start_clock = my_clock();
	start_msec = lib_msec();
	len = nok = tot = 0;
	xcache_init();
	for (i=0; i<500; i++)  {
		n = last-i;
		x = item_xover(grp,n,FALSE);	
		if (x!=NULL) {
			nok++;
			len += str_len(x);
		}
		tot++;
		n = g_list_next();
	}
	cpu = my_clock() - start_clock;
	cpu = cpu / CLOCKS_PER_SEC;
	msec = lib_msec() -  start_msec;
	lprintf(bf,200,"Reverse %d of %d items ok, cpu=%3.6g seconds, time=%dmsec %d bytes %dk/sec\n",nok,tot,cpu,msec,len,(len/msec)); chan_write_s(chan,bf);
	
	/* ----------- Now fetch in random order from 3 groups -------- */
	
	start_clock = my_clock();
	start_msec = lib_msec();
	len = nok = tot = 0;
	xcache_init();
	srand(1);
	for (i=0; i<500; i++)  {
		n = rand()%(last-first);
		if (n<0) n = -n;
		n += first;
		x = item_xover(grp+n%3,n,FALSE);	
		if (x!=NULL) {
			nok++;
			len += str_len(x);
		}
		tot++;
		n = g_list_next();
	}
	cpu = my_clock() - start_clock;
	cpu = cpu / CLOCKS_PER_SEC;
	msec = lib_msec() -  start_msec;
	lprintf(bf,200,"Random %d of %d items ok, cpu=%3.6g seconds, time=%dmsec %d bytes %dk/sec\n",nok,tot,cpu,msec,len,(len/msec)); chan_write_s(chan,bf);
	
	
	return TRUE;
}

/*
Find last item in xover file, then add any recent items then
clear the xover write cache for this group
*/
int item_xover_patchend(int grp)
{
	int n;
	int nlost=0;
	int nok=0;
	int start=group_getfrom(grp);
	int end=group_getend(grp);
	int last,first;
	String *x,*hdr;
	
	
	if (grp<0) return TRUE;
	item_xover_close();
	last = item_xover_findlast(grp);
	first = wcache_find_first(grp);
	
	imsg("Last message in file was %d, first in xover is %d\n",last,first);
	if (first<0) first = end+1;
	if (last+1>=first) goto skipit;
	
	/*	for (n=last+1;n<first; n++) {*/
	
	n = g_list_start(grp);
	for (;n<first; n = g_list_next()) {
		if (n<=0) break;
		if (n<last+1) continue;
		hdr = item_head(grp,n);
		if (hdr==NULL) {
			imsg("Xover patchend, could not find %s:%d\n",group_name(grp),n);
			continue; 
		}
		x = xover_make(hdr,n);
		item_xover_write(grp,n,x,TRUE);
	}
skipit:	
	wcache_write_one(grp,last);
	return TRUE;
}
int item_xover_findlast(int grp)
{
	FILE *f;
	char fname[BFSZ];
	char bf[MAX_XOVER];
	int last=0;
	int len,upto;
	item_xover_close();
	ncpy(fname,file_xover(grp),300);
	f = fopen(fname,"rb");
	if (f==NULL) {
		filemsg("Could not open %.200s to find last item\n",fname);
		return -1;
	}
	len = file_length(fileno(f));
	for (upto=len-1000;upto>0;upto-=1000) {
		if (len-upto>40000) break; 
		if (fseek(f,upto,SEEK_SET)!=0) break;
		if (fgets(bf,MAX_XOVER-1,f)==NULL) continue;
		last = 0;
		for (;!feof(f);) {
			if (fgets(bf,MAX_XOVER-1,f)==NULL) break;
			last = atoi(bf);
		}
		if (last!=0) break;
	}
	fclosep(f);
	return last;
}
#define FILE_XPATCH file_work("xovpatch.dat")
int xtell_master_patch(char *gname)
{
	FILE *f;
	f = fopen(FILE_XPATCH,"a");
	fprintf(f,"%s\n",gname);
	fclose(f);
	return TRUE;
}
static int xtupto;
void xtell_master_reset(void)
{
	xtupto = 0;
	remove(FILE_XPATCH);
}
void xtell_check(void)
{
	FILE *f;
	int start;
	char *s;
	int g;
	if (!suck_spawned_expire()) return;
	start = lib_gmtime();
	
	f = fopen(FILE_XPATCH,"rb");
	if (f==NULL) return;
	if (xtupto!=0) fseek(f,xtupto,SEEK_SET);
	for (;!feof(f);) {
		if (fgets(bf,BFSZ-1,f)==NULL) break;
		s = strchr(bf,'\r'); if (s!=NULL) *s = 0;
		s = strchr(bf,'\n'); if (s!=NULL) *s = 0;
		g = group_find(bf);
		if (g>=0) {
			imsg("Patch end of xover file (%s)\n",bf);
			item_xover_patchend(group_find(bf));
		}
		if ((lib_gmtime()-start)>10) break;
	}
	xtupto = ftell(f);
	imsg("patch: upto %d\n",xtupto);
	fclose(f);
}
String *item_repair_crlf(String *xitem)
{
	char *s;
	int max = 2600;
	String *fixed;
	fixed = str_new();
	
	if (xitem==NULL) return xitem;
	s = str_s(xitem);
	if (s==NULL) return FALSE;
	s++;
	
	for (;*s!=0;s++,max--) {
		if (*s=='\r') if (s[1]!='\n') return FALSE;
		if (*s=='\n') if (s[-1]!='\r') return FALSE;
		if (max<0) break;
	}
	return TRUE;
}
