/* Copyright (C) 1995 NetWin World, New Zealand, All rights reserved. */

/*......................................................................  

  Problem:
  Large groups:

  Solution:
	If 'last 10 minutes' total items > 10 then add block of 'n' size * 10
	Write entries as 'deleted' or something so they will vanish during resort 
	Remember how many left in block etc for this group
	Also write to 'adding.idx' file for slaves
	Delete adding.idx file once an hour or so.
  Solution 2:
	Cache incoming index entries for 10 minutes, then flush to disk in sorted order ? 


  
	
	  
	
	  bug:
	There is some situation where the index entries
	become faulty or are written faulty.



1) Write all index entries to a log file initially.
2) 
  
	
	  Seeking to wrong point in file (could be bug in pile stuff)
	
	
	  
>stream 1 1000 -lines 3

tellnews xcached "test.??"
tellnews expire_xover
tellnews showgroup test.3
>tellnews killitem test.3 976
............................................  

Find why buckets are not shown, but are counted in over
and are not deleted.

When reading index, throw away items for deleted buckets
When deleting bucket, scan cache for deleted items and remove
  
	Check every 5 minutes for excess buckets
Delete them
Remember not to re-use them.
Remember to process them correctly on the next expire.
When fetching item, if in bucket that is deleted give nice error in log file
----------------------


-----------------------------------------------------------------	 
Individual tests:
		1) Writing to piles correctly
		2) Expire deletes right pile buckets
		3) Expire deletes right entries in history file
		4) Expire resort_index works correctly.
		5) Item ranges are corrected.
		6) Test check buckets
		7) Process for rebuilding database using new rules (modified rebuild_gids)
		8) test rebuild index
		9) check if bucket lenghs are maintained correctly
-----------------------------------------------------------------
		New expire rules. 
				pile 1 groups * use 10%
				pile 2 groups rec.*,comp.*,news.*,soc.* use 50%
				pile 3 groups alt.* use 20%
				pile 4 groups *binaries*,*pictures*,*warez* use 30%
				pile 5 groups local.*,netwin.* use 10%
				pile 6 groups control* use 0%
				reserve 20%
		Find where gage is set and remove if expire_ispile() is true
		Choose pile based on largest numeric group in list. 

		Cyclic file system changes required:
				db_nnn.itm (if pile 0 which is old style)
				db_nnn_n.itm (mmm, make this a config setting?)
				Must record bucket pile and order.

		Setting to enable it.
				(existence of pile commands instead of upto commands)
				(IF both exist exit with rude warning)
		New expire process.
				For each pile
						If exceeds rules, mark oldest buckets for 
						deletion. Don't expire last bucket unless 
						max=0.
				Scan index, and record all items that are going
				to be deleted. 
				Scan history, and remove deleted items
				Scan index and remove deleted items.
		Process for rebuilding database using new rules
				(modified rebuild_gids)
		Process for finding buckets 	
				(Modified check_buckets)
		Pre-Extend setting
				If true, then when open new file, seek to end
				if true, when closing after write, write .siz file
				when opening existing, read .siz and seek to it
				When deleting, remove .siz
		New default bucket size for large systems, 100mb or 200mb


  command to scan history and remove items that no longer exist.
  Only check items older than 4 weeks

  Known bugs
		expire_pile stuff assumes one entry per pile when adding them up

*/

#include <stdlib.h>
#include <stdio.h>
#include <ctype.h>
#include <string.h>
#include <errno.h>
#ifdef __MWERKS__
#include "Mac.h"
#endif
#ifdef WIN32
#include <windows.h>
#include <io.h>
#endif
#include <ctype.h>
#include <string.h>
#include <assert.h>
#ifdef VMS
#include <unixio.h>
#endif
#ifndef WIN32
#ifndef Mac
#ifndef NOUNISTD
#include <unistd.h>
#endif
#endif
#endif		  
#ifdef OS2
#include <io.h>
#endif
#ifndef NO_FCNTL
#include <fcntl.h>
#else
/*RHP not for mac*/
#ifndef Mac
#include <file.h>
#endif
#endif


void db_init_spool(void);
#include "emsg.h"
#include "cmd.h"
#include "str.h"
#include "sary.h"
#include "lib.h"
char *get_ahead(String *sitem, char *head);
#include "group.h"
#include "hist.h"
#include "item.h"
#include "chan.h"
#include "db.h"
#include "dbf.h"
#include "expire.h"
#include "mutex.h"
#define MAX_ITEMS 90000
/*----------- External functions --------------*/

/*---------- Internal Functions ---------------*/
/* nbuf = Number of buffers to keep for this file */
/* szbuf = Size of each buffer to keep */


/*---------- File formats ---------------*/
/* 
INODE
  uint2 fidx; uint2 gid; uint4 num; uint pnt, uint4 len; uint4 next; (20 bytes)





				


DBF.IDX
		Used file Numbers, and Disk they are stored on.
		64,000 * 2	 int1 used; int1 disk
DBG.IDX
		gid index into dbi.idx
DBI.IDX
		db.idx
		64000 * 15
		Links...
DB_NNNNN.ITM	
---------- How it works -------------------
Initializing:
		If Running files exists, then do rebuild of index
		Read the index of used files
		Read the DBI index, and find lost items, find the last 
		item for every group.  If it is not an integral length, add
		bytes to make it so.
t Writing an article
		Open current file for APPEND if not open for write already
		Append to .itm file
		Append to .idx file
		Append to .idx buffer.	
		If change is in buffer, then update that
		If group in cache, add to group cache
t Reading a group
		if (grouppointer null) 
		Seek dbf.idx 
				seek / read / seek / read 
t Reading a header / article   (keep only one file open for read or write)
		Get group pointer
		Open item file / seek / read

t Expire: 
		kill(item)	Store up 100,000 and then write to .idx file
		Remove item from group cache if it exists.
		in order, or via a re-write process (test each)
		purge(upto)
				Scan index file to find how much each file is being used
				Delete empty ones
				rewrite one with most wasted space
				Stop if total wasted < 20%

t Flush 
		Close all files
		i_flush()

When we open the files
		When open dbi.idx, check it's an integral length, and or fixit.
---------------------------------------------*/

/*--------------- Static data -------------*/
static int fdbi = -1;	/* dbi.idx */
static int fdbs = -1;	/* dbi.new	(sorted version) */
static int fdbs_len = -1;	/* dbi.new	(sorted version) */
static int fdbg = -1;	 /* Pointers into dbi.idx file */
static int fdbi_len = 0;
static int fdbn = -1;	/* db_nnnnnn.itm */
static int fdbn_len = 0;		/* Length of file we are writing */
static int fdbn_fid = -1;		/* File id of current file we are writing */
static char fdbn_name[BFSZ];	/* File name of current article file */
static int fdbp_len = 0;		/* Non purged files  */
static int fdbp_fid = -1;		
static int fdbp = -1;
int db_check_fid(void);
static int idx_write_cache,idx_write_size;

static char fdbp_name[BFSZ];	
static int fdbr = -1;	/* db_nnnnnn.itm for reading */
static int fdbr_fid = -1;		/* File id of currently open one */
void index_log(char *in);

static int isrebuild,isrenumber,isregid,isaddlost,rebuild_fast,rebuild_rules;
void debug_index_show(char *gname);
int bsort_que_add_ifpile(int bkt);
static int next_grp_free = -1;


#define FILE_CHK file_work("dbi.chk")  /* if exists then db is dirty */
#define FILE_DBI file_work("dbi.idx") 
#define FILE_DBI_OLD file_work("dbi.old") 
#define FILE_SORT file_work("dbi.new") 
#define FILE_FIRST file_work("dbfirst.idx") 
#define FILE_LAST file_work("dblast.idx") 
#define FILE_FID_OLD file_work("dbfid.idx") 
#define FILE_FID file_work("dbpile.idx") 
int idx_dopatch_part(void); /* returns TRUE when finished */ 
static int insrt;
static int last_len;			/* Length of last item, used for cross posts */
static int item_min=10000000;	/* Increase to 10MB after testing */
static int cancel_bsort;
/* #define IDXSIZE 20 */
/* 2 gig per day  16 gig = 4milion articles = 80mb index*/
int xmax_groups;
int dbusy(int i);

/* If 0, then that group is empty, */

#define GROUP_CACHE 400
#define MAX_GRPS 100 /* Max groups in ONE item :-) */
#define NEED_RESORT file_work("resort.dat")
void out_setlen(int x);
int scan_free_all(void);
int pile_remove_one(int p, int i, int istrim, char *reason);
int db_reindex_stack_flush(int h);
int db_idle_slave(void);
int db_idle_addnew(int start, int xend);
void dblog(char * arg_list, ...);
int idx_write_add(int fid, int grp,int num,int pnt, int len);

int db_piles_trim(void);

/*typedef struct MREC {int ipnt; short fid; unsigned int gid; int num; int len; int pnt; int jump; struct MREC *next;} Mrec;
*/
static Mrec *group_dir[GROUP_CACHE];
static Mrec *group_dir_next[GROUP_CACHE];
static int group_lastused[GROUP_CACHE];
static Mrec *group_last[GROUP_CACHE]; /* Pointer to last one in list */
static int group_grp[GROUP_CACHE];
static int group_delay[GROUP_CACHE]; /* How long it took to read into memory */
static int group_nitems[GROUP_CACHE]; /* Number of items in this group	*/
static int nspool;

extern int xtime_now;

#define MAX_DBN 15000  /* 10,000 * 20 = 200 Gig    */
typedef struct {int fid,disk,used,free,inuse,cur,found;} DBN_OLD;
typedef struct {int fid,disk,used,free,inuse,cur,found,pile,cdate,reserved[5];} DBN;
static DBN dbn[MAX_DBN];
static int dbn_piledel[MAX_DBN];
static char dbn_isdel[MAX_DBN];
static char dbn_istrim[MAX_DBN];
static int dbn_isbusy[MAX_DBN];
static int dbn_wasused[MAX_DBN];
static int dbn_purged[MAX_DBN];
static int dbn_pending[MAX_DBN];
static int dbn_start[MAX_DBN];
static int dbn_end[MAX_DBN];
static int dbn_rerun[MAX_DBN];
#define BIGBF 10000
static int total_lost;
typedef struct PREC {int pnt; int num; unsigned int gid; short fid; struct PREC *next;} Prec;
char *get_aheadraw(String *sitem, char *head);
void db_kill_free(void);
int db_bucket_ranges(void);
void db_del_in_cache(int grp, int num);

int expire_isgiveup(void);
void idx_patch(int grp, int num, int fid, int pnt);
void patch_free(void);
void patch_free_one(Prec *p);
void idx_dopatch(void);
Mrec *db_item_read(int grp, int num) ;
char *db_spool_dirh(int d, int h);
int db_cancel_find(int grp, int num, int *islink);

void ex_cancel_read(void);
void ex_cancel_add(int gid, int num);

static int dbn_expired[MAX_DBN];

#define dbn_bounds(x)  ( ((x)<0) || ((x)>=MAX_DBN	 ) )
#define dir_bounds(x)  ( ((x)<0) || ((x)>=GROUP_CACHE) )
#define last_bounds(x) ( ((x)<0) || ((x)>=xmax_groups) )
#define last_boundsu(x) ( (((int) (x))<0) || (((int) (x))>=xmax_groups) )

/*-------Local functions ---------------------------------------------*/
int db_read_last(void);
int dbn_trim_load(void);
#define  group_gid	  group_getgroupid
int db_bucket_ranges_clear(void);
int max_items(void);
int m_insert_sort_start(void);
int m_insert_free(void);
int m_insert_sort_done(void);
int m_insert_sort(int c, Mrec *mm);

int out_close(int h);
void m_insert(int c, Mrec *mm);
void m_insert_mid(int c,Mrec *mnew);
int m_max_use(void);
int db_srt_finish(void);
void del_init(int max);
int dbn_trim(int x);
int feed_setnorules(int x);

int g_clear(char *reason);
int db_write_fid(void);
int db_trimold(String *xitem);
void *dbmalloc(int sz);
void dbfree(void *x);
int out_closeall(void);

int db_open_n(void);
int db_add_idx(int fid, int grp,int num,int pnt, int len);
char *to_idx(int xfi, int grp, int num, int pnt, int len, int jump);
int db_write_idx(int grp,char *rec);
int db_openi(void);
int g_cache_find(int grp);
int g_cache_add(int grp,char *rec, int pnt);
int g_cache_lru(int emptyok, int skipti);
Mrec *g_get_mrec(int grp, int num);
int m_update(Mrec *m);
int g_group_read(int grp);
void g_free(int c, char *reason);
int dbg_open(void);
void purge_scan(int fi);

#define fid_name(x) fid_name_line(x,__LINE__)
char *fid_name_line(int fid, int line);
int frec_to_mrec(char *rec, int pnt, Mrec *m);
void del_add(int fid, int grp, int num);
int kill_max(void);
int db_next_fid(void);
int dbn_remove(int fid);
void purge_one(int fi);
int del_read(int fi, int important);
String *db_scanitem(int fi, int f, int *gid, int *num);
int del_this(int gid,int num); /* is this item a deleted one (_n.del file) */
void db_init_fid(void);
void db_init_idx(void);
void bit_setused(char *bit, int x);
void group_fixranges(int grp, int num); /* not sure if we use this */
void group_donefix(void);
int bit_notused(char *bit, int x);
int idx_addmissing(int idxoffset); /* add missing one back in */
int db_unpurge(int gid, int num, String *xitem);
void mrec_to_frec(Mrec *m,char *bf);
char *del_name(int fid);
void patch_add(int fi);
int db_total_space_left(void);

int db_reindex_new_sort(int *upto) ;
char *db_stack_file(int i);
int db_reindex_stack(int g, Mrec *m);
int db_reindex_new_scan(int *upto); /* Return true when finished */
int db_reindex_new(int start) ;

void m_cache_free(int x);
Mrec *m_cache_find(int gid, int num);
void m_cache_add(int fid, int grp, int num, int pnt, int totlen);
void m_cache_clear(void);
Mrec *g_get_mrec_cache(int grp, int num);
void g_list_clear(int x);
char *fid_name_lost(int fid);
int db_jumpin(int gid, int jump, int lastjump);
int m_update_del(Mrec *m);
int db_open_n_g(int sz);
static int split_set;
static short *re_grps;
static int *gid_last; /* Last record for this group */
static int *gid_first; /* Last record for this group  */

int suck_malloc_groups(void);
int group_malloc_groups(void);
int expire_malloc_groups(void);

int db_malloc_groups(void)
{
		max_groups = atoi(config_get(C_MAX_GROUPS));
		if (max_groups==0) max_groups = 55000;
		if (max_groups<1000) max_groups = 1000;
		if (max_groups>200000) max_groups = 200000;
		xmax_groups = max_groups + 5000;
				
	   
	re_grps = (short *) mymalloc2(sizeof(short) * max_groups,M_DB);
		gid_last = (int *) mymalloc2(xmax_groups * sizeof(int),M_DB);
		gid_first = (int *) mymalloc2(xmax_groups * sizeof(int),M_DB);
		suck_malloc_groups(); /* 3 * int*/
		group_malloc_groups(); /* 8*int + 1 */ 
		expire_malloc_groups(); /* 1 */

	
		return TRUE;
}
static int stopdbi;
void db_set_stopdbi(int x)
{
		stopdbi = x;
}
void db_setsplit(int x)
{
		split_set = x;
}
static int remove_check;
int db_set_check(int x)
{
		remove_check = x;
		return TRUE;
}
/*--------------------------------------------------------------------*/
int db_test_nulls(char *s, int len, char *gname, int n, char *info);
int db_add(int grp, int num, String *xitem)
{
											
		char xtop[BFSZ];
		char *eoh = "\015\012\015\012";
		char *s;
		int xlen;
		int tlen;
		int totlen;
		int hlen; 
		int failed;
		int r1,r2;

		stat_start(X_DBADDD);

	  group_addsize(grp,str_len(xitem));
		xlen = str_len(xitem);
		if (xlen>(MAX_STRLEN-100)) {
				emsg("db_add: rejecting item too big %d max is %d\n",xlen,MAX_STRLEN);
				stat_end(X_DBADDD);
				return TRUE; /* intentionaly wrong */			
		}
		/* Make fixed length so the real length can be calculated if necessary */
		/* Add gid so index can be rebuilt, and so we know which 
		one is the 'primary' file */

		lprintf(xtop,BFSZ-1,"~%08d:%010d:%06d:",str_len(xitem),num,group_gid(grp));
		tlen = strlen(xtop);
		totlen = xlen + tlen + 1;
		s = mystrstr(str_s(xitem),eoh);
		if (s==NULL) {emsg("item: Item has no body, will not save \n"); stat_end(X_DBADDD); return FALSE;}
		hlen = s - str_s(xitem) + 2 + tlen;
		str_addsn(xitem,"^");

		stat_start(X_DB_ADD2);
		if (!db_open_n_g(grp)) {emsg("db_open_n_g() failed\n"); stat_end(X_DBADDD); return FALSE;}
		stat_end(X_DB_ADD2);
		if (fdbn_fid<0) {emsg("fdbn_fid < 0\n"); stat_end(X_DBADDD); return FALSE;} /* Should never happen */
		if (dbn_bounds(fdbn_fid)) {emsg("fdbn_Fid bounds failed\n"); stat_end(X_DBADDD); return FALSE;}
		

		dbf_seek(fdbn,0,SEEK_END);
		stat_start(X_DB_WRITE);
		failed = FALSE;
		r1 = dbf_write(fdbn,xtop,strlen(xtop));
		if (r1!= (int) strlen(xtop)) failed = TRUE;
	if (config_true(C_TEST_NULLS)) {
		db_test_nulls(str_s(xitem),str_len(xitem),group_name(grp),num,"db_add"); 
	}
		r2 = dbf_write(fdbn,str_s(xitem),str_len(xitem));
		if (r2!= str_len(xitem)) {failed=TRUE;}
	/* Start thread output if valid*/
	/*thout_start(fdbn);*/
		if (failed) filemsg("Writing item to disk failed\n");
		stat_end(X_DB_WRITE);
	
		db_add_idx(fdbn_fid,grp,num,fdbn_len,totlen);
		m_cache_add(fdbn_fid,grp,num,fdbn_len,totlen);
		last_len = totlen;
		fdbn_len += (r1+r2);
		out_setlen(fdbn_len);
		dbn[fdbn_fid].used += totlen;
		if (dbn[fdbn_fid].used != fdbn_len) {
				emsg("Disk usage drifted for bucket db_%d.itm corrected from %d to %d\n",
						fdbn_fid,dbn[fdbn_fid].used,fdbn_len);
				dbn[fdbn_fid].used = fdbn_len;
		}
		dbn[fdbn_fid].fid = fdbn_fid;
		str_setlen(xitem,xlen);
		diskuse_add(totlen); /* not counting size of index */
		if (failed) {
				lib_setpaused(); /* Pause for 60 seconds at least */
				stat_end(X_DBADDD);
				return FALSE;
		}
		stat_end(X_DBADDD);
		return TRUE;
}
int db_test_nulls(char *s, int len, char *gname, int n, char *info)
{
	int tot=len;
	FILE *f;

	if (!config_true(C_TEST_NULLS)) return FALSE;
	
	for (;*s!=0 ;len--) {
		s++;
	}
	if (len>1) {
		emsg("null found in item we are writing %d  %s:%d\n",len,gname,n);
		f = fopen(file_log("null.log"),"a");
		if (f!=NULL) {
			fprintf(f,"%s Null found in item %d/%d left %s:%d \n",info,tot-len,tot,gname,n);
			fclose(f);
		}
	}
	return TRUE;

}
int out_open(int h);
int next_disk(void);
int db_next_fid_type(int odd, int type);
void db_chose_size(int *n,int odd);
static int out_fid[100];
static int out_h[100];
static int out_len[100];
static int out_inited = FALSE;
int out_init(void)
{
		int i;
	static int xdone; 
	if (!xdone) db_init_spool();
	xdone = TRUE;

		dbn_trim_load();
	idx_write_cache = config_true(C_IDX_WRITE_CACHE);
	idx_write_size = atoi(config_get(C_IDX_WRITE_CACHE));
	if (idx_write_cache) if (idx_write_size==0) idx_write_size = 20000; 
		if (!db_ispile()) return TRUE;
		if (out_inited) return TRUE;
		imsg("Out init called %d piles\n",get_sort_piles_nf());
		out_inited = TRUE;
		for (i=0; i<get_sort_piles_nf(); i++) {
				out_h[i] = -1;
				out_fid[i] = -1;
		}
		if (is_readonly()||get_expire_mode()) return TRUE;
		for (i=0; i<get_sort_piles_nf(); i++) {
				if (out_h[i]<0) out_open(i);
		}
		return TRUE;
}
int out_reset(void)
{
	int i;
	out_closeall();
		for (i=0; i<get_sort_piles_nf(); i++) {
		out_close(i);
/*				  out_h[i] = -1;
				out_fid[i] = -1;
		*/
		}
	return TRUE;
}
int out_flush(void)
{
		int i;
		if (!db_ispile()) return TRUE;
		if (!out_inited) return TRUE;
		for (i=0; i<get_sort_piles_nf(); i++) {
				if (out_h[i]>=0) dbf_flush(out_h[i]);
		}		
		return TRUE;
}
#ifdef NETWARE
#define READLOCK 1
#endif

#ifdef Mac
#define READLOCK 1
#endif

#ifdef VMS
#define READLOCK 1
#endif

int out_flush_id(int id)
{
		int i;
		if (!db_ispile()) return TRUE;
		if (!out_inited) return TRUE;
		for (i=0; i<get_sort_piles_nf(); i++) {
		if (out_fid[i]==id) if (out_h[i]>=0) {
#ifdef READLOCK
			out_close(i);
#else
			dbf_flush(out_h[i]);
#endif
		}
		}		
		return TRUE;
}
int out_close_id(int id)
{
		int i;
		if (!db_ispile()) return TRUE;
		if (!out_inited) return TRUE;
		for (i=0; i<get_sort_piles_nf(); i++) {
		if (out_fid[i]==id) if (out_h[i]>=0) {
			out_close(i);
		}
		}		
		return TRUE;
}
int out_closeall(void)
{
		int i;
		if (!db_ispile()) return TRUE;
		if (!out_inited) return TRUE;
		for (i=0; i<get_sort_piles_nf(); i++) {
				if (out_h[i]>=0) dbf_close(out_h[i]);
				out_h[i] = -1;
		}		
		return TRUE;
}
static int next_wanted;
int out_switch(void)
{
		int i;
		if (!db_ispile()) return TRUE;
		if (!out_inited) return TRUE;
		for (i=0; i<get_sort_piles_nf(); i++) {
				out_close(i);
				next_wanted = (i % nspool) + 1;
				out_fid[i] = db_next_fid_type(i,TRUE);
				next_wanted = 0;
		}		
		return TRUE;
}
int out_close(int h)
{
		if (!out_inited) return TRUE;
		if (out_fid[h]>=0) dbn_isbusy[out_fid[h]] = FALSE;
		if (out_h[h]>=0) dbf_close(out_h[h]);
		out_h[h] = -1;
		out_fid[h] = -1;
		return TRUE;
}
int out_open(int h)
{
		char fname[BFSZ];

		if (is_readonly()) return TRUE;
		if (out_fid[h]<0) {
				/* Scan for any bucket which is not full */
				/* Next choose a disk based on h if possible */
				/* lastly choose any disk that has free space */
				next_wanted = (h % nspool) + 1;
				out_fid[h] = db_next_fid_type(h,FALSE);
				next_wanted = 0;
				imsg("Opening bucket %d for pile [%d]=%d\n",out_fid[h],h,expire_htopile(h));
		}
		if (out_fid[h]<0) return FALSE;
		imsg("reopen append (%d) %s\n",h,fid_name(out_fid[h]));
		strcpy(fname,fid_name(out_fid[h]));
		out_h[h] = item_open_append(fname);
#ifdef READLOCK
		if (out_h[h]<0) {
		if (fdbr>=0) dbf_close(fdbr);
			fdbr = -1;
			out_h[h] = item_open_append(fname);
	}
#endif
		if (out_h[h]<0) {
				filemsg("Unable to append to %s\n",fname);
				return FALSE;
		}
		out_len[h] = dbf_len(out_h[h]);
	if (out_len[h]==0) {
		dblog("Opened a new EMPTY bucket (%s)\n",fname);
		dbn_start[out_fid[h]] = 1000000000;
		dbn_end[out_fid[h]] = 0;
	} else {
/*		dbn_start[out_fid[h]] = -1;
		dbn_end[out_fid[h]] = 0;*/
	}
		dbn_isbusy[out_fid[h]] = TRUE;
		return TRUE;
}
static int cur_out = -1;

int db_open_n_g(int g)
{		
		int loop=0;
	char bf[BFSZ];
		int h;
	static int waspile;


		out_init();
		if (!db_ispile()) {
		if (waspile) {
			emsg("FATAL, PILE settings changed on live system, please report to support@netwinsite.com\n");
			dnews_ex("FATAL, PILES EXPIRE.CONF CHANGED");
		}
		return db_open_n();
	}
	waspile = TRUE;
		fdbn = -1;
		fdbn_fid = -1;

		if (is_readonly()) return FALSE;
		h = gage_getnext();
		if (h==0) emsg("gage_getnext zero %d = %d\n",g,h);
try_again:
		stat_start(X_DB_SEEK);
		if (out_h[h] < 0) if (!out_open(h)) return	FALSE;
		stat_end(X_DB_SEEK);
		fdbn = out_h[h];
		fdbn_fid = out_fid[h];
		fdbn_len = out_len[h];
		cur_out = h;
		dbn[fdbn_fid].used = fdbn_len;

	if (config_true(C_INDEX_DEBUG)) {
		lprintf(bf,BFSZ-1,"open [%d] fdbn=%d fid=%d pnt=%d\n",h,fdbn,fdbn_fid,fdbn_len);
		index_log(bf);	
	}

		if (out_len[h]>item_min) { /* Switch to next file */
		if (config_true(C_BUCKETS_SORT)) bsort_que_add(out_fid[h]);
		else bsort_que_add_ifpile(out_fid[h]);
				out_close(h);
				fdbn = -1;
				imsg("Choosing next bucket for pile %d\n",h);
				if (loop++ < 100) goto try_again;
		}

		return TRUE;
}
void out_setlen(int x)
{
		if (cur_out<0) return;
		out_len[cur_out] = x;
}
int db_open_n(void)
{
		if (is_readonly()) return -1;
		if (fdbn<0) {
		 if (strlen(fdbn_name)==0) {
				imsg("Db_current should be set by db_init from last entry in dbfid.idx\n");
				strcpy(fdbn_name,fid_name(0));
		 }
		 fdbn = item_open_append(fdbn_name);
		 imsg("fid_name dbn_name {%s} %d\n",fdbn_name,fdbn);
		 
		 if (fdbn<0) {filemsg("%s\n",fdbn_name); return FALSE;}
		 fdbn_len = dbf_len(fdbn);
		}
		if (fdbn<0) {
				filemsg("Could not append to {%s} \n",fdbn_name);
				return FALSE;
		}
		if (fdbn_len>item_min) { /* Switch to next file */
				if (!db_choose_next()) return FALSE;
		}
		return TRUE;
}
int db_fidn(void)
{
		int i;
		int tot=0;
		for (i=0;i<MAX_DBN; i++) {
				if (dbn[i].inuse) tot++;
		} 
		return tot;
}
#define FILE_LATER file_work("db_later.dat")
void remove_later(int i)
{
		FILE *f;
		f = fopen(FILE_LATER,"a");
		if (f==NULL) {filemsg("Cannot write %s\n",FILE_LATER); return;}
		fprintf(f,"%d\n",i);
		fclosep(f);
		if ((i>=0) && (i<MAX_DBN)) dbn_isdel[i] = TRUE;
}
void db_do_later(void)
{
		int todel[7000];
		int i,n;
		char bf[BFSZ];
		FILE *f;

		if (is_readonly()) return ;
		todel[0] = 0;
		f = fopen(FILE_LATER,"r");
		if (f==NULL) return;
		for (n=0;(n<7000) && !feof(f);) {
				if (fgets(bf,BFSZ-10,f)==NULL) break;
				if (strlen(bf)==0) continue;
				if (!isdigit(bf[0])) continue;
				todel[n++] = atoi(bf);
		}
		fclosep(f);
		remove(FILE_LATER);
		for (i=0; i<n; i++) {
				imsg("Remove Later (%s) \n",fid_name(todel[i]));
				dbn[todel[i]].inuse = FALSE;
				dbn_isdel[todel[i]] = FALSE;
				dbn_pending[todel[i]] = FALSE;
		dblog("Removing later %s\n",fid_name(todel[i]));

				if (remove(fid_name(todel[i]))!=0) { /*ISCLOSED*/
			if (errno!=ENOENT) {
						 filemsg("Remove later {%s} failed\n",fid_name(todel[i]));
						 remove_later(todel[i]);
						 dbn[todel[i]].inuse = TRUE;
						 dbn_isdel[todel[i]] = TRUE;
			}
				} 
				dbn[todel[i]].used = 0;
				dbn[todel[i]].free = 0;
		}
}
int db_set_rebuild_rules(void)
{
	rebuild_rules = TRUE;
	return TRUE;
}
int db_isregid(void)
{
	return isregid;
}
int db_wasused_clear(void)
{
	int i;
	for (i=0; i<MAX_DBN; i++) dbn_wasused[i] = FALSE;	   
	return TRUE;
}

int db_nextused(int fi)
{
		static int wasused[MAX_DBN];
		int i;
		int oldest = -1;
		int cdate = lib_gmtime()+100000;
		if (fi < 0) {
				for (i=0; i<MAX_DBN; i++) {
						if (dbn[i].inuse) wasused[i] = TRUE;
						else wasused[i] = FALSE;
			dbn_wasused[i] = wasused[i];
				}
				if (isrenumber || isregid) db_choose_next();
		}
		if (fi< -1) fi = -1;

		if (isrenumber || isregid) {
		  for (i=0; i<MAX_DBN; i++) {
		dbn_wasused[i] = wasused[i];
				if (wasused[i]) {
						if (dbn[i].cdate<cdate) {
								oldest = i;
								cdate = dbn[i].cdate;
						}
				}
		  } 
		  imsg("Next oldest bucket that was being used %d\n",oldest);
		  if (oldest>=0) wasused[oldest] = FALSE;
		  return oldest;
		}

		for (i=fi+1;i<MAX_DBN; i++) { 
				if (dbn[i].inuse) return i;
		} 
		return -1;
}
int next_fid(void)
{
		int i;
		for (i=0; i<MAX_DBN; i++) {
			if (dbn[i].used>0) dbn[i].inuse = TRUE;
			if (!dbn[i].inuse) return i;
		}
		emsg("No free fidS PANIC\n");
		return 0;
}
void db_slave_bucket(int n,int dsk, int pile)
{
		imsg("Master just told us bucket %d is on disk %d\n",n,dsk);
		if ((n<0) || (n>=MAX_DBN)) return;
		dbn[n].inuse = TRUE;
		dbn[n].disk = dsk;
		dbn[n].fid = n;
		dbn[n].pile = pile;
}
int db_next_fid_new(void)
{
		return db_next_fid_type(0,TRUE);		
}
int db_next_fid(void)
{
		return db_next_fid_type(0,FALSE);		
}

int debug_dbn(void)
{
		int i;
		for (i=0; i<MAX_DBN; i++) {
		imsg("find_free [%d] used %d, inuse %d  pending=%d isbusy=%d, piledel=%d wasused=%d istrim=%d\n",
				i,dbn[i].used,dbn[i].inuse,dbn_pending[i],dbn_isbusy[i],dbn_piledel[i],dbn_wasused[i],dbn_istrim[i]);
			if (dbn_pending[i]) continue;
			if (dbusy(i)) continue;
			if (i==fdbp) continue;
			if (dbn[i].used>0) dbn[i].inuse = TRUE;
			if (!dbn[i].inuse) break;
		}
	emsg("All buckets used, this can be caused by two things\n");
	emsg("Either you have a large spool (e.g. 100gig or more) and \n");
	emsg("you haven't increased bucket_size in dnews.conf\n");
	emsg("Or, your directory protections are wrong so dnews can't\n");
	emsg("create buckets (Check this first)\n");
	emsg("Then increase bucket_size to 150000000 (50mb,or 500mb) in dnews.conf and\n");
	emsg("immediately run an expire, also you should be using piles based expiration\n");
	emsg("IF that doesn't do the trick, delete the oldest 20 bucket files, and try again (run another expire)\n");
	emsg("Also set your expire.conf pile rules to expire 10% of your news, and run an expire as soon as you start dnews\n");
	emsg("Lastly set in dnews.conf bucket_n 14000\n");
		dnews_ex("dbn ran out");
		return TRUE;
}

int real_db_next_fid_type(int odd, int newonly);
int db_next_fid_type(int odd, int newonly)
{
	int r;
	r = real_db_next_fid_type(odd,newonly);
	return r;
}
int real_db_next_fid_type(int odd, int newonly)
{
		FILE *f;
	static int lastfound;
		int i;
		int j;
		int lasti = fdbn_fid;
	odd = expire_htopile(odd);
		if (dbn_bounds(lasti)) lasti = -1;
search_again:
		if (config_true(C_BUCKET_FILL) || db_ispile())	{
		  if (!newonly) {
				for (i=0; i<MAX_DBN; i++) {
					if (dbn[i].pile != odd) continue;
					if (dbusy(i)) continue;
					if (dbn_pending[i]) continue;
					if (i==fdbp) continue;
					if (dbn_isdel[i]) continue; /* Don't use ones that are going to be deleted */
					if (i==fdbp_fid) continue; /* Don't choose the purge bucket */
					if (dbn[i].used>0) dbn[i].inuse = TRUE;
					if (dbn[i].inuse) if (dbn[i].used<9000000) if (dbn[i].used<item_min) break;
			   }
			   if (i<MAX_DBN) {
						imsg("Found un-full bucket, filling it up [%d] %d\n",i,dbn[i].used);
						return i;
			   }
		  }
		}
		for (i=0; i<MAX_DBN; i++) {
/*imsg("find_free [%d] used %d, inuse %d odd=%d i&1 = %d\n",
	 i,dbn[i].used,dbn[i].inuse,odd,i&1);*/
			if (dbn_pending[i]) continue;
			if (dbusy(i)) continue;
			if (i==fdbp) continue;
			if (dbn[i].used>0) dbn[i].inuse = TRUE;
			if (!dbn[i].inuse) break;
		}
		if (i>=MAX_DBN) {emsg("FATAL: Used up all file names\n"); debug_dbn(); return 0;}
		imsg("next bucket chosen %d pile %d\n",i,odd);
		dbn[i].inuse = TRUE;
		dbn[i].disk = next_disk();
		dbn[i].cdate = lib_gmtime();
		dbn[i].fid = i;
		dbn[i].pile = odd;
		if (lib_exists(fid_name(i))) {
				emsg("Lost bucket found %s, checking buckets\n",fid_name(i));
				db_check_buckets();
				goto search_again;		
		}
		slave_bucket(i,dbn[i].disk,odd);
		remove(del_name(i)); /* Clear any deleted info sitting around */
		f = fopen(FILE_FID,F_APPEND);
		if (f==NULL) {filemsg("FATAL Cannot append to {%s} \n",FILE_FID); return 0;}
		if (!dbn_bounds(lasti)) if (lasti>=0) {
				dbn[lasti].fid = lasti;
				dbn[lasti].cur = FALSE;
				if (fwrite(&dbn[lasti],sizeof(DBN),1,f)!=1) filemsg("while writing to file_fid\n");
		}
		for (j=0; j<MAX_DBN; j++) dbn[j].cur = FALSE;
		dbn[i].cur = TRUE;
		if (fwrite(&dbn[i],sizeof(DBN),1,f)!=1) filemsg("error writing to file_fid\n");

		if (fclosep(f)!=0) filemsg("closing file_fid\n");

		return i;
}
char *fid_name2(int dsk, int fid);
char *fid_name_line(int fid, int line)
{
		if ((fid<0) || (fid>=MAX_DBN)) return "FID_NAME_RANGE";
		if (expire_ispile()) if (dbn[fid].pile==0) dmsg("db: db.c:%d, pile 0 used fid=db_%d.itm\n",line,fid);
		return fid_name2(dbn[fid].disk,fid);
}
char *fid_name_pile(int fid);
char *fid_name_pile(int fid)
{
		static char bf[BFSZ];
		static char bf2[BFSZ];
		int dsk;
		if ((fid<0) || (fid>=MAX_DBN)) return "FID_NAME_RANGE";

		dsk = dbn[fid].disk;
		if ((dsk<0) || (dsk>=nspool)) {
				emsg("FATAL, DISK OUT OF RANGE spool[%d] fid=%d\n",dsk,fid);
				dsk = 0;
		}
#ifdef NETWARE
		lprintf(bf,BFSZ-1,"d%d_%d.itm",fid,dbn[fid].pile);
#else
		lprintf(bf,BFSZ-1,"db_%d_%d.itm",fid,dbn[fid].pile);
#endif
		if (config_true(C_SPOOL_HASH)) {
				lprintf(bf,BFSZ-1,"%d/db_%d_%d.itm",fid % 100,fid,dbn[fid].pile);
		}
		strcpy(bf2,file_spool(dsk,bf)); 		
		return bf2;
}
char *fid_name_lost(int fid)
{
		static char bf[BFSZ];
		static char bf2[BFSZ];
		int dsk = dbn[fid].disk;
		if ((dsk<0) || (dsk>=nspool)) {
				emsg("FATAL, DISK OUT OF RANGE spool[%d] fid=%d\n",dsk,fid);
				dsk = 0;
		}
		lprintf(bf,BFSZ-1,"db_%d.kil",fid);
		strcpy(bf2,file_spool(dsk,bf)); 		
		return bf2;
}
char *fid_name2(int dsk, int fid)
{
		static char bf[BFSZ];
		static char bf2[BFSZ];

		if ((dsk<0) || (dsk>=nspool)) {
				emsg("FATAL, DISK OUT OF RANGE spool[%d] fid=%d\n",dsk,fid);
				dsk = 0;
		}
		if (expire_ispile())  {
				if (dbn[fid].pile<=0) goto old_style;
				if (fid<0) goto old_style;
#ifdef NETWARE
				lprintf(bf,BFSZ-1,"d%d_%d.itm",fid,dbn[fid].pile);
#else
				lprintf(bf,BFSZ-1,"db_%d_%d.itm",fid,dbn[fid].pile);
#endif
		} else {
old_style:		lprintf(bf,BFSZ-1,"db_%d.itm",fid);
		}
		if (config_true(C_SPOOL_HASH)) {
				lprintf(bf,BFSZ-1,"%d/db_%d.itm",fid % 100,fid);
		}
		strcpy(bf2,file_spool(dsk,bf)); 		
		return bf2;
}
char *del_name(int fid)
{
		static char bf[BFSZ];
		static char bf2[BFSZ];
		if ((fid<0) || (fid>=MAX_DBN)) {emsg("del_name with bad fid %d\n",fid); fid=0;}
		lprintf(bf,BFSZ-1,"db_%d.del",fid);
		if (config_true(C_SPOOL_HASH)) {
				lprintf(bf,BFSZ-1,"%d/db_%d.del",fid%100,fid);
		}
		strcpy(bf2,file_spool(dbn[fid].disk,bf));				
		return bf2;
}
char *file_spoolh(int d, char *s, int h)
{
		static char bf[BFSZ];
		if (config_true(C_SPOOL_HASH)) {
				lprintf(bf,BFSZ-1,"%d/%s",h % 100,s);
				return file_spool(d,bf);
		} else {
				return file_spool(d,s);
		}
}
int db_add_link(int grp, int num) /* Points to the last item added */
{
		if (fdbn<0) {
		imsg("db_add_link: Thats odd, fdbn is closed %d\n",fdbn);
		return FALSE;
	}
		return db_add_idx(fdbn_fid,grp,num,fdbn_len-last_len,last_len);
}
char *db_cheat_get(void)
{
		static char bf[BFSZ];
		if (fdbn<0) return "";
		lprintf(bf,BFSZ-1,":%d:%d:%d:%d:%d",fdbn_fid,fdbn_len-last_len,last_len,dbn[fdbn_fid].disk,dbn[fdbn_fid].pile);
		return bf;
}
static int ischeat;
static Mrec mcheat;
void db_cheat_set(char *s)
{
		static char bf[BFSZ];
		if (s==NULL) {ischeat = FALSE; return;}
		ncpy(bf,s,BFSZ-1);
		s = strtok(bf,":");  if (s==NULL) return;  /* group */
		s = strtok(NULL,":"); if (s==NULL) return; /* Num */ 
		mcheat.num = atoi(s);
		s = strtok(NULL,":"); if (s==NULL) return; /* fid */ 
		mcheat.fid = atoi(s);
		s = strtok(NULL,":"); if (s==NULL) return; /* offset */ 
		mcheat.pnt = atoi(s);
		s = strtok(NULL,":"); if (s==NULL) return; /* len */ 
		mcheat.len = atoi(s);
		ischeat = TRUE;
}
void dbf_setbf(int h, int n);
int db_openi(void)
{
	int min;
	int ntry=0;
		int nbf = atoi(config_get(C_INDEX_BLOCKS));
try_again:
		ntry++;
		if (!dbg_open()) return FALSE;
		if (fdbi>=0) return TRUE; /* Already open */
		fdbi = idx_open(FILE_DBI);
		if (fdbi<0) {filemsg("Can't open append to {%s} \n",FILE_DBI); return FALSE;}
		fdbi_len = dbi_len(fdbi);
	min = atoi(config_get(C_INDEX_MIN));
	if (min>0) if (min>fdbi_len) {
		emsg("Index is below min size, copy dnews*.log and expire*.log then remove index_min setting and restart dnews\n");
		dnews_ex("Index is below min size, copy dnews*.log and expire*.log then remove index_min setting and restart dnews");
	}
		dbi_seek(fdbi,0,SEEK_END); 
		if ((fdbi_len % IDXSIZE) !=0) {
				if (is_readonly()) {
						fdbi_len = fdbi_len - (fdbi_len % IDXSIZE);
						return TRUE;
				}
				emsg("FATAL: dbi.idx is corrupt, performing consistency check, rebuild_index may be needed\n");
				db_check_index();
				if (ntry>2) return FALSE;
				goto try_again;
		}
		return TRUE;
}
int db_test_idx(int chan, int grp, int len)
{
		char bf[IDXSIZE];
		static Mrec m;
		imsgf("Testing gid %d len %d",grp,len);
		m.gid = grp;
		m.len = len;
		mrec_to_frec(&m,bf);
		frec_to_mrec(bf,0,&m);
		imsgf("Result %d==%d %d==%d\n",grp,m.gid,len,m.len);
		return TRUE;
}
void show_hex(char *x, int len)
{
   int i;
   printf("Hex: ");
   for (i=0; i<len; i++,x++)  {
	  printf("%2x ",*x);
   }
  printf("\n");
}
void index_log(char *in)
{
	static FILE *f;
	char *s;
	char tmpstr[BFSZ];
	time_t xt=time(NULL);
	char t[BFSZ];
	s = safe_ctime(&xt,tmpstr);
	if (s!=NULL) strncpy(t,s+8,11); t[11] = 0;
	if (f==NULL) f = myfopen(file_log("index.log"),"a");
	if (f==NULL) return;
	fprintf(f,"%s %s",t,in);	
}
void index2_log(char *in)
{
	static FILE *f;
	char *s;
	char tmpstr[BFSZ];
	time_t xt=time(NULL);
	char t[BFSZ];
	s = safe_ctime(&xt,tmpstr);
	if (s!=NULL) strncpy(t,s+8,11); t[11] = 0;
	if (f==NULL) f = myfopen(file_log("index2.log"),"a");
	if (f==NULL) return;
	fprintf(f,"%s %s",t,in);	
}
static int ispart;
void db_setpart(int x)
{
	ispart = x;
}
int db_add_idx(int fid, int grp,int num,int pnt, int len)
{
		char *rec;
		int c;
	char bf[BFSZ];
	int at;

	if (idx_write_cache) return idx_write_add(fid,grp,num,pnt,len);

	if (config_true(C_INDEX_DEBUG)) {
		lprintf(bf,BFSZ-1,"%s:%d %d %d %d\n",group_name(grp),num,fid,pnt,len);
		index_log(bf);	
	}
	if (ispart) { /* Don't put in index until complete */
		part_callback(fid,grp,num,pnt,len);
		return TRUE;
	}

	dmsg("db: Add_idx %s:%d %d:%d len=%d\n",group_name(grp),num,fid,pnt,len);
		if (fdbi<0) {
				if (!db_openi()) {
			filemsg("Cant open index");
			return FALSE;
		}
		}		
	if (dbn_start[fid]>=0) {
		at = fdbi_len/IDXSIZE;
		if (dbn_start[fid]>at) dbn_start[fid] = at;
		if (dbn_end[fid]<at+2) dbn_end[fid] = at+2;
	}
		rec = to_idx(fid,grp,num,pnt,len,-1);
		c = g_cache_find(grp);
		if (c>=0) {
				dmsg("db: Add_cache[%d] %s:%d %d:%d\n",c,group_name(grp),num,fid,pnt);
				g_cache_add(c,rec,fdbi_len);
		}
		return db_write_idx(grp,rec);
}
int m_update(Mrec *m)
{
		char bf[IDXSIZE];
		if (is_readonly()) {
				imsg("slave: cannot update record\n");
				return TRUE;
		}

		if (fdbi<0) {
				if (!db_openi()) return FALSE;
		}		
		if (m->ipnt>fdbi_len) {
				emsg("Update past end of index %d %d \n",m->ipnt,fdbi_len);
				return FALSE;
		}
		dbi_seek(fdbi,m->ipnt,SEEK_SET);
		mrec_to_frec(m,bf);
		if (dbi_write(fdbi,bf,IDXSIZE)!=IDXSIZE) filemsg("m_update, rebuild_index needed\n");
		dbi_seek(fdbi,0,SEEK_END);
		return TRUE;
}
int m_update_del(Mrec *m)
{
		char bf[IDXSIZE];		
		Mrec mm;
		if (is_readonly()) {
				imsg("slave: cannot update record\n");
				return TRUE;
		}
		if (m==NULL) return FALSE;
		if (fdbi<0) {
				if (!db_openi()) return FALSE;
		}		
		if (m->ipnt>fdbi_len) {
				emsg("Update past end of index %d %d \n",m->ipnt,fdbi_len);
				return FALSE;
		}
		dbi_seek(fdbi,m->ipnt,SEEK_SET);
		dbi_read(fdbi,bf,IDXSIZE);
		frec_to_mrec(bf,m->ipnt,&mm);
		mm.len = 0;
		mrec_to_frec(&mm,bf);
		dbi_seek(fdbi,m->ipnt,SEEK_SET);
		if (dbi_write(fdbi,bf,IDXSIZE)!=IDXSIZE) filemsg("m_update, rebuild_index needed\n");
		dbi_seek(fdbi,0,SEEK_END);
		return TRUE;
}
static int ispatching;
void db_setpatching(int x)
{
	ispatching = x;
}
static int inresort;
static int nidxbf;
int max_nidxbf(void)
{
		static int n;
		if (n>0) return n;
		n = atoi(config_get(C_NIDXBF));
		if (n==0) n = 4000;
		return n;
}

/*struct BFrec {int grp; char rec[IDXSIZE];} ;
static struct BFrec *idx_bf;
*/
#define FILE_IDXBF file_work("idx.bf")
static FILE *fidxbf;
int db_write_idx_bf(int grp, char *rec)
{
		int r;
		if (fidxbf==NULL) {
				fidxbf = myfopen(FILE_IDXBF,F_WRITE);
				if (fidxbf==NULL) {
						filemsg("Could not write to (%s) \n",FILE_IDXBF);
						return TRUE;
				}
		}
		r = fwrite(&grp,sizeof(int),1,fidxbf);
		r += fwrite(rec,IDXSIZE,1,fidxbf);
		if (r!=2) {
				filemsg("Error writing to (%s) \n",FILE_IDXBF);
				fclosep(fidxbf);
				fidxbf = NULL;
				remove(FILE_IDXBF);
		}
		nidxbf++;
		return TRUE;
}
int db_write_idx_now(void)
{
		int r;
		int i;
		char rec[BFSZ];
		int grp;
		imsgf("Sending buffered data to index, %d records\n",nidxbf);
		if (nidxbf==0) return TRUE;
		if (!db_openi()) {
				filemsg("Unable to write buffered data to index\n");
				return FALSE;
		}
		if (fidxbf!=NULL) {
				fclosep(fidxbf);
		}
		nidxbf = 0;
		fidxbf = myfopen(FILE_IDXBF,F_READ);
		if (fidxbf==NULL) {
				filemsg("Could not read (%s) \n",FILE_IDXBF);
				return TRUE;
		}

		for (i=0; !feof(fidxbf); i++) {
				r = fread(&grp,sizeof(int),1,fidxbf);
				r += fread(rec,IDXSIZE,1,fidxbf);
		if (feof(fidxbf)) break; 
				if (r<2) {
			filemsg("Buffered read failed in resort %d\n",r);
			break;
		}
				db_write_idx(grp,rec);
		}
		fclosep(fidxbf);
		fidxbf = NULL;
		nidxbf = 0;
		imsg("Finished sending buffered index data, %d items\n",i);
		return TRUE;
}
int db_write_idx(int grp,char *rec)
{ 
		int last;
		int gid;
		
		gid = group_gid(grp);
		if (gid<0) {
				emsg("Write to negative gid %d\n",gid);
				return FALSE;
		}
		if (inresort) return db_write_idx_bf(grp,rec);
		dbi_seek(fdbi,0,SEEK_END);
		if (dbi_write(fdbi,rec,IDXSIZE)!=IDXSIZE) {
				filemsg("Cannot append idx data to %s\n",FILE_DBI);
				lib_setpaused(); /* Pause for 60 seconds at least */
				db_flush2(); /* Close so if wrong lenght it will be corrected*/
				return FALSE;
		}

		/* If last rec in buffer, update last pointer in buffer*/
		if (last_bounds(gid)) {
				emsg("Gid out of bounds in write_idx %d\n",gid);
				return FALSE;
		}
		last = gid_last[gid];
		if (last == -1) {
				if (!dbg_open()) return FALSE;
				dbf_seek(fdbg,gid * 4,SEEK_SET);
				dbf_write(fdbg,&fdbi_len,4);
				dbf_seek(fdbg,0,SEEK_END);
		} else {
				if (last>fdbi_len) {
						emsg("Add idx past end gid=%d, at %d past %d\n",gid,last+IDXSIZE-4,fdbi_len);
				} else {
						dbi_seek(fdbi,last+IDXSIZE-4,SEEK_SET);
						dbi_write(fdbi,&fdbi_len,4); 
				}
				dbi_seek(fdbi,0,SEEK_END);
		}				


		fdbi_len += IDXSIZE;

	
		/* Update 'lasti' for each group */
		gid_last[gid] = fdbi_len - IDXSIZE;
		return TRUE;
}
/* fi[2],gid[2],num[4],pnt[4],len[4],jump[4] 
  max_dbn = 10000

  64,000,000 * 16 = 


*/
char *to_idx(int xfi, int grp, int num, int pnt,  int len, int jump)
{
		static char bf[BFSZ];
		short fi = (short) xfi;
		unsigned int gid = (unsigned int) group_gid(grp);
		unsigned short xgid;
		unsigned int xlen;
		
		xgid = gid & 0xffff;
		xlen  = len & 0x0fffffff;
		xlen =	xlen | ((gid & 0xf0000)<<12);
		memcpy(bf,&fi,2);
		memcpy(bf+2,&xgid,2);
		memcpy(bf+4,&num,4);
		memcpy(bf+8,&pnt,4);
		memcpy(bf+12,&xlen,4);
		memcpy(bf+16,&jump,4);
		return bf;
}
static int last_read_len;
void show_mrec(int grp, int num)
{
		Mrec *m;
		m = g_get_mrec(grp,num); 
		if (m==NULL) return;
		imsg("Corrupt item? [%d] fid=%d gid=%d num=%d pnt=%d len=%d\n"
						,m->ipnt,m->fid,m->gid,m->num,m->pnt,m->len);
}

void show_mrec_s(char *bf, int grp, int num)
{
		Mrec *m;
		m = g_get_mrec(grp,num); 
		if (m==NULL) {
				lprintf(bf,IDXSIZE-1,"No index entry found for that item\n");	  
				return;
		}
	
		lprintf(bf,IDXSIZE-1,"Index Entry [%d] gid=%d num=%d, at=%d len=%d %s\n"
						,m->ipnt,m->gid,m->num,m->pnt,m->len,fid_name(m->fid));
}
static int waslost;
String *db_get(int grp, int num, int from, int len, int *totlen)
{
		Mrec *mrec;
		static String *xitem;
		/*char start[BFSZ];*/
		int xlen,islink;
		int c;
		char *s;
	int inhead;
		int issame = FALSE;
		int rlen;
		int checked_fid = FALSE;
		vtrace();
		dmsg("db: Get Mrec %d:%d from %d, len %d step1\n",grp,num,from,len);
		waslost = TRUE;
		if (ischeat) {
				mrec = &mcheat; 
				if (dbn_bounds(mrec->fid)) {
						imsg("FID error feeding item\n");
						return NULL;
				}
				if (dbn_istrim[mrec->fid] || !dbn[mrec->fid].inuse) {
						dmsg("live: Feeding item that has been expired\n");
						return NULL;
				}
				goto  skip_mrec;
		}
		if (db_cancel_find(grp,num,&islink)) {
				imsg("db: item has been cancelled %s:%d\n",group_name(grp),num);
				waslost = FALSE;
				return NULL;
		}
		mrec = m_cache_find(group_gid(grp),num);
		if (mrec==NULL) {
		 c = g_cache_find(grp);
		 if (c<0) {
				mrec = db_item_read(grp,num);
				if (mrec==NULL) imsg("db: bin no index entry for %d:%d\n",grp,num);
		 } else {
				mrec = g_get_mrec_cache(grp,num); /* Might be worth caching last one ? */
				if (mrec==NULL) imsg("db: No index entry for %d:%d\n",grp,num);
		 }
		}
		if (mrec==NULL) {
				return NULL;
		}
		if (mrec->len==0) {
				dmsg("db: index entry gives zero length for item\n");
				return NULL;
		}
skip_mrec:
		if (xitem==NULL) xitem = str_new();
		str_clearbig(xitem);
		if (is_readonly()) {
				if (fdbr_fid>=0)  if (!dbn[fdbr_fid].inuse) db_check_fid();
		}
		/* Close fdbr_fid */
		if (mrec->fid>=0) {
				if (dbn_istrim[mrec->fid]) {
						imsg("User read item in cyclic deleted bucket %d\n",mrec->fid); 
			next_grp_free = grp; /* Free the internal cache of this group */
						return NULL;
				}
		}
		out_flush_id(fdbr_fid);
		out_flush_id(mrec->fid); /* Flush the one we are going to read */
		if ((fdbr_fid != mrec->fid) || (fdbr<0)) {
		if (!db_ispile()) db_open_n_g(grp);
				if (fdbr>=0) {
						dbf_close(fdbr); fdbr = -1;
				}
				fdbr_fid = mrec->fid;
				if (fdbn<0) fdbn_fid = -1; /* slave so it's not open dummy */
				
				if ((fdbr_fid == fdbn_fid) && !db_ispile()) {
						fdbr = fdbn;	/* It's the same file we are writing to */
						issame = TRUE;
				} else {
retryit:
						fdbr = item_open_read(fid_name(fdbr_fid));
						if (fdbr<0) {
								if (!is_readonly() || checked_fid || get_expire_mode()) {
									filemsg("Could not open %s\n",fid_name(fdbr_fid));
				}
								/* db_cancel(grp,num); */
								if (is_readonly() || get_expire_mode()) if (!checked_fid) {
										checked_fid = TRUE;
										db_check_fid();
					imsg("did checkfid name is now (%s)\n",fid_name(fdbr_fid));
										goto retryit;
								}
								return NULL;
						}
						dmsg("db: fid_open %s %d\n",fid_name(fdbr_fid),fdbr);
				}
		}
		dmsg("db: opened file %d %d\n",fdbr,fdbr_fid);

		if (fdbr<0) {
				filemsg("Unable to open {%s} for read\n",fid_name(fdbr_fid));
				/* db_cancel(grp,num); */
				return NULL;
		}
		dmsg("db: db_get [%d:%d] fid=%d:%d seek=%d len=%d  %s:%d gid=%d\n",
				fdbr,fdbn,fdbr_fid,fdbn_fid,mrec->pnt+from,mrec->len
				,group_name(grp),num,mrec->gid);
		if (!dbf_seek(fdbr,mrec->pnt+from,SEEK_SET)) {filemsg("Cannot seek to %d in fid=%d\n",mrec->pnt,fdbn_fid); goto fail;}
		/* Now read the right part of the item */
		xlen = mrec->len - from;
		if (xlen<len) len = xlen;
		*totlen = mrec->len;
		last_read_len = *totlen;
		if ((len>(MAX_STRLEN)) || (len<0)) {
				emsg("db_get: cannot get item, too big %d\n",len); 
				goto fail;
		}
		str_setlen(xitem,len+1);
		rlen = dbf_read(fdbr,str_s(xitem),len);
		dmsg("db: got %d of %d, from %d rest %d\n",rlen,len,from+mrec->pnt,xlen);
		if (ischeat) goto skip_retry;
		if (rlen<len) {
		 if (!config_true(C_NOFLUSHRETRY)) {
				emsg("db: got %d of %d, from %d rest %d\n",rlen,len,from+mrec->pnt,xlen);

				if (rlen==0) {
				if (issame) {
								emsg("db: fdbr==fdbn, attempting reopen (%s) (%d) (%d)\n",fdbn_name,fdbr_fid,fdbn_fid);
								fdbr = -1;
								dbf_close(fdbn);
								fdbn = -1;
								emsg("db: Opening (%s) fid=%d\n",fid_name(fdbr_fid),fdbr_fid);
retryit2:						fdbr = item_open_read(fid_name(fdbr_fid));
								if (fdbr<0) {
										filemsg("Could not open %s\n",fid_name(fdbr_fid));
										/* db_cancel(grp,num); */
	
							if (is_readonly() || get_expire_mode()) if (!checked_fid) {
											checked_fid = TRUE;
									 db_check_fid();
									  goto retryit2;
									}
										
					
					return NULL;
								}
								dbf_seek(fdbr,mrec->pnt+from,SEEK_SET);
								rlen = dbf_read(fdbr,str_s(xitem),len);
								dbf_close(fdbr);
								db_open_n_g(grp);
								fdbr = fdbn;			
								emsg("db: reopen read got %d of %d, from %d rest %d\n",rlen,len,from+mrec->pnt,xlen);
						} else {
/*
FILE *f;
emsg("db: Trying a reopen (%s) to get new data fid=%d\n",fid_name(fdbr_fid),fdbr_fid);
dbf_close(fdbr); fdbr = -1;
fflush(NULL);
f = fopen(fid_name(fdbr_fid),"rb");
if (f==NULL) {
	filemsg("couldn't open it\n");
	abort();
}
emsg("file length %d, seek %d \n",file_length(fileno(f)),mrec->pnt+from);
if (fseek(f,mrec->pnt+from,SEEK_SET)!=0) {
	filemsg("could not seek to %d\n",mrec->pnt+from);
	abort();
}
errno = 0;
rlen = fread(str_s(xitem),1,len,f);
filemsg("REad %d bytes\n",rlen);
filemsg("Could not read {%s:%d} at %d in %s got %d of %d bytes\n",group_name(grp),num,mrec->pnt+from,fid_name(fdbr_fid),rlen,len);
abort();
*/

#ifdef hpux
								emsg("db: Trying a reopen (%s) to get new data fid=%d\n",fid_name(fdbr_fid),fdbr_fid);
								sleep(1);
				fflush(NULL);
								dbf_close(fdbr);
								fdbr = -1;
								fdbr = item_open_read(fid_name(fdbr_fid));
								if (fdbr<0) {
										filemsg("Could not open %s\n",fid_name(fdbr_fid));
										return NULL;
								}
								dbf_seek(fdbr,mrec->pnt+from,SEEK_SET);
								rlen = dbf_read(fdbr,str_s(xitem),len);
								emsg("db: reopen read got %d of %d, from %d rest %d\n",rlen,len,from+mrec->pnt,xlen);
#endif
						}
				}
		 }
		}
skip_retry:;
		if (rlen<len) {
				if (ischeat) goto fail;
				filemsg("Could not read {%s:%d} at %d in %s got %d of %d bytes\n",group_name(grp),num,mrec->pnt+from,fid_name(fdbr_fid),rlen,len);
				goto fail;
		}
		str_setlen(xitem,len);
	inhead = TRUE;
		/* To fix bug in 5.0beta2, search for full stop in message */
	if (config_true(C_FULLSTOP_FIX)) {
		s = str_s(xitem);
		if (s!=NULL) {
				for (;*s!=0;s++) if (*s=='\n') {
						if (s[1]=='.') if (s[2]=='\r') if (s[3]=='\n') {
								s++;
								*s = 0;
								len = s-str_s(xitem);
								emsg("Removing faulty item caused by bug in 5.0beta2 %s:%d\n",group_name(grp),num);
								str_setlen(xitem,len);
								*totlen = from + len; /*mrec->len;*/
								item_delete(grp,num);
								goto fail;
						}
			if (inhead) {
			 if (s[1]=='M') {
				if (strncmpnc(s+1,"Message-ID:",11)==0) {
					if (strncmp(s+1,"Message-ID:",11)!=0) {
						strncpy(s+1,"Message-ID:",11);
					}
				}

			 }
			 if (s[1]=='\r') inhead = FALSE;
			}
				}
		}
	}		 
		/*ncpy(start,str_s(xitem),80);*/
		if (rlen<len) {
				filemsg("Could not read {%s:%d} at %d in %s got %d of %d bytes\n",group_name(grp),num,mrec->pnt+from,fid_name(fdbr_fid),rlen,len);
				goto fail;
		}
		if (issame) {
				dbf_seek(fdbn,0,SEEK_END);
				fdbr = -1;
		}												 
		return xitem;
fail:;
		/* db_cancel(grp,num); */
		if (issame) {
				dbf_seek(fdbn,0,SEEK_END);
				fdbr = -1;
		}
		return NULL;
}
#define FILE_LOST file_work("lostidx.dat")
static int nkill_lost;
int readel_total(void)
{
	return nkill_lost;
}
void db_kill_lost(int grp, int num)
{
		Mrec *m;
		FILE *f;
		int start,end;
		m = g_get_mrec(grp,num); 
		if (m==NULL) return;
	total_lost++;	
	nkill_lost++;
		start = group_getstart(grp);
		end = group_getend(grp);
		if (!config_true(C_REMOVE_ALL_LOST)) {
	if (num+5>end) {
				emsg("Not removing lost item as it is a new item and may simply be a cache problem %s:%d\n",group_name(grp),num);
				return;
		}
	}
		if (start!=num) group_setrange(grp,-1);
		if (!is_readonly()) {
				/*if (dbn_bounds(x)) goto doinfo;
				if (dbn_istrim[mm.fid]) goto doinfo;*/
				imsg("Removing hanging index entry %s:%d\n",group_name(grp),num);
				imsg("Additional info [%d] fid=%d gid=%d num=%d pnt=%d len=%d\n"
						,m->ipnt,m->fid,m->gid,m->num,m->pnt,m->len);
		} else {
		if (inresort) {
					imsg("In resort so not Removing hanging index entry %s:%d\n",group_name(grp),num);
		} else {
					imsg("Removing hanging index entry %s:%d\n",group_name(grp),num);
					imsg("Additional info [%d] fid=%d gid=%d num=%d pnt=%d len=%d\n"
						,m->ipnt,m->fid,m->gid,m->num,m->pnt,m->len);
		}
		}
		if (is_readonly()) {
				f = fopen(FILE_LOST,"a");
				if (f==NULL) return;
				fprintf(f,"%s %d\n",group_name(grp),num);
		db_del_in_cache(grp,num);
				fclosep(f);
		} else {
				if (!inresort) {
			if (suck_spawned_expire()) {
				imsg("Expire in progress so bucket may have just been deleted\n");
			} else {
				if (config_true(C_DNEWS_TESTING)) {
					emsg("fixing bug here\n");
					abort();
				}
				item_delete(grp,num);
			}
		} else {
					imsg("In resort so not Removing hanging index entry %s:%d\n",group_name(grp),num);
		}
		}
}
void db_del_in_cache(int grp, int num)
{
		Mrec *m;
	m = m_cache_find(group_gid(grp),num);
		if (m!=NULL) m->len = 0;
}
void db_kill_lost_bkgnd(void)
{
		char *s;
		FILE *f;
	int start = lib_gmtime();
		int g,n;
	int ndone=0;
		char bf[BFSZ];
		if (is_readonly() || inresort) return;

	if (config_true(C_KILL_LOST_DISABLE)) return;
		f = fopen(FILE_LOST,"r");
		if (f==NULL) return;
		imsg("Found some hanging index entries reported by slave, processing them now...\n");
		for (;!feof(f);) {
				if (fgets(bf,BFSZ-10,f)==NULL) break;
		ndone++;
				s = strtok(bf," \n\r"); if (s==NULL) continue;
				g = group_find(s);
				s = strtok(NULL," \n\r"); if (s==NULL) continue;
				n = atoi(s);
				if (g>=0) db_kill_lost(g,n);
		if (ndone>20) if ((lib_gmtime()-start)>10) {
			emsg("db_kill_lost_bkgnd stopping after 10 seconds, %d entries done file size was %d bytes\n"
				,ndone,file_length(fileno(f)));
			break;
		}
		}
		fclosep(f);
		imsg("Finished processing hanging entries\n");
		if (remove(FILE_LOST)) filemsg("Unable to remove (%s)\n",FILE_LOST);
		return;
}
String *db_get_head(int grp, int num)
{
		String *abit;
		int i;
		char bf[BFSZ];
		char *e;
		int totlen;
		char *s;
		int len;
		int zlen,znum,zgid;
		static String *xitem;
		static String *rslt2;

		if (xitem==NULL) xitem = str_new(); str_clear(xitem);
		if (rslt2==NULL) rslt2 = str_new(); str_clear(rslt2);

		if (config_true(C_DB_GET_BUG)) {
				for (i=0,e=NULL;(e==NULL) && (i<30);i++) {
						abit = db_get(grp,num,0,4000,&totlen);
						if (abit==NULL) break;
						str_add(xitem,abit);
						e = strstr(str_s(xitem),"\r\n\r\n");
				}				
		} else {
				e = NULL;
				abit = db_get(grp,num,0,4000,&totlen);
				if (abit==NULL) goto failed;
				str_add(xitem,abit);
				e = strstr(str_s(xitem),"\r\n\r\n");
				if (e==NULL) {
						dmsg("dbbug: re-fetch required for %d:%d\n",grp,num);
						abit = db_get(grp,num,0,60000,&totlen);
						if (abit==NULL) goto failed;
						str_copy(xitem,abit);
						e = strstr(str_s(xitem),"\r\n\r\n");
				}
		}
failed:
		if ((e==NULL) && (abit==NULL)) {
				if (!ischeat) {
						if (abit==NULL) imsg("Could not find the item %s:%d\n",group_name(grp),num);
						else  emsg("Could not find the end of the header %s:%d\n",group_name(grp),num);
						db_kill_lost(grp,num);
				}
				return NULL;
		}
		if (e==NULL) {
				if (!ischeat) {
						emsg("Could not find end of header in item %s:%d\n",group_name(grp),num);
						db_kill_lost(grp,num);
				}

				/* db_cancel(grp,num); */
				return NULL;
		}
		e[2] = 0; /* Wipe out the second crlf */ 
		s = str_s(xitem) + 1;
		zlen = atoi(s)+29;
		s = strchr(s,':'); if (s==NULL) goto return_null;
		znum = atoi(s+1);
		s = strchr(s+1,':'); if (s==NULL) goto return_null;
		zgid = atoi(s+1);
		s = strchr(s+1,':'); if (s==NULL) goto return_null;
		/* Match item number and length of item */
		if (ischeat) {
		 if (zlen!=mcheat.len) {
				ncpy(bf,str_s(xitem),50);
				imsg("cheat: length wrong %d %d (%s)\n",zlen,mcheat.len,bf);
				goto return_null;
		 }
		 if (znum!=mcheat.num) {
				ncpy(bf,str_s(xitem),50);
				imsg("cheat: num wrong %d %d (%s)\n",znum,mcheat.num,bf);
				goto return_null;
		 }
		}
		len = e - s+1;
		str_copys(rslt2,s+1,len);
		return rslt2;
return_null:;
		if (!ischeat) {
				emsg("Header corrupt %s:%d\n",group_name(grp),num);
				show_mrec(grp,num);
				db_kill_lost(grp,num);
		}
		/* db_cancel(grp,num); */
		return NULL;
}
int thin_trim_dot(String *xitem);
String *db_get_body(int grp, int num, int from, int len, int *rfrom)
{
		static String *xitem;
		int totlen;
		char *e,*s;
		int xlen;
		char bf[BFSZ];
		int xmore=0;
try_again:;
		xitem = db_get(grp,num,from,len+xmore,&totlen);
		if (xitem==NULL) {
				*rfrom = -1;
				show_mrec(grp,num);
				return NULL;
		}
//	if (totlen>20000) imsg("db_get_body getting %d of %d from %d\n",len,totlen,from);
		if (from==0) { /* Strip off header and correct return len */
				e = strstr(str_s(xitem),"\r\n\r\n"); 
				if (e==NULL) {
						xmore += 20000;
						if (xmore>100000) goto return_null;
						goto try_again;
				}
				s = str_s(xitem);
				xlen = str_len(xitem)-(e+4-s); /* Probably wrong */
				memmove(s,e+4,xlen);
				if (xlen>(MAX_STRLEN)) {
						emsg("db_get_body: cannot get item, too big %d\n",xlen); 
						return NULL;
				}
				str_setlen(xitem,xlen);
		}

/*				s = str_s(xitem)+str_len(xitem);
				for (j = -10; j<3; j++) {
						imsg("Byte [%d]  %d\n",j,s[j]);
				}
  */

		*rfrom = from+len;
		xlen = str_len(xitem);
		if (xlen==0) {
				emsg("body empty? %s:%d\n",group_name(grp),num);
				return NULL;
		}
		if ((from+len)>=totlen) *rfrom = -1; /* Flag 'end of item' */
		else if ((xlen+10) > totlen) {xlen -= 10; *rfrom -= 10;}

		if (*rfrom<0) if (str_s(xitem)[xlen-1] != '^') {
				emsg("End ^ missing %s:%d \n",group_name(grp),num);
				if (str_len(xitem)>50) {
						ncpy(bf,str_s(xitem) + str_len(xitem)-50,50);
						imsg("End of article (%s)\n",bf);
						ncpy(bf,str_s(xitem),50);
						imsg("Start of article (%s)\n",bf);
				}
				show_mrec(grp,num);
				db_kill_lost(grp,num); /*db_cancel(grp,num);*/
				return NULL;
		}
		if (*rfrom<0) xlen--;

	if (config_true(C_THREAD_FIX)) {
	  if (*rfrom<0) {
			str_setlen(xitem,xlen);
		thin_trim_dot(xitem);
		xlen = str_len(xitem);	
	  }
	}
		if (xlen>(MAX_STRLEN)) {
				emsg("db_get_body: cannot get item, too big %d\n",xlen); 
				return NULL;
		}
		str_setlen(xitem,xlen);
		return xitem;
return_null:;
		emsg("Body corrupt %s:%d\n",group_name(grp),num);
		show_mrec(grp,num);
		/* db_cancel(grp,num); */
		return NULL;
}
static int nm;
Mrec *g_get_mrec_cache(int grp, int num)
{
		int lastgrp = -99;
		Mrec *m;
		int gg;

		if (grp<0) return NULL;
		m = m_cache_find(group_gid(grp),num);
		if (m!=NULL) return m;
		m = g_get_mrec(grp,num);
		if (lastgrp!=grp) if (m!=NULL) {
				gg = gid_to_group(m->gid);
				if (gg>=0) m_cache_add(m->fid,gg,m->num,m->pnt,m->len);
		}
		lastgrp = grp;
		return m;
}
Mrec *g_get_mrec(int grp, int num)
{
		int c;
		Mrec *m;
		int i;
	int zmax;

		if (grp<0) return NULL;
		c = g_cache_find(grp);
		if (c<0) c = g_group_read(grp);
		if (c<0) return NULL;
		if (group_dir[c]==NULL) return NULL;
	group_lastused[c] = xtime_now;
		m = group_dir_next[c];
	zmax = max_items();

		for (i=0;(m!=NULL) && (i<zmax); m=m->next,i++) {
				if (m->num==num) {
			group_dir_next[c] = m;
			return m;
		}
		}
		m = group_dir[c];
	zmax = max_items();
		for (i=0;(m!=NULL) && (i<zmax); m=m->next,i++) {
				if (m->num==num) {
			group_dir_next[c] = m;
			return m;
		}
		}
		return NULL;
}


/* mrec cache, I want to cache the last 1000 group,num->mrec pairs
		Hashed, linked list, of things, containing the mrec. 
typedef struct MREC {int ipnt; short fid; short gid; int num; int len; int pnt; int jump; struct MREC *next;} Mrec;

*/		
#define M_NHASH 1000
static Mrec *m_cache;
static Mrec *m_cache_idx[M_NHASH];
static int m_size;
static int m_upto;
#define m_hash(g,n) (abs((g+n) % M_NHASH))
void m_cache_clear(void)
{
		int i;
		if (m_cache!=NULL) memset(m_cache,0,m_size*sizeof(Mrec));
		for (i=0; i<M_NHASH; i++) {
				m_cache_idx[i] = NULL;
		}
		m_upto = 0;
}
void m_cache_add(int fid, int grp, int num, int pnt, int totlen)
{
		int sz;
		int h;
		Mrec m;

		if (config_true(C_MCACHE_DISABLE)) return ;

		m.ipnt = 0;
		m.fid = fid;
		m.gid = group_gid(grp);
		m.num = num;
		m.len = totlen;
		m.pnt = pnt;
		m.jump = 0;
		if (m_cache==NULL) {
				sz = atoi(config_get(C_DB_INDEX_CACHE));
				if (sz==0) sz = 2000;
				m_cache = mymalloc2(sz*sizeof(Mrec),M_DB);
				memset(m_cache,0,sz*sizeof(Mrec));
				m_size = sz;
		}
		m_cache_free(m_upto);
		m_cache[m_upto] = m;
		h = m_hash(m.gid,m.num);
		if ((h<0) || (h>=M_NHASH)) {emsg("H OUT OF BOUNDS %d\n",h); h=0;}
		m_cache[m_upto].next = m_cache_idx[h];

		m_cache_idx[h] = &m_cache[m_upto];
		m_upto++;
		if (m_upto>=m_size) m_upto = 0;
}
Mrec *m_cache_find(int gid, int num)
{
		int h;
		Mrec *m;
		int z;

		if (config_true(C_MCACHE_DISABLE)) return NULL;
		h = m_hash(gid,num);
		if ((h<0) || (h>=M_NHASH)) {emsg("H OUT OF BOUNDS %d\n",h); h=0;}
		m = m_cache_idx[h];
		for (z=0;m!=NULL ;m = m->next,z++) {
				if (z>9000000) {emsg("loop detected %d %d \n",gid,num); dnews_ex("cache_find");}
				if (m->gid==(unsigned int) gid) if (m->num == num) {
						return m;
				}
		}				
		return NULL;
}
void m_cache_free(int x)
{
		int h,z;
		Mrec *mf = &m_cache[x];
		Mrec *m,*last;
		h = m_hash(mf->gid,mf->num);
		if ((h<0) || (h>=m_size)) {emsg("H OUT OF BOUNDS %d\n",h); h=0;}
		if (mf->gid==0) if (mf->num==0) if (mf->jump==0) return;
		m = m_cache_idx[h];
		last = m;
		for (z=0;m!=NULL;m = m->next,z++) {
				if (z>9000000) {emsg("loop detected %d \n",x); dnews_ex("cache_free");}
				if (m==mf) {
						if (last==m) m_cache_idx[h] = m->next;
						else last->next = m->next;
						return;
				}
				last = m;
		}				
		emsg("FATAL, M_CACHE_FREE, %d  %d:%d\n",x,mf->gid,mf->num);
}
int g_get_group(int grp)
{
		int c;
		c = g_cache_find(grp);
		if (c<0) c = g_group_read(grp);
		if (c<0) return -1;
		if (group_dir[c]==NULL) return -1;
	group_lastused[c] = xtime_now;
		return c;
}
int g_cache_add(int c,char *rec, int pnt)
{
		Mrec mm;
		if (c<0) return TRUE;
		frec_to_mrec(rec,pnt,&mm);
		m_insert(c,&mm);
		return TRUE;
}
void g_free_all(char *reason)
{
		int i;
		for (i=0; i<GROUP_CACHE; i++) g_free(i,reason);
		g_list_clear(-1);
}
void db_clear_group(int g)
{
		static int now;
		db_free_cache(g,"db_clear_group");
		g_list_clear(g);
		if (is_readonly()) {
				if (fdbi>=0) dbi_close(fdbi);
				fdbi = -1;
		}
		if (lib_gmtime()==now) return;
		db_check_fid();
		now = lib_gmtime();
/* Flush input caching on index file, for readonly servers*/
		
}
void db_free_cache(int g,char *reason)
{
		int c = g_cache_find(g);
		if (c>=0) g_free(c,reason);
}
void g_free(int c, char *reason)
{
		int z;
		Mrec *m,*mnext;
		if (c<0) return;
		if (dir_bounds(c)) {
				emsg("g_free, directory bounds check  failed %d\n",c);
				return;
		}
		if (group_dir[c]!=NULL) dmsg("db: g_free %d\n",c);

	if (group_grp[c]>=0) imsg("g_free(%d) age=%d %s %s\n",c,xtime_now-group_lastused[c],reason,group_name(group_grp[c]));
		for (z=0,m=group_dir[c];m!=NULL;m=mnext,z++) {
				if (z>9000000) {emsg("loop detected %d \n",c); dnews_ex("g_Free");}
				mnext = m->next;
				myfree2(m,M_DIR);	   nm--;
		}
		group_last[c] = NULL;
		group_dir[c] = NULL;
		group_dir_next[c] = NULL;
		group_grp[c] = -1;
		g_list_clear(c);
}
void g_free_item(int grp, int num);
void g_free_item(int grp, int num)
{
		int c = g_cache_find(grp);
		int z;
		Mrec *m,*mnext,*mlast;
		if (c<0) return;
		if (dir_bounds(c)) {
				emsg("g_free, directory bounds check  failed %d\n",c);
				return;
		}
		if (group_dir[c]==NULL) return;
		group_dir_next[c] = NULL;

		mlast = NULL;
		for (z=0,m=group_dir[c];m!=NULL;m=mnext,z++) {
				if (z>9000000) {emsg("xx loop detected %d \n",c); dnews_ex("g_Free");}
				mnext = m->next;
				if (m->num == num) {
						myfree2(m,M_DIR);	   nm--;
						if (mlast==NULL) group_dir[c] = mnext;
						else mlast->next = mnext;
						imsg("removing item from group_dir cache %s:%d\n",group_name(grp),num);
						group_last[c] = NULL;
						g_list_clear(c);
						break;
				}
				mlast = m;
		}
}

void m_trimuse(int c);
void m_trimuse(int c)
{
		static int xthis = -19;
		int i;
	int k;
		int ok = FALSE;
		if (c==xthis) return;

		imsg("db: Trimming directory cache from %d items [%d]\n",nm,c);
		imsg("db: Trimming directory %d \n",nm>m_max_use());

	for (i=0; (i<GROUP_CACHE) && (nm>m_max_use()); i++) {
		k = g_cache_lru(FALSE,c);
		if (k<0) {
			imsg("db: Trimming stopped because k<0\n");
			break;
		}
		if (k==c) {
			imsg("db: Trimming stopped because k==c %d\n",c);
			break;
		}

				if (group_dir[k]==NULL) continue;
			group_dir_next[k] = NULL;
				if (k!=c) if (group_dir[k]!=NULL) {
						g_free(k,"trimuse");
						ok = TRUE;
				}
	}
		imsg("db: Trimming directory cache smart got it to %d items\n",nm);

		for (i=0; i<GROUP_CACHE && nm>m_max_use(); i++) {
				if (group_dir[i]==NULL) continue;
			group_dir_next[i] = NULL;
				if (i!=c) if (group_dir[i]!=NULL) {
						g_free(i,"trimuse");
						ok = TRUE;
				}
		}
		if (!ok) {
				wmsg("db: Nothing to trim %d\n",nm);
				xthis = c;
		} else imsg("db: Trimed directory cache to %d items\n",nm);
}
void m_trimuse_old(int c)
{
		static int xthis = -19;
		int i;
		int ok = FALSE;
		if (c==xthis) return;
		imsg("db: Trimming directory cache from %d items\n",nm);
		for (i=0; i<GROUP_CACHE && nm>m_max_use(); i++) {
				if (group_dir[i]==NULL) continue;
			group_dir_next[i] = NULL;
				if (i!=c) if (group_dir[i]!=NULL) {
						g_free(i,"trimuse");
						ok = TRUE;
				}
		}
		if (!ok) {
				wmsg("db: Nothing to trim %d\n",nm);
				xthis = c;
		} else imsg("db: Trimed directory cache to %d items\n",nm);
}
int m_max_use(void)
{
		static int sz;
		if (sz!=0) return sz;
		sz = atoi(config_get(C_DB_DIR_CACHE));
		if (sz!=0) return sz;
		sz = 20000;
		if (db_total_space() > 1000) sz = 50000;
		if (db_total_space() > 9000) sz = 150000;
		if (db_total_space() > 19000) sz = 300000;
		if (db_total_space() > 59000) sz = 600000;
	if (sz<max_items()) sz = max_items()*2;
		return sz;
}
void m_insert(int c, Mrec *mm)
{
		Mrec *m;
		if (dir_bounds(c)) return;
		if (nm>m_max_use()) m_trimuse(c);
		group_dir_next[c] = NULL;
		if (group_dir[c]==NULL) {
				m = group_dir[c] = mymalloc2(sizeof(Mrec),M_DIR);
				nm++;

				*m = *mm;
				m->next = NULL;
				group_last[c] = m;
		} else {
				m = group_last[c];
				if (m==NULL) {
						for (m=group_dir[c]; m!=NULL; m=m->next) group_last[c] = m;
						m = group_last[c];
				}
				if (mm->num <= m->num) {
						m_insert_mid(c,mm);
				} else {
						m->jump = mm->ipnt; /* correct index links as well*/
						m->next = mymalloc2(sizeof(Mrec),M_DIR); 
						nm++;
						m = m->next;
						*m = *mm;
						m->next = NULL;
						group_last[c] = m;
				}
		}
}		

void m_insert_mid(int c,Mrec *mnew)
{
		Mrec *m,*mm,*last;
		int i,max;
		if (dir_bounds(c)) return;
		mm = mymalloc2(sizeof(Mrec),M_DIR);
	nm++;
		*mm = *mnew;
		m = group_dir[c];
		group_dir_next[c] = NULL;
		last = NULL;
	max = max_items();
		for (i=0;(m!=NULL) && (i<max);m=m->next,i++) {
				if (m->num==mm->num) goto freeit;
				if (m->num>mm->num) break;				
				last = m;
		}
		if (i>=max) {
				emsg("FATAL, insert failed, group too big\n");
				goto freeit;
		}
		if (m==NULL) return;
		mm->next = m;
		if (last!=NULL) last->next = mm;
		else group_dir[c] = mm;
		group_dir_next[c] = NULL;
		return;
freeit: nm--;
		myfree2(mm,M_DB);
}
int xg_group_read(int grp);
static int got_end_ok;
int g_group_read(int grp) 
{
		int r;
	int done_twice = FALSE;
	stat_start(X_GROUPREAD);
again:	got_end_ok = FALSE;
		r = xg_group_read(grp);
	if (!got_end_ok) if (!done_twice) {
		imsg("==Didn't get end so re-reading after closing files\n");
		done_twice = TRUE;
		db_flush2();
		goto again; 
	}
	stat_end(X_GROUPREAD);
		return r;
}
static int basep,baseend;
int xg_group_read(int grp) 
{
		int c = g_cache_lru(TRUE,-1);
		int pnt;
		int xgid = group_gid(grp);
	int xend = group_getend(grp);
	int ndel = 0;
		int xloop = -1;
		int lastpnt=0;
		int ncan,nj,nin;
	int nexp=0;
	int g_first=0;
	int g_last=0;
		int nitems=0, nnext = 0;
		int start = lib_gmtime();
	int done_twice=FALSE;
		char rec[IDXSIZE];
		Mrec *m,mm;
	int max;
	int sort_read=FALSE;
		int isread = is_readonly();
		int isslave = is_slave();

	if (config_true(C_REPLICATE)) sort_read = TRUE;

imsg("Group_read (%s) \n",group_name(grp));

		ncan = nj = nin = 0;
		m = NULL;
		if (dir_bounds(c)) {emsg("group_read bounds %d\n",c); c=0;}
		if (group_dir[c]!=NULL) g_free(c,"group_read");
		group_dir_next[c] = NULL;
		group_grp[c] = grp;
		group_dir[c] = NULL;
		group_last[c] = NULL;
		group_delay[c] = 0;

	if (sort_read) m_insert_sort_start();
		if (!db_openi()) return -1;
		dbf_seek(fdbg,group_gid(grp)*4,SEEK_SET);		
		dbf_read(fdbg,&pnt,sizeof(int));
		if (insrt) {/* Jump into sorted region, remap the jump*/
				pnt = db_jumpin(xgid,pnt,-1);
		}
	max = max_items();
		for (;pnt>=0 && nitems<max;nitems++) {
				nj++;
				if (isread) {
						if (pnt>=fdbi_len) if (fdbi>=0) {
								dbi_close(fdbi);
								fdbi = -1;
								db_openi();
						}
				} 
				if ((pnt>=fdbi_len)) {
			if (get_expire_mode()) { /* close index so we read right to the end */ 
				imsg("read past end of group, this is ok\n");
				break;
			}
/*			{
				imsg("expire: reopen to get last bit of index\n");
				db_master_dbi_close();
					if (!db_openi()) break;
			}
*/
					if ((pnt>=fdbi_len)) {		
							if (!isread) emsg("Read group, seek past end of index %d %d\n",pnt,fdbi_len);
							break;
			}
				}
				if (pnt!=lastpnt+IDXSIZE) nnext++;
				if (!dbi_seek(fdbi,pnt,SEEK_SET)) {filemsg("dbi: Seek failed %d\n",pnt); return c;}
				if (dbi_read(fdbi,rec,IDXSIZE)!=IDXSIZE) {filemsg("IDX read failed %d\n",pnt); return c;}
				if (!frec_to_mrec(rec,pnt,&mm)) {
			emsg("Idx read bad item from index? %s:%d\n",group_name(grp),mm.num);
						break;	
				}

/*imsg("Mrec [%d] fid=%d gid=%d num=%d pnt=%d len=%d jump=%d\n"
						,mm.ipnt,mm.fid,mm.gid,mm.num,mm.pnt,mm.len,mm.jump);*/
				lastpnt = pnt;
				pnt = mm.jump;	
				if (insrt) {/* Jump into sorted region, remap the jump */
						pnt = db_jumpin(mm.gid,mm.jump,lastpnt);
				}
				if (mm.len==0) {
			ndel++;
						continue;
				}
				if (mm.gid!=(unsigned int) xgid) {
						emsg("Index corrupt, item in wrong group %s %d:%d %d, at %d, issrt=%d pnt=%d:%d srtbase=%d\n",group_name(grp),mm.gid,mm.num,xgid,nitems,insrt,pnt,mm.jump,basep);
						break;
				}

		if (xend==mm.num) got_end_ok = TRUE;				
		if(!hist_test_consist) {
			if (db_iscancel(grp,mm.num)){
				if (nitems>1000) if (mm.num+100>xend) {
					imsg("Cancelled item near end of group %s:%d max %d\n",group_name(grp),mm.num,xend);
				}
				ncan++;
				continue;
			}
			/* Should extend to dmulti file already exists containing list of trimmed buckets */
			if (dbn_istrim[mm.fid]) {
				if (nitems>1000) if (mm.num+100>xend) {
					imsg("Expired item near end of group, that is very odd %s:%d of %d\n",group_name(grp),mm.num,xend);
				}
				nexp++; continue;
			}

		}

		if (sort_read) m_insert_sort(c,&mm);
				else m_insert(c,&mm);
		if (g_first==0) g_first = mm.num;
		if (g_last<mm.num) g_last = mm.num;
				nin++;
				if (nitems==90000) imsgf("Wow, over 90,000 items in one group %s, consider using small pile for this group\n",group_name(grp));
				if (xloop>=0) if (xloop==pnt) {emsg("Loop detected at %d %d:%d , you need to do a,  REBUILD_INDEX\n",lastpnt,mm.gid,mm.num); break;}
				if (nitems==30000) xloop = pnt;
		}
	if (sort_read) m_insert_sort_done();
/*imsg("group read nj=%d nin=%d ncan=%d\n",nj,nin,ncan);*/
		if ((lib_gmtime()-start)>1) imsg("db: group_read %s took %d seconds %d items\n",
				group_name(grp),lib_gmtime()-start,nitems);
		dmsg("group_read: %s took %d seconds, %d items, %d jumps, %d cancelled \n",
				group_name(grp),lib_gmtime()-start,nitems,nnext,ncan);

imsg("group_read: %s took %d seconds, %d items, %d jumps, %d cancelled, %d expired, %d ndel \n",
				group_name(grp),lib_gmtime()-start,nitems,nnext,ncan,nexp,ndel);
		
	group_delay[c] = lib_gmtime()-start;
	group_lastused[c] = xtime_now;
	group_nitems[c] = nitems;
		if (nitems>=max_items()) emsg("Too many items in one group, move (%s) into short retention pile, or increase max_items from %d \n",group_name(grp),max_items());
		if (nitems==0) {
				group_grp[c] = -1;
				group_dir[c] = NULL;
			group_dir_next[c] = NULL;
		got_end_ok = TRUE; 
				return -1;
		}
	if (g_first!=g_last) if (nitems>0) { /* If we are a readonly system then fix it up */
		if (is_readonly()) group_fix_byindex(grp,g_first,g_last,nitems);
	}
		return c;
}
Mrec *xdb_item_read(int grp, int num);
Mrec *db_item_read(int grp, int num) 
{
		Mrec *r;
		if (grp<0) return NULL;
	stat_start(X_ITEMREAD);
		r = xdb_item_read(grp,num);
	stat_end(X_ITEMREAD);
		return r;
}
Mrec *xdb_item_read(int grp, int num) 
{
		static Mrec mm,*m;
		char rec[IDXSIZE];
		int base ;
		int lower=group_getfrom(grp);
		int upper=group_getend(grp);
		int xfrom = group_getfrom(grp);
		int middle = xfrom + group_rawrange(grp) - (group_getend(grp) - num) - 1;
		int pnt;
		int gid = group_gid(grp);
		int nseek = 0;

		if (grp<0) return NULL;
		if (middle<lower) middle = (upper+lower)/2;

		if (!db_openi()) return NULL;
		dbf_seek(fdbg,gid*4,SEEK_SET);	
		dbf_read(fdbg,&base,sizeof(int));
		nseek = 2;
		goto start;

		for (;TRUE;) {
				nseek++;
				middle = (upper+lower)/2;
start:;
				if (middle<0) {
						break;
				}
				pnt = base+IDXSIZE*(middle-xfrom);
				if (!is_readonly() && (pnt>fdbi_len)) { dmsg("db: bin Read group, seek past end of index %d %d\n",pnt,fdbi_len); break; }
				if (pnt>fdbi_len) break;
				if (pnt<0) { imsg("bin Read group, seek past beg of index %d %d   %d-%d-%d %d %d\n",pnt,fdbi_len,lower,middle,upper,base,xfrom); break; }
				if (!dbi_seek(fdbi,pnt,SEEK_SET)) {filemsg("dbbin: Seek failed %d\n",pnt); break;}
				if (dbi_read(fdbi,rec,IDXSIZE)!=IDXSIZE) {filemsg("bin: IDX read failed %d\n",pnt); break;}
				if (!frec_to_mrec(rec,pnt,&mm)) {
						dmsg("db: frectomrec failed\n");
						break; 
				}
				m = &mm;
/* imsg("db_bin Read item %d:%d  %d>%d>%d\n",m->gid,m->num,lower,middle,upper);*/
				if (m->gid!=(unsigned int) gid) break; 
				if (m->num==num) {
						if (m->len==0) return NULL;
						dmsg("db: Found Item %d seeks \n",nseek);
						return m;
				}
				if (middle>=upper) {
						break;
				}
				if (m->num>num) upper = middle;
				else if (lower>=middle) {
						break;
				}
				else lower = middle; 
		}		
		return g_get_mrec(grp,num);
}
int g_cache_find(int grp)
{
		int i;
		if (grp<0) return -1;
		for (i=0; i<GROUP_CACHE; i++) {
				if (grp==group_grp[i]) {
						if (group_dir[i]==NULL) group_grp[i] = -1;
				}
				if (grp==group_grp[i]) return i;
		}
		return -1;
}
/* How are we going to iterate thru, to find which ones exist' */
/* or is this ever needed if we build in a coiuple of routines */
/* to avoid this. */
static int db_msz;
void *dbmalloc(int sz)
{
		static void *xxx;
		void *n;
		if (xxx==NULL) {
				db_msz = sz;
				xxx = mymalloc2(sz,M_DBMEM);
				return xxx;
		}
		if (sz<=db_msz) return xxx;
		emsg("Growing dbmalloc area, not serious but does use up extra memory %d->%d\n",db_msz,sz);
		n = mymalloc2(sz,M_DBMEM);
		memcpy(n,xxx,db_msz);
		myfree2(xxx,M_DBMEM);
		xxx = n;
		db_msz = sz;
		return n;
}
void dbfree(void *x)
{
		/* Don't free it, prevents fragmentation */
}
int rechunk(void)
{
		static int chnk;
		if (chnk==0) {
				chnk = atoi(config_get(C_SORT_CHUNK));
				if (chnk==0) {	  
						chnk = (db_total_space()) * 20;
						if (chnk<10000) chnk = 10000;  
						if (chnk>300000) chnk = 300000;
				}
				if (chnk>2000000) chnk =2000000;
				imsg("Index Sort chunk size is %d items\n",chnk);
		}
		return chnk;
}
static Mrec *resort;
static int resort_size;
static int nresort;
int resort_cmp(const void *aa, const void *bb);
int resort_cmp(const void *aa, const void *bb)
{
		Mrec *a = (Mrec *) aa;
		Mrec *b = (Mrec *) bb;
		if (a->gid < b->gid) return -1;
		if (a->gid > b->gid) return 1;
		if (a->num < b->num) return -1;
		if (a->num > b->num) return 1;
		return 0;
}
void resort_clear(void)
{
		nresort = 0;
}
void reindex_free(void);
void reindex_free(void)
{
		if (resort==NULL) return;
		dbfree(resort);
		wmsg("mem: dbmmem freed RESORT\n");
		resort = NULL;
		resort_size = 0;
}
void resort_grow(int x)
{
		resort = dbmalloc(x * sizeof(Mrec));
		wmsg("mem: dbmmem malloced RESORT\n");
		resort_size = x;
}
void readd(Mrec *m)
{
		if (nresort+10 > resort_size) {
				resort_grow(resort_size+10000+resort_size/4);
		}
		resort[nresort++] = *m;
}
int reindex_switch(void);
int reindex_pass(void);
int reindex_start(void);
static int re_inpass;

void db_set_in_resort(void)
{
	inresort = TRUE;
}
void db_index_unpause(void)
{
	inresort = FALSE;
	imsg("expire: Told us to continue writing to the dbi.idx file again\n");
	/* Next two lines should fix index bug in spawn_expire */
	db_clear_cache("index_unpause");
	db_flush2();
	db_write_idx_now();
}
void db_clear_in_resort(void)
{
	inresort = FALSE;
	imsg("db_read_last()\n");
	db_read_last();
	db_write_idx_now();
}
void db_slave_flush_pause(void) {
	slave_flush(TRUE); /* Tell it to close all files and wait*/
	readonly_pause();
}
void db_slave_flush_unpause(void) {
	slave_flush(FALSE); /* Tell it to close all files and wait*/
	readonly_unpause();
}
void db_master_dbi_close(void) {
	if (fdbi>=0) dbi_close(fdbi); 
	fdbi = -1;
}


int db_reindex(void) /* returns -1 when completed*/
{
		static int upto = 0;
		int tot;
		int g=0;
		int ng=0;
		static int incancel;
		int nthis=0;
		int donecheck=FALSE;
		int r;
	static int in_new;

		if (re_inpass) goto doing_pass;

		if (resort==NULL) resort_grow(rechunk()+1000);
		inresort = TRUE;
		tot = 0;

	if (in_new) if (config_true(C_REINDEX_STACK)) {
		if (db_reindex_new(FALSE)) {
			inresort = FALSE;
			upto = 0;
			in_new = FALSE;
			return -1;
		}
		return 0;
	}
		
	
	if ((!incancel) && (upto==0)) {
				wmsg("Processing cancelled items, this could take some time\n");
				inresort = FALSE;
				incancel = TRUE;
				db_cancel_start();
		} 
		if (incancel) {
				if (db_cancel_backgnd()) return 0;
		}
		if (incancel && (upto==0)) {
				incancel = FALSE;
				inresort = TRUE;
				wmsg("Checking index for consistency (nightly check)\n");
				db_check_index();
				imsgf("Index check completed\n");
				if (!reindex_start()) {
						inresort = FALSE;
						return -1;
				}
				db_clear_cache("db_reindex");

		}
	if (config_true(C_REINDEX_STACK)) {
		db_reindex_new(TRUE);
		in_new = TRUE;
		return 0;
	}
		for (g=0; g<group_n(); g++) re_grps[g] = FALSE; 
		imsg("Resort upto group %d/%d\n",upto,group_n());
		for (g=upto;(g<group_n()) && (tot<rechunk());g++) {
				r = group_rawrange(g);									
				tot += r;
				if (tot>5000) if ((tot>(rechunk()-5000)) && (ng>0)) break;
				re_grps[g] = TRUE;
				upto = g+1;
				ng++;
				nthis++;
		}

doing_pass:
		if (!reindex_pass()) {
				emsg("Reindex_pass failed?\n");
				inresort = FALSE;
				upto = 0;
				return -1;
		}
		if (re_inpass) return 0; /* Still sorting it */

		if (upto>=group_n()) {
				wmsg("Finished reindex procedure\n");
				reindex_switch();
				inresort = FALSE;
				if (!get_expire_mode()) db_write_idx_now(); /* Write any buffered index entries*/
				g_free_all("reindex");
				reindex_free();
				db_clear_cache("db_reindex2");
				upto = 0;
				return -1;
		}
		return 0;
}
int max_items(void)
{
	static int x;
	if (x>0) return x;
	x = atoi(config_get(C_MAX_ITEMS));
	if (x>0) return x;
	x = MAX_ITEMS;
	return x;
}
int reindex_pass(void)
{
		char bf[IDXSIZE];
		int n;
		int xstart = lib_gmtime();
		Mrec m;
		int nexp=0;
		int ncorrupt = 0;
		int len;
	int g;
	static int lastprune;
	int maxrange = max_items()-1000;
		static int upto;
		int i;
	int ishistmax = atoi(config_get(C_HIST_MAX));
	if (ishistmax!=0) ishistmax = TRUE;
	if (maxrange>30000) maxrange = maxrange - (maxrange /10);

		if (fdbi<0) db_openi(); if (fdbi<0) return FALSE;
		if (!re_inpass) {
				imsg("reindex_pass starts\n");
				dbi_seek(fdbi,0,SEEK_SET);
				re_inpass = TRUE;
				upto = 0;
		} else {
				dbi_seek(fdbi,upto*IDXSIZE,SEEK_SET);
		}
		if (lib_gmtime()-xstart>1) {
		imsg("reindex_seek took %d seconds\n",lib_gmtime()-xstart);
		xstart = lib_gmtime();
	}
		for (n=upto;(n*IDXSIZE)<fdbi_len;n++) {
				len = dbi_read(fdbi,bf,IDXSIZE);
				if (len==0) break;
				if (len<IDXSIZE) {
						emsg("FATAL, idx is corrupt length, panic, got %d bytes\n",len);
						break;
				}
				if (!frec_to_mrec(bf,n*IDXSIZE,&m)) {
						ncorrupt++;
						continue;
				}
				if (m.len==0) nexp++;
		g = gid_to_group(m.gid);
				if (g<0) imsg("gidtogroup lost %d %d\n",m.gid,gid_to_group(m.gid));
				else if (re_grps[gid_to_group(m.gid)]) if (m.len>0)  {
			if (m.num > group_getend(g)-maxrange) {
				if ((!ishistmax) && (m.num<min_get(g))) {
					imsg("Triming item not in history index %s:%d\n",group_name(g),m.num);
				} else readd(&m);	   
			} else {
				if (g!=lastprune) imsg("Pruning large group %s:%d\n",group_name(g),m.num);
				lastprune = g;
				}	}
				if ((n%5000)==500) if (lib_gmtime()-xstart>expire_quanta()) {
						upto = n;
						re_inpass = TRUE;
						imsg("reindex_reading %d seconds done %d \n",lib_gmtime()-xstart,n);
						return TRUE;
				}
		}
		re_inpass = FALSE;
		if (ncorrupt>0) emsg("Removed %d corrupt lines, a rebuild_index might be needed\n",ncorrupt);
		imsg("reindex_sort %d\n",nresort);
		if (nresort>1) qsort(resort,nresort,sizeof(Mrec),resort_cmp);
		imsg("reindex_write %d records removed %d records\n",nresort,nexp);
		for (i=0; i<nresort; i++) {
				if (last_boundsu(resort[i].gid)) {
						imsg("resort gid funny %d %d \n",i,resort[i].gid);
						continue;
				}
				if (i==0) {
						gid_first[resort[i].gid] = fdbs_len;
						/*	imsg("gid_first[%d] = %x\n",resort[i].gid,fdbs_len); */
				} else {
						if (resort[i].gid!=resort[i-1].gid) {
								gid_first[resort[i].gid] = fdbs_len;
								/* imsg("gid_first[%d] = %x\n",resort[i].gid,fdbs_len); */
						}
				}
				if (i+1>=nresort) goto endgrp;
				if (resort[i+1].gid==resort[i].gid) {
						if (resort[i+1].num==resort[i].num) {
								imsg("db: Removing duplicate item from group gid=%d:%d\n",resort[i].gid,resort[i].num);
								goto skipdup;
						}
						resort[i].jump = fdbs_len+IDXSIZE;
				} else {
endgrp: 				resort[i].jump = -1;
						gid_last[resort[i].gid] = fdbs_len;
						/* imsg("Moved %d to %d\n",resort[i].gid,fdbs_len);*/
				}
				mrec_to_frec(&resort[i],bf);
				m = resort[i];
				if (dbf_write(fdbs,bf,IDXSIZE)<IDXSIZE) {
						filemsg("Failed writing new index file \n");
						return FALSE;
				}
				fdbs_len += IDXSIZE;
skipdup:;
		}
		imsg("reindex_done %d seconds \n",lib_gmtime()-xstart);
		resort_clear();
		g_clear("reindex done");
		return TRUE;
}
int db_write_first(void);

int reindex_switch(void)
{
		int count;
		char fname[BFSZ];
		ncpy(fname,FILE_DBI,BFSZ-1);

	dblog("stack: reindex switching\n");
		if (get_expire_mode()) {
			tell_master("slave_flush_pause");
		}
		else {
			slave_flush(TRUE); /* Tell it to close all files and wait*/
			readonly_pause();
		}
		if (!dbf_close(fdbs)) {
				fdbs = -1;
				filemsg("DBf close failed, index switch failed\n");
				if (get_expire_mode()) {
					tell_master("slave_flush_unpause");
				}
				else {
					slave_flush(FALSE); /* Tell it to start again */
					readonly_unpause();
				}
				return FALSE;
		}
		fdbs = -1;
		imsg("Switching to new index file\n");
		if (fdbi>=0) dbi_close(fdbi); 
		fdbi = -1;
		imsg("Writing new first stuff\n");
		db_write_first();
		m_cache_clear();

		remove(FILE_DBI_OLD);						   
		count = 0;
again:
	dblog("stack: renaming (%s) %d\n",fname,lib_length_file(fname));
		if (rename(fname,FILE_DBI_OLD)!=0) {
				filemsg("Rename to old failed, waiting for slaves...(%s->%s)\n",fname,FILE_DBI_OLD);
				lib_sleep(1);
				count++;
				if (count>930) dnews_ex("Couldn't rename dbi.idx");
				if (count==60) slave_flush(TRUE); /* Tell it to close all files and wait*/
				if (count==160) slave_flush(TRUE); /* Tell it to close all files and wait*/
				if (count==260) slave_flush(TRUE); /* Tell it to close all files and wait*/
				if (count==360) slave_flush(TRUE); /* Tell it to close all files and wait*/
				if (count==460) slave_flush(TRUE); /* Tell it to close all files and wait*/
				if (count==560) slave_flush(TRUE); /* Tell it to close all files and wait*/
				if (count==660) slave_flush(TRUE); /* Tell it to close all files and wait*/
				if (count==760) slave_flush(TRUE); /* Tell it to close all files and wait*/
				goto again;
		}
	if (count>0) imsg("Rename suceeded ok at last, ignore previous error\n");
	dblog("stack: renaming NEW (%s) %d\n",FILE_SORT,lib_length_file(FILE_SORT));
		if (rename(FILE_SORT,fname)!=0) filemsg("Rename sort to current failed\n");
		remove(FILE_DBI_OLD);

		if (get_expire_mode()) {
			tell_master("slave_flush_unpause");
		}
		else {
			slave_flush(FALSE); /* Tell it to start again */
			readonly_unpause();
		} 
	db_bucket_ranges_clear();
		return TRUE;
}
int db_write_first(void)
{
		int i;
		int r;
		if (is_readonly()) return TRUE;
		if (!dbg_open()) return FALSE;
		if (fdbg<0) return FALSE;
		dbf_seek(fdbg,0,SEEK_SET);
		for (i=0; i<xmax_groups; i++) {
				if (dbf_write(fdbg,&gid_first[i],4)<4)	{
						filemsg("Cannot write to %s\n",FILE_FIRST);
						dbf_close(fdbg);
						fdbg = -1;
						return FALSE;
				}
		}
		r = dbf_close(fdbg);
		fdbg = -1;
		if (!r) {
				filemsg("Cannot write to %s\n",FILE_FIRST);
				return FALSE;
		}
		fdbg = -1;
		return TRUE;
}
int db_write_last(void)
{
		int i;
		int f;

		if (is_readonly()) {
				imsg("slave: not writing last file\n");
				return TRUE;
		}
		remove(FILE_LAST);
		f = dbf_open(FILE_LAST,"w+",2,4000);
		if (f<0) {
				filemsg("Could not write %s\n",FILE_LAST);
				return FALSE;
		}
		dbf_seek(f,0,SEEK_SET);
		for (i=0; i<xmax_groups; i++) {
				if (dbf_write(f,&gid_last[i],4)<4) {
						filemsg("Cannot write to %s\n",FILE_LAST);
						dbf_close(f);
						remove(FILE_LAST);
						return FALSE;
				}
		}
		if (!dbf_close(f)) {
				filemsg("Cannot close %s\n",FILE_LAST);
				return FALSE;
		}
		return TRUE;
}
int db_read_last(void)
{
		int i;
		int f;
		int p;
		if (is_readonly()) {
				f = dbf_open(FILE_LAST,"r",2,4000);
		} else {
				f = dbf_open(FILE_LAST,"r+",2,4000);
		}
		if (f<0) {filemsg("Can't read {%s} \n",FILE_LAST); return FALSE;}
		dbf_seek(f,0,SEEK_SET);
		for (i=0; i<xmax_groups; i++) {
				if (dbf_read(f,&p,4)<4) break;
				gid_last[i] = p;
		}
		dbf_close(f);
		return TRUE;
}

int reindex_start(void)
{
		int i;

		if (is_readonly()) {
				emsg("cannot reindex from slave!!!\n");
				return TRUE;
		}
		if (fdbs>=0) {
				emsg("New index already open?\n");
				dbf_close(fdbs);
				fdbs = -1;
		}
		remove(FILE_SORT);
		fdbs = dbf_open(FILE_SORT,"w+",10,2000);

		fdbs_len = 0;
		if (fdbs<0) {
				filemsg("Problem opening %s\n",FILE_SORT);
				return FALSE;
		}
		for (i=0; i<xmax_groups; i++) {
				gid_first[i] = -1;
				gid_last[i] = -1;
		}
/*		for (i=0; i<MAX_DBN; i++) {
				remove(del_name(i));
		}
*/
		return TRUE;
}


typedef struct KILL {int gid; int num; int sz; char islink; short fid; struct KILL *next;} Kill;
static Kill *xkill;
static int nkill;
#define KILL_NHASH 10000
static Kill *kill_hash[KILL_NHASH];
int db_killhash(int g,int n)
{
		return abs((g+n) % KILL_NHASH);
}
int xdb_cancel(int grp, int num);

int db_cancel(int grp, int num)
{
		int r;
	stat_start(X_CANCEL);
		r = xdb_cancel(grp,num);
	stat_end(X_CANCEL);
		return r;
}
void db_del_item(int grp,int num,Mrec *m);
void db_del_item(int grp,int num,Mrec *m)
{
		FILE *f;
		if ((m->fid<0) || (m->fid>MAX_DBN)) {
				emsg("db_del_item, invalid %d:%d %d\n",grp,num,m->fid);
				return;
		}
		f = fopen(del_name(m->fid),"a");
		if (f==NULL) return;
		fprintf(f,"%d %d %d\n",group_gid(grp),num,m->len);
		fclosep(f);
		dbn[m->fid].free += m->len;
}
static FILE *fc;
typedef struct CAN {int grp; int num; struct CAN *next; char done,islink;} Can;
#define CAN_NHASH 5000
static Can *can_h[CAN_NHASH];
static int ncan;
#define can_hash(g,n) ((g+n+2) % CAN_NHASH)
#define FILE_CANCEL file_work("cancel2.dat")
void db_cancel_freedone(void);
void db_cancel_setdone(void);
void db_cancel_add(int grp, int num, int islink);
void db_cancel_fast(int grp, int num, char *from, int islink)
{
		if ((grp<0) || (num<0)) return;
		if (grp>=group_n()) return;
		if (fc==NULL) {
				fc = myfopen(FILE_CANCEL,"a");
				if (fc==NULL) { filemsg("Can't append to (%s)\n",FILE_CANCEL); return; }
		}
		fprintf(fc,"%d %d %d %s\n",group_gid(grp),num,islink,from);
		db_cancel_add(grp,num,islink);
	ex_cancel_add(group_gid(grp),num);
		dmsg("db: Cancel %s:%d\n",group_name(grp),num);
		g_free_item(grp,num); /* remove from group_dir() cache */
		group_addrange(grp,-1); /* Remove from count of items in group */
}
void db_cancel_read(void)
{
		int grp,num,islink,i;
		char bf[BFSZ];
		char *s;
	int gid;

		if (fc!=NULL) fclosep(fc);
		fc = myfopen(FILE_CANCEL,"r");
		if (fc==NULL) return;
		imsg("Reading %s\n",FILE_CANCEL);
		for (i=0;!feof(fc);i++) {
				if (fgets(bf,BFSZ-10,fc)==NULL) break;
				s = strtok(bf," \t\r\n"); if (s==NULL) continue;
				gid = atoi(s);
				s = strtok(NULL," \t\r\n"); if (s==NULL) continue;
				num = atoi(s);
				s = strtok(NULL," \t\r\n"); if (s==NULL) continue;
				islink = atoi(s);
		grp = gid_to_group(gid);
		if (grp>=0) db_cancel_add(grp,num,islink);
		}
		fclosep(fc);
		fc = NULL;
		imsg("Read %d lines from cancel.dat\n",i);
}
/* Remove from head cache */
/* Empty 'db' caches after index rewrite. */
void db_cancel_add(int grp, int num, int islink)
{
		int h = can_hash(grp,num);
		int max = atoi(config_get(C_CANCEL_STORE));
		Can *m;
		static int done;

		if (h<0) h = -h;
		dmsg("dbk: cancel_fast %s:%d %d\n",group_name(grp),num,islink);
		if (max==0) max = 10000; 
		if (ncan>(max*2)) {
				if (!done) emsg("db: Cancel overflow, some cancels ignored\n");
				done = TRUE;
				return; /* Limit how far behind we can get */
		}
		m = mymalloc2(sizeof(*m),M_DB);
		memset(m,0,sizeof(*m));
		m->next = can_h[h];
		can_h[h] = m;
		m->grp = grp;
		m->num = num;
		m->islink = islink;
		m->done = FALSE;
		ncan++;
		xcache_forget(grp,num);
		item_freecache(grp,num);
		if (ncan>max) {
				imsg("Cancel start %d>%d\n",ncan,max);
				db_cancel_start();
		}
}


static int incancel;
static int canupto;
void db_cancel_start(void)
{
/*		if (insrt) return;*/
		if (incancel) return;
		if (inresort) return;
		incancel = TRUE;
		canupto = 0;
}
void slave_dbflush(void);
int db_cancel_backgnd(void) 
{
		Mrec m;
		char bf[IDXSIZE];
		int n;
		int g;
		int t=0;
		int tstart;
		int len;
		int islink;
	int can_release = config_true(C_CANCEL_RELEASE);
		if (!incancel) return FALSE;
		imsg("db: cancel upto %d\n",canupto);

	if (get_expire_mode()) can_release = FALSE;
		if (canupto==0) {
		idx_write_flush("cancel");
				db_cancel_setdone();
				if (fc!=NULL) fclosep(fc);
				fc = NULL;
				remove(FILE_CANCEL);
		}
		dmsg("db: cancel openi %d\n",canupto);
		if (fdbi<0) db_openi();
		tstart = lib_gmtime();
		if (fdbi<0) {
				incancel = FALSE;
				return FALSE;
		}
		dmsg("db: cancel seek %d\n",canupto);
		dbi_seek(fdbi,canupto*IDXSIZE,SEEK_SET);
		imsg("Cancel processing %d buffered cancels, upto %d \n",ncan,canupto);
		for (n=canupto;!dbi_eof(fdbi);n++,t++) {
				if (t>100) {
						t = 0;
						if (lib_gmtime()-tstart>0) {
								canupto = n;
								return TRUE;
						}
			if (can_release) {
				thin_release();
				/* Seek back incase fdbi got changed */
				if (fdbi<0) db_openi();
					dbi_seek(fdbi,n*IDXSIZE,SEEK_SET); 
			}
				}
				len = dbi_read(fdbi,bf,IDXSIZE);
				if (len==0) break;
				if (len<IDXSIZE) break;
				if (!frec_to_mrec(bf,n*IDXSIZE,&m)) continue;
				if (m.len==0) continue;
				g = gid_to_group(m.gid);
				if (db_cancel_find(g,m.num,&islink)) {
						if (!islink) {
								db_del_item(g,m.num,&m);
						}
						m.len = 0;
						m_update(&m);							
						dbi_seek(fdbi,(n+1)*IDXSIZE,SEEK_SET);
				}
				
		}
		db_cancel_freedone();
		canupto = 0;
		incancel = FALSE;
		db_clear_cache("cancel_background");
		slave_dbflush();
		return FALSE;
}

void db_cancel_setdone(void)
{
		int h;
		Can *m;
		for (h=0;h<CAN_NHASH;h++) {
				m = can_h[h];
				for (;m!=NULL;m=m->next) {
						m->done = TRUE;
				}		
		}
}
void db_cancel_freedone(void)
{
		int h;
		Can *m,*n,*r;
		for (h=0;h<CAN_NHASH;h++) {
				m = can_h[h];
				r = NULL;
				for (;m!=NULL;m=n) {			
						n = m->next;
						if (m->done) {
								myfree2(m,M_DB);
								ncan--;
						} else {
								m->next = r;
								r = m;
						}
				}		
				can_h[h] = r;
		}
}
int db_iscancel(int grp, int num)
{
		int islink;
		return db_cancel_find(grp,num,&islink);
}
int db_cancel_find(int grp, int num, int *islink)
{
		int h = can_hash(grp,num);
		Can *m;
		if (h<0) h = -h;
		m = can_h[h];
		for (;m!=NULL;m=m->next) {
				if (m->grp==grp) if (m->num==num) {
						*islink = m->islink;
						return TRUE;
				}
		}
		return FALSE;
}


int db_purge_items(void) 
{
		Mrec m;
		char bf[IDXSIZE];
		int n;
		int g;
		int t=0;
		int tstart;
		int len;

		if (fdbi<0) db_openi();
		tstart = lib_gmtime();
		if (fdbi<0) return FALSE;
		dbi_seek(fdbi,canupto*IDXSIZE,SEEK_SET);
		imsgf("Purge_items processing starts\n");
		for (n=0;!dbi_eof(fdbi);n++) {
				if (n%10000==0) imsgf("Upto %d \n",n);
				len = dbi_read(fdbi,bf,IDXSIZE);
				if (len==0) break;
				if (len<IDXSIZE) break;
				if (!frec_to_mrec(bf,n*IDXSIZE,&m)) continue;
				if (m.len==0) continue;
				g = gid_to_group(m.gid);
				if (g>=0) if (group_isdeleted(g)) {
						db_del_item(g,m.num,&m);
						m.len = 0;
						m_update(&m);							
						dbi_seek(fdbi,(n+1)*IDXSIZE,SEEK_SET);
				}
				
		}
		return FALSE;
}
void debug_index_show(char *gname)
{
		Mrec m;
		char bf[IDXSIZE];
		int n;
		int g;
		int t=0;
		int len;
		int r;


		if (fdbi<0) db_openi();
		if (fdbi<0) return ;
		dbi_seek(fdbi,0,SEEK_SET);
		g = group_find(gname);
		imsgf("debug_index_show for %s %d\n",gname,g);
		for (n=0;!dbi_eof(fdbi);n++) {
				len = dbi_read(fdbi,bf,IDXSIZE);
				if (len==0) break;
				if (len<IDXSIZE) break;
				r = frec_to_mrec(bf,n*IDXSIZE,&m);
				if (g!=gid_to_group(m.gid)) continue;
				imsg("[%d] %s:%d ok=%d len=%d pnt=%d jump=%d\n",n*IDXSIZE,gname
						,m.num,r,m.len,m.pnt,m.jump);
		}
		return ;
}



int xdb_cancel(int grp, int num)
{
		Mrec *m;
		m = db_item_read(grp,num);
		if (m==NULL) { /* Lost item, add it back in*/ 
				return FALSE;
		} else {
				db_del_item(grp,num,m);
				m->len = 0;
				m_update_del(m);
				dmsg("db: Removing item %s:%d\n",group_name(grp),num);
				/*
				Length is set to zero, so it shouldn't re-appear anyway.
				c = g_cache_find(grp);
				if (c>=0) g_free_one(c,);*/
				m = m_cache_find(group_gid(grp),num);
				if (m!=NULL) m->len = 0;
		}				   
		return TRUE;
}
void db_kill_free(void)
{
		if (xkill == NULL) return;
		dbfree(xkill);
		wmsg("mem: dbmmem freed KILL\n");
		xkill = NULL;
		nkill = 0;
}
int db_kill(int g, int n, int islink) /* Change expire so it gets called with each link */
{
		int h;
		dmsg("dbk: %s:%d %d\n",group_name(g),n,islink);
		if (xkill==NULL) {
				imsg("kill_max is set to %d history records \n",kill_max());
				xkill = dbmalloc(kill_max()*sizeof(Kill));
				wmsg("mem: dbmmem malloced KILL\n");

		}

		if (nkill>=kill_max()) {emsg("fatal: item kill exceeded, bug in code \n"); return FALSE;}
		xkill[nkill].gid = group_gid(g);
		xkill[nkill].num = n;
		xkill[nkill].islink = islink;
		xkill[nkill].next = NULL;
		xkill[nkill].fid = -1;
		xkill[nkill].sz = 0;
		h = db_killhash(xkill[nkill].gid,xkill[nkill].num);
		xkill[nkill].next = kill_hash[h];
		kill_hash[h] = &xkill[nkill];
		nkill++;		
		return TRUE;
}
int db_iskilled(int gid, int num, int fid, int sz, int *islink);
int db_iskilled(int gid, int num, int fid, int sz, int *islink)
{
		int z;
		int h = db_killhash(gid,num);
		Kill *k;
		k = kill_hash[h]; 
		*islink = FALSE;
		for (z=0;k!=NULL;k = k->next,z++) {
				if (z>9000000) {emsg("loop detected %d %d \n",gid,num); dnews_ex("db_iskilled");}

				if (k->gid==gid) if (k->num==num) {
						k->fid = fid;
						k->sz = sz;
						*islink = k->islink;
						return TRUE;			
				}
		}
		return FALSE;
}
/*
		
*/
enum {P_INDEX,P_FID_SORT,P_FID,P_PURGE};
int idx_dokill(void);
int kill_cmp(const void *aa, const void *bb);
int kill_cmp(const void *aa, const void *bb)
{
		Kill *a = (Kill *) aa;
		Kill *b = (Kill *) bb;
		if (a->fid < b->fid) return -1;
		if (a->fid > b->fid) return 1;
		return 0;
}

int db_doexpire(void) /* Called until it returns -1, must deal with 100K deleted items */
{
		static int upto = -1;
		static int x;
		int i;
		int r;
		int fid = -1;
		static FILE *f = NULL;
		int start = lib_gmtime();
		static int indokill;
		if (is_readonly()) {
				emsg("expire does nothing from slave\n");
				return -1;
		}


		vtrace();
		if (upto<0) upto = P_INDEX;
		switch (upto) {
		  case P_INDEX:
				vtrace();
				if (!indokill) {
						imsg("expire: dokill\n");
						scan_free_all();

						for (i=0; i<MAX_DBN; i++) dbn_expired[i] = FALSE;
				}				
				ex_start("IDX Kill");
				indokill = TRUE;
				if (idx_dokill()) {
						upto = P_FID_SORT;
						indokill = FALSE;
				}
				ex_end("IDX Kill");
				x = 0;
				break; 
		  case P_FID_SORT: /* Sort into FID order */
				vtrace();
				imsg("expire: sort fid \n");
				hist_check("sort");
				upto = P_FID;
				ex_start("Sort");
				qsort(xkill,nkill,sizeof(Kill),kill_cmp);
				ex_end("Sort");
		  case P_FID: /* Write all the .del files */
				vtrace();
				imsg("expire: write .del files \n");
				dmsg("db: P_FID\n");
				ex_start("Write Del");
				for (;(x<nkill) && (lib_gmtime()-start==0);x++) {
						if (xkill[x].fid<0) {
								dmsg("dbk: Can't find items FID %d:%d (probably already cancel'd)\n",xkill[x].gid,xkill[x].num);
								continue;
						}
						if (xkill[x].islink) continue;
						if (fid!=xkill[x].fid) {
								if (f!=NULL) fclosep(f);
								f = myfopen(del_name(xkill[x].fid),"a");
						}
						if (f==NULL) {
								filemsg("Fatal: couldn't append to {%s} \n",del_name(xkill[x].fid));
								expire_failed("Writing *.del files");
								return -1;
						}
						fprintf(f,"%d %d %d\n",xkill[x].gid,xkill[x].num,xkill[x].sz);
				}		
				ex_end("Write Del");
				if (x>=nkill) upto = P_PURGE;
				if (f!=NULL) fclosep(f); f = NULL;
				imsg("expire: purge db files\n");
				for (i=0; i<MAX_DBN; i++) dbn_purged[i] = FALSE;
				for (i=0; i<MAX_DBN; i++) dbn_pending[i] = FALSE;
				break; 
		  case P_PURGE:
				vtrace();
				dmsg("db: P_PURGE\n");
				for (r = 0;(r==0) && ((lib_gmtime()-start)<=expire_quanta());) {
						ex_start("Purge");
						r = db_purge();
						ex_end("Purge");
				}
				if (r<0) { /* Finished reset everything we can */
						vtrace();
						upto = -1; 
						nkill = 0;
						for (i=0; i<KILL_NHASH; i++) {
								kill_hash[i] = NULL;
						}
						db_kill_free();
						g_free_all("expire");
						hist_check("finished db purge");
				}
				break; 
		}
		vtrace();
		dmsg("db: doexpire_exit state=%d\n",upto);
		return upto;
}
int idx_dokill(void)
{
/*		Process entire index file, setting all deleted items as such.
		store FID for each item as we read/write them */
		int n;
		int len;
		int islink;
		Mrec m;
		char bf[IDXSIZE];
		static int upto;
		int start = lib_gmtime();


		imsgf("idx_dokill processing upto %d/%d\n",upto,fdbi_len/IDXSIZE);
		if (fdbi<0) db_openi(); if (fdbi<0) return TRUE;
		dbi_seek(fdbi,upto*IDXSIZE,SEEK_SET);
		for (n=upto;(n*IDXSIZE)<fdbi_len;n++) {
				if ((n%3000)==2000) if ((lib_gmtime()-start)>expire_quanta()) {
						upto = n;
						return FALSE;
				}
				len = dbi_read(fdbi,bf,IDXSIZE);
				if (len==0) break;
				if (len<IDXSIZE) {
						emsg("FATAL, idx is corrupt length, panic, got %d bytes\n",len);
						break;
				}
				if (!frec_to_mrec(bf,n*IDXSIZE,&m)) {
						continue;
				}
				if (dbn_bounds(m.fid)) {
						emsg("idx_dokill: Corrupt %s:%d m.fid=%d\n",group_name(m.gid),m.num,m.fid);
						continue;
				}
				if (db_iskilled(m.gid,m.num,m.fid,m.len,&islink)) {
						dmsg("dbk: Kill Index Entry %s:%d %d\n",group_name(gid_to_group(m.gid)),m.num,islink);
						if (!islink) dbn[m.fid].free += m.len;
						dbi_seek(fdbi,IDXSIZE*n,SEEK_SET);
						m.len = 0;
						mrec_to_frec(&m,bf);
						if (dbi_write(fdbi,bf,IDXSIZE)!=IDXSIZE) filemsg("Fatal error writing IDX file, rebuild_index will be needed\n");				
						dbi_seek(fdbi,IDXSIZE*(n+1),SEEK_SET);
				} 
		}
		imsgf("Finished idx_dokill\n");
		upto = 0;
		return TRUE;
}
int db_convert_item(String *xitem, int num);
static int isold;
void db_convert(int grp, int upto)
{
		int f;
		int gid,num;
		String *xitem=NULL;
		int fi,fg;
		int tot=0;
		int nfail=0;
		cvt_numfile2(upto,&fg, &fi);
		f = item_open_read(sys_gname(grp,fg));
		if (f<0) {
				dmsg("db: Could not read {%s}\n",sys_gname(grp,fg));
				remove(sys_iname(grp,fg));
				return;
		}
		for (;TRUE;) {
				isold = TRUE;
				xitem = db_scanitem(fi,f,&gid,&num);
				isold = FALSE;
				if (xitem==NULL) break;
				db_trimold(xitem);	/*Remove header info and tail ^ */
				if (!db_convert_item(xitem,num)) nfail++;
				else {
						db_fix_hist(xitem,gid,num,0,0);
				}
				tot++;
		}
		dbf_close(f);
		remove(sys_iname(grp,fg));
		if ((tot==0) || (nfail>0)) {
				imsg("Converted %d/%d items, leaving %s\n",tot-nfail,tot,sys_iname(grp,fg));
		} else {
				imsg("Converted %d/%d items, removing %s\n",tot-nfail,tot,sys_iname(grp,fg));
				remove(sys_gname(grp,fg));
		}
}
extern	int scan_missing;
static int scan_fix;
void db_convert_file(char *fname)
{
		int f;
		int gid,num;
		String *xitem=NULL;
		int tot=0;
		int nfail=0;

		if (strstr(fname,".idx")!=NULL) {
				imsg("Removing index file %s\n",fname);
				remove(fname);
				return;
		}
		f = item_open_read(fname);
		if (f<0) {
				dmsg("db: Could not read {%s}\n",fname);
				return;
		}
		for (;TRUE;) {
				scan_fix = TRUE;
				xitem = db_scanitem(-1,f,&gid,&num);
				scan_fix = FALSE;
				if (xitem==NULL) break;
				db_trimold(xitem);	/*Remove header info and tail ^ */
				if (scan_missing) {
						if (!item_suck(xitem)) nfail++;
				} else {
						if (!db_convert_item(xitem,num)) nfail++;
				}
				tot++;
		}
		dbf_close(f);
		if ((tot==0) || (nfail>0)) {
				imsg("dbConverted %d/%d items leaving %s\n",tot-nfail,tot,fname);
		} else {
				imsg("dbConverted %d/%d items, removing %s\n",tot-nfail,tot,fname);
				remove(fname);
		}
}
int db_convert_item(String *xitem, int num)
{
		char xref[BIGBF];
		int i;
		char bf[BFSZ];
		int grp = -1;
		char *s;
		char group[BFSZ];
		int g;
		char newsgroups[BFSZ];
		char *xpoint[MAX_GRPS];
		int nx;
		char xcontrol[BFSZ],xsubject[BFSZ];

		ncpy(newsgroups, get_aheadraw(xitem,"Newsgroups:"),BFSZ-1);
		ncpy(xref, get_ahead(xitem,"Xref:"),BIGBF-1);
		ncpy(xcontrol, get_ahead(xitem,"Control:"),BFSZ-1);
		ncpy(xsubject, get_ahead(xitem,"Subject:"),BFSZ-1);

		if (strlen(xcontrol)==0) if (strncmp(xsubject,"cmsg ",5)==0) {
				ncpy(xcontrol,xsubject+5,BFSZ-1);
		} 

		nx = token_split(newsgroups,xpoint,", \t\015\012",MAX_GRPS-3);
		if (nx>MAX_GRPS) nx=MAX_GRPS-3;

		if (strlen(xcontrol)>0) {
				strlwr(xcontrol);
				g = -1;
				if (strstr(xcontrol,"cancel")!=NULL) 
						g = group_find("control.cancel");
				if (g<0) g = group_find("control");
				if (g<0) return TRUE;
				if (!db_add(g,num,xitem)) return FALSE;
				return TRUE;
		}		
		for (i=0; i<nx; i++) {
				g = group_find(xpoint[i]);
				if (g<0) continue;
				if (group_isdeleted(g)) continue;
				if (!group_iscached2(g)) if (config_true(C_CACHEDONLY)) continue;
				grp = g; /* Found primary group */
				break;							   
		}
		if (grp<0) {
				ncpy(bf,str_s(xitem),150);
				emsg("convert: Can't find groups this item is in %s\n",bf);
				return FALSE;
		}
		dmsg("db: Add item %s:%d\n",group_name(grp),num); 
		if (!db_add(grp,num,xitem)) return FALSE;
		nx = token_split(xref,xpoint,", \t\015\012",MAX_GRPS-3);
		for (i=2; i<nx; i++) {
				ncpy(bf,xpoint[i],BFSZ-1);
				s = strtok(bf,":"); if (s==NULL) break;
				ncpy(group,s,BFSZ-1);
				s = strtok(NULL,":"); if (s==NULL) break;
				num = atoi(s);
				grp = group_find(group); 
				dmsg("db: Add item %s:%d link\n",group,num);

				db_add_link(grp,num);
		}
		return TRUE;
}
static int ref_control;
int db_find_ref(String *xitem,int gid, int num,int grps[],int nums[], int isrebuild)
{
		char xref[BIGBF];
		int i;
		char bf[BFSZ];
		char *s;
		char group[BFSZ];
		int g;
		char newsgroups[BFSZ];
		char *xpoint[MAX_GRPS];
		char xcontrol[BFSZ];
		int nx;
		int n=0;
										
		ncpy(xcontrol, get_aheadraw(xitem,"Control:"),BFSZ-1);
		ref_control = FALSE;
		if (strlen(xcontrol)>0) ref_control = TRUE;
		ncpy(newsgroups, get_aheadraw(xitem,"Newsgroups:"),BFSZ-1);
		ncpy(xref, get_ahead(xitem,"Xref:"),BIGBF-1);
		nx = token_split(newsgroups,xpoint,", \t\015\012",MAX_GRPS-3);
		
		if (gid>=0) {
		  if (gid_to_group(gid)>=0) {
				grps[0] = gid_to_group(gid);
				nums[0] = num;
				n = 1;

				for (i=0; i<nx; i++) {
						if (strcmpnc(group_name(grps[0]),xpoint[i])==0) goto foundit;
				}
				if (strncmp(group_name(grps[0]),"control",7)==0) {
						n = 1;
						goto foundit;
				}
				emsg("Gid/group deleted? gid=%d %s\n",gid,group_name(grps[0]));
				n = 0;
				goto foundit;
		  } else {
				emsg("Gid/group deleted? gid=%d\n",gid);
				n = 0; goto foundit;
		  }
		} else {
		 for (i=0; i<nx; i++) {
				g = group_find(xpoint[i]);
				if (g<0) continue;
				if (group_isdeleted(g)) continue;
				if (!isrebuild) if (!group_iscached2(g)) if (config_true(C_CACHEDONLY)) continue;
				grps[n] = g;
				nums[n++] = num;
				break;							   
		 }
		}
foundit:;
		if (n==0) {
				ncpy(bf,str_s(xitem),150);
				imsg("Can't find groups this item is in %s\n",bf);
				return 0;
		}
		if (grps[0]<0) {
				ncpy(bf,str_s(xitem),150);
				imsg("Can't find gid-grp this item is in %s\n",bf);
				return 0;
		}
		nx = token_split(xref,xpoint,", \t\015\012",MAX_GRPS-3);
		for (i=1; (i<nx) && (n<98); i++) {
				if (i==1) n = 0; /* If xref exists, base decision on that*/
				ncpy(bf,xpoint[i],BFSZ-1);
				s = strtok(bf,":"); if (s==NULL) break;
				ncpy(group,s,BFSZ-1);
				s = strtok(NULL,":"); if (s==NULL) break;
				num = atoi(s);
				grps[n] = group_find(group); 
				if (grps[n] >=0) nums[n++] = num;
		}
		return n;
}
int db_trimold(String *xitem)
{
		char *s;
		int newlen;
		s = str_s(xitem);
		if (s==NULL) return FALSE;
		s = strchr(s,':'); /* End of len */
		if (s==NULL) return FALSE;
		s = strchr(s+1,':'); /* End of num */
		if (s==NULL) return FALSE;
		if (isdigit(s[1])) {
		 s = strchr(s+1,':'); /* End of gid */
		 if (s==NULL) return FALSE;
		}
		s++;
		newlen = str_len(xitem)-(s-str_s(xitem)) - 1;
		memmove(str_s(xitem),s,newlen); 
		str_setlen(xitem,newlen);
		return TRUE;
}
void db_open_purge(void)
{
		int i;
		if ((fdbp_len>item_min) || (fdbp<0)) { /* Switch to next file */
				imsg("Purge switch to next file, was %d \n",fdbp_fid);
				if (fdbp>=0) dbf_close(fdbp);
				fdbp = -1;
				if ((fdbp_fid>=0) && (fdbp_fid<MAX_DBN)) {
						i = fdbp_fid;
						fdbp_len = dbn[fdbp_fid].used;
						if (fdbp_len < item_min) goto addon;
				}
				i = db_next_fid_new();
				fdbp_fid = i;
				fdbp_len = 0;
				remove(del_name(i));
				remove(fid_name(i));
		dblog("Removed during open_purge %s\n",fid_name(i));
addon:			fdbp = item_open_append(fid_name(i));
				dmsg("db: fid_open append %s %d\n",fid_name(i),i);

				if (fdbp<0) {filemsg("Could not append to {%s} \n",fid_name(i)); return ;}
				dbf_seek(fdbp,0,SEEK_END);
				fdbp_len = dbf_tell(fdbp);
		}
}
void ex_stat_add(char *name, int n);
static int in_purge_one;
int db_purge(void) /* Purge files, call until 1 second elapsed */
{
		/* use .del files to know which ones to delete
		 use xref to find non deleted items and rewrite index */
		int i;
		int minuse;
		double total_used = 0;
		int best,best_i;
		double total_free = 0;
		double pcent;
		static int patch_part;

		if (patch_part) {
				if (idx_dopatch_part()) {
						patch_part = FALSE;
						return -1; /* finished */
				}
				return 0; /* Still patching index file */
		}
		db_open_purge();
		if (fdbp<0) {
				imsg("Purge couldn't write new file, probably no disk space left\n");
				expire_failed("Writing new bucket file");
				return -1;
		}
		if (in_purge_one) {
				ex_start("Purge One");
				purge_one(-1); 
				ex_end("Purge One");
				return 0;
		}
		best_i = -1;
		best = 0;

		if (fdbp_fid>0) dbn_purged[fdbp_fid] = TRUE;
		if (fdbn_fid>0) dbn_purged[fdbn_fid] = TRUE;
		if (db_ispile()) {
				for (i=0; i<get_sort_piles_nf(); i++) {
						if (out_fid[i]>=0) dbn_purged[out_fid[i]] = TRUE;
				}
		}
		imsg("purge search\n");
		for (i=0; i<MAX_DBN; i++) if (i!=fdbp_fid) if (i!=fdbn_fid) if (dbn[i].inuse) {
				if (dbn_purged[i]) continue;	
				dmsg("db: purge[%d] used=%d free=%d\n",i,dbn[i].used,dbn[i].free); 
				if (dbn_expired[i]) continue;
				if (dbn[i].used>0) if (dbn[i].free==dbn[i].used) {
						dbn_purged[i] = TRUE;
						imsg("Removing empty bucket %s %d/%d\n",fid_name(i),dbn[i].free,dbn[i].used);
						ex_stat_add("BucketsDeleted",1);

				/*		slave_flush(TRUE);	 Tell it to stop */
						if (fdbr>=0) if (fdbr_fid==i) {
								dbf_close(fdbr);
								fdbr = -1;
						}
						remove(del_name(i));
						dbn[i].inuse = FALSE;
			dblog("Removed db_purge %s\n",fid_name(i));
						if (remove(fid_name(i))!=0) { /*ISCLOSED*/
			  if (errno!=ENOENT) {
								filemsg("Remove {%s} failed\n",fid_name(i));
								remove_later(i);
								dbn_pending[i] = TRUE;
								dbn[i].inuse = TRUE;
			  }
						} 
						dbn_pending[i] = FALSE;
						diskuse_add(-dbn[i].used); 
						dbn[i].used = 0;
						dbn[i].free = 0;
				/*		slave_flush(FALSE);  Tell it to start again */
						return 0;
				}
				if (best< dbn[i].free) {
						best = dbn[i].free;
						best_i = i;
				}
				total_used += dbn[i].used;
				total_free += dbn[i].free;
		}
		if (total_used==0) pcent = 0; 
		else pcent = total_free * 100.0 / total_used;
		minuse = atoi(config_get(C_EXPIRE_MIN)); 
		if (minuse==0) minuse = 10;
		if (lib_pused()>90) {
				minuse = minuse/2;
		}
		if ((pcent <= minuse) || (best_i<0)) {
				if (best_i < 0) imsg("expire: Can't find any more db files to delete\n");
				imsg("db: Slack down to %g%% which should be less than expire_min %d%%\n",pcent,minuse);
				dbf_close(fdbp);
				fdbp = -1;
				patch_part = TRUE;
				return	0;	/* Finished, nothing else worth deleting*/
		}
		i = best_i;
		dbn_expired[i] = TRUE;
		imsg("dbk: Purging file %d, used %d/%d free=%d\n",i,dbn[i].used-dbn[i].free,dbn[i].used,dbn[i].free);
		ex_start("Purge One");
		dbn_purged[i] = TRUE;
		dbn_pending[i] = TRUE;
		ex_stat_add("BucketsPurged",1);
		if (config_true(C_PURGE_SEEK)) {
				purge_scan(i);
		} else {
				purge_one(i);
		}
		ex_end("Purge One");
		return 0;
}
void db_purge_test(int fi)
{		
/*	if (expire_ispile()) {
		db_rebuild_bucket(fi);
		return;
	}
*/
		db_open_purge();
		purge_one(fi);
		for (;in_purge_one;) {
				purge_one(-1); 
		}
		dbf_close(fdbp);
		fdbp = -1;
		idx_dopatch();
}
static int ndel;
static int *del_gid;
static int *del_num;
static int *del_sz;
static int *del_next;
static int max_del;
#define DEL_NHASH 1000
static int del_hashp[DEL_NHASH];
void del_init(int max)
{
		int bsz;
		if (max_del>0) return;
		max_del = max;
		if (max_del==0) {
				max_del = 15000;
				bsz = atoi(config_get(C_BUCKET_SIZE));
				if (bsz>10000000) {
						max_del = bsz/1000;
						imsg("Items in a bucket (bucket_items) set to %d\n",max_del);
				}
		}

		del_gid = (int *) mymalloc2(sizeof(int) * max_del,M_DB);
		del_num = (int *) mymalloc2(sizeof(int) * max_del,M_DB);
		del_sz = (int *) mymalloc2(sizeof(int) * max_del,M_DB);
		del_next = (int *) mymalloc2(sizeof(int) * max_del,M_DB);

}

int del_hash(int gid, int num)
{
		int h = gid + num;
		h = h % DEL_NHASH;
		if (h<0) h = -h;
		return h;
}
int del_read(int fi, int important)
{
		FILE *f;
		int h;
		char *s;
		char bf[BFSZ];
		int z;
		int i;

		for (i=0; i<DEL_NHASH; i++) del_hashp[i] = -1;
		ndel = 0;
		f = fopen(del_name(fi),"r");
		if (f==NULL) {return FALSE;}
		for (;!feof(f);) {
				if (fgets(bf,BFSZ-10,f)==NULL) break;
				s = strtok(bf," \n"); if (s==NULL) continue;
				del_gid[ndel] = atoi(s);
				s = strtok(NULL," \n"); if (s==NULL) continue;
				del_sz[ndel] = 0;
				del_num[ndel++] = atoi(s);
				z = ndel-1;
				h = del_hash(del_gid[z],del_num[z]);
				del_next[z] = del_hashp[h];
				del_hashp[h] = z;

/*imsg("del_read %s:%d\n",group_name(gid_to_group(del_gid[z])),del_num[z]);*/
				if (ndel>=max_del) {
						if (important) emsg("FATAL: Too many items in one file %d, increase bucket_items in dnews.conf to %d\n",fi,ndel*2);
						break;
				}
				s = strtok(NULL," \n"); if (s==NULL) continue;
				del_sz[ndel-1] = atoi(s);
		}
		fclosep(f);
		return TRUE;
}
int del_this_old(int gid, int num)		/* Not super efficient, should use upto counter */
{
		int i;
		for (i=0; i<ndel; i++) {
				if ((gid==del_gid[i]) && (num==del_num[i])) return TRUE;
		}
		return FALSE;
}
int del_this(int gid, int num)	/* Not super efficient, should use upto counter */
{
		int i;
		int h;
		int j=0;
		h = del_hash(gid,num);

		for (i=del_hashp[h]; i>=0; i = del_next[i]) {
				j++;
				if (j>1000000) {
						emsg("Loop in del_this() %d:%d\n",gid,num);
						dnews_ex("loop in del_this ");
				}
				if ((gid==del_gid[i]) && (num==del_num[i])) return TRUE;
		}
		return FALSE;
}
static int nprec;
static int patch_fi[200];
static int patch_nfi;

typedef struct {int id; int free;} Pairs;
static Pairs bktfree[MAX_DBN]; 
static int bkt_left[MAX_DBN];
static Mrec *bkt_m[MAX_DBN];
int cmp_pairs(const void *aa, const void *bb)
{
		const Pairs *a,*b;
		a = (const Pairs *) aa;
		b = (const Pairs *) bb;
		return b->free - a->free;
}
int find_empty_scan(void);
int find_empty(int xbkt)
{
		/* Find the buckets which are empties, include xbkt add up 'x' mbs of them */
		int i,j;
		int wanted=100000000; /* 100mb */

		for (i=0; i<MAX_DBN; i++) {
				bktfree[i].free = 0;
				bktfree[i].id = i;
				if (i!=fdbp_fid) if (i!=fdbn_fid) if (dbn[i].inuse) {
						if (dbn[i].free==dbn[i].used) continue;
						if (dbn_purged[i]) continue;	
						if (dbn_expired[i]) continue;
						bktfree[i].free = dbn[i].free;
						bktfree[i].id = i;
				}
		}
		qsort(bktfree,MAX_DBN,sizeof(Pairs),cmp_pairs);
		for (i=0; i<MAX_DBN ; i++) {
				bkt_left[i] = FALSE; 
		}
		bkt_left[xbkt] = TRUE;
		for (i=0; (i<200) && (wanted>0); i++) {
				j = bktfree[i].id;
				bkt_left[j] = TRUE;
				wanted -= (dbn[j].used - dbn[j].free);
				if (i<5) imsg("Found next empty bucket is [%d] %d %d\n",j,bktfree[i].free,dbn[j].free);
		}
		find_empty_scan();
		return TRUE;
}
int free_scan_list(int fi);
int scan_free_all(void)
{
		int i;
		for (i=0; i<MAX_DBN; i++) {
				free_scan_list(i);
		}
		return TRUE;
}
int free_scan_list(int fi)
{
		Mrec *m,*mnext;

		for (m = bkt_m[fi]; m!=NULL; m = mnext) {
				mnext = m->next;
				myfree2(m,M_DB);
		}
		bkt_m[fi] = NULL;
		return TRUE;
}
int find_empty_scan(void)
{
		int islink;
		Mrec m;
		Mrec *mp;
		char bf[IDXSIZE];
		int n;
		int len;

		if (fdbi<0) db_openi();
		if (fdbi<0) return TRUE ;
		ex_start("EmptyScan");
		dbi_seek(fdbi,0,SEEK_SET);
		for (n=0;!dbi_eof(fdbi);n++) {
				len = dbi_read(fdbi,bf,IDXSIZE);
				if (len<IDXSIZE) break;
				if (!frec_to_mrec(bf,n*IDXSIZE,&m)) {
						continue;
				}
				if (m.fid>=0) {
						if (bkt_left[m.fid]) {
						  if (!db_iskilled(m.gid,m.num,m.fid,m.len,&islink)) {
								if (islink) continue;
								if (m.len<=0) continue;
dmsg("dbk: Not killed %s:%d %d\n",group_name(gid_to_group(m.gid)),m.num,islink);

								mp = mymalloc2(sizeof(Mrec),M_DB);
								*mp = m;
								mp->next = bkt_m[m.fid];
								bkt_m[m.fid] = mp;
						  }
						}
				}
		}
		ex_end("EmptyScan");
		return TRUE;
}
int cmp_mrec(const void *aa, const void *bb)
{
		Mrec *a = (Mrec *) aa;
		Mrec *b = (Mrec *) bb;
		return a->pnt - b->pnt;
}
void db_purge_scan_test(int fi)
{
		db_open_purge();
		if (fdbp<0) {
				imsg("Purge couldn't write new file, probably no disk space left\n");
				expire_failed("Writing new bucket file");
				return ;
		}

		purge_scan(fi);
		dbf_close(fdbp);
		fdbp = -1;
		idx_dopatch();
}
void purge_scan(int fi)    /* Should break up into 2 second chunks */
{
		int nmv=0;
		int ni=0;
		int lastpnt= -1;
		int i;
		int f;
		int xstart=lib_gmtime();
		Mrec *m,*ma;
		int n,num,gid;
		String *xitem;
		
		if (bkt_m[fi]==NULL) find_empty(fi);
		/* Sort the items */
		for (i=0, m = bkt_m[fi]; m!=NULL; m = m->next, i++) ;
		n = i;
		ma = mymalloc2(sizeof(Mrec) * (n+1),M_DB);
		for (i=0, m = bkt_m[fi]; m!=NULL; m = m->next, i++) {
				ma[i] = *m; 	
		}
		imsg("purge_scan bucket %d\n",fi);
		qsort(ma,n,sizeof(Mrec),cmp_mrec);
		f = item_open_read(fid_name(fi));
		if (f<0) {
				filemsg("Couldn't open (%s) for read\n",fid_name(fi));
				return;
		}
		for (i=0,ni=0; i<n; i++) { 
				if (ma[i].pnt==lastpnt) continue;
				lastpnt = ma[i].pnt;
				dbf_seek(f,ma[i].pnt,SEEK_SET);
				xitem = db_scanitem(fi,f,&gid,&num);
				if (xitem==NULL) break;
				if (gid<0) { imsg("Scan Item GID -ve? [fid=%d] num=%d\n",fi,num); continue; }
				if (gid_to_group(gid) == -1) {
						imsg("Removing item from deleted group %d:%d\n",gid,num);
						goto skipit;
				} else dmsg("dbk: seek: Unpurge [%d] %s:%d \n",fi,group_name(gid_to_group(gid)),num);					
				ni++;
				nmv++;
				imsg("dbk: Unpurge [%d] %s:%d \n",fi,group_name(gid_to_group(gid)),num);						
				db_unpurge(gid,num,xitem);				
skipit:;
		}
		myfree2(ma,M_DB);
		free_scan_list(fi); 	
		ex_stat_add("ItemsMoved",nmv);
		imsg("PurgeScan-done Seconds %d, items %d\n",lib_gmtime()-xstart,ni);
		dbf_close(f);
		diskuse_add(-dbn[fi].used); 
		patch_add(fi);
		in_purge_one = FALSE;
		dbn[fi].free = 0;
		dbn[fi].used = 1;
}
void purge_one(int fi)	   /* Should break up into 2 second chunks */
{
		int f;
		int ni;
		int gid,num;
		int nmv;
		static int in_patch;
		static int upto,upto_fi;
		int xstart=lib_gmtime();
		String *xitem=NULL;
		
		if (in_patch) {
				if (idx_dopatch_part()) {
						in_patch = FALSE;
				}
				return;
		}
		if (fi<0) goto purge_continue; /* In purge operation still */
		del_read(fi,TRUE); /* read .del file into memory */
		
/* How can this fail, it only writes to free ones, so it can't be purging */
		if (dbusy(fi)) return;
		if (fdbn_fid==fi) return; /* Never delete or purge current one */
		if (fdbp_fid==fi) return; /* Also, never do the one we are expiring */
		upto_fi = fi;

purge_continue:
		if (fdbp==fdbn) {emsg("Using same handle %d %d \n",fdbp_fid,fdbn_fid); goto fail;}
		if (fdbp_fid==fdbn_fid) {emsg("Using same fid %d %d \n",fdbp_fid,fdbn_fid); goto fail;}
		fi = upto_fi;
		f = item_open_read(fid_name(fi));
		if (in_purge_one) dbf_seek(f,upto,SEEK_SET);	
		in_purge_one = TRUE;
		nmv = 0;
		for (ni=0;TRUE;) {
				xitem = db_scanitem(fi,f,&gid,&num);
				if (xitem==NULL) break;
				if (gid<0) { imsg("Scan Item GID -ve? [fid=%d] num=%d\n",fi,num); continue; }
				if (gid_to_group(gid) == -1) {
						imsg("Removing item from deleted group %d:%d\n",gid,num);
						goto skipit;
				} else dmsg("dbk: Unpurge [%d] %s:%d \n",fi,group_name(gid_to_group(gid)),num);
				if (!del_this(gid,num)) {
						ni++;
						dmsg("dbk: Unpurge %d %s:%d\n",gid,group_name(gid_to_group(gid)),num);
						nmv++;
						db_unpurge(gid,num,xitem);				
				} else dmsg("db: Removing %d:%d\n",gid,num);
						
skipit:;
				if ((lib_gmtime()-xstart)>=expire_quanta()) {
						upto = dbf_tell(f);
						dbf_close(f);	
						in_purge_one = TRUE;
						ex_stat_add("ItemsMoved",nmv);
						return;
				}
				if ((patch_nfi>90) || (nprec>13000)) {
						imsg("Starting idx_patch as %d>90 or %d>13000\n",patch_nfi,nprec);
						upto = dbf_tell(f);
						dbf_close(f);	
						in_purge_one = TRUE;
						ex_stat_add("ItemsMoved",nmv);
						in_patch = TRUE;
						return;
				}
		}
		ex_stat_add("ItemsMoved",nmv);
		imsg("Purge-done Seconds %d, items %d\n",lib_gmtime()-xstart,ni);
		dbf_close(f);
		if (dbn_bounds(fi)) {
				in_purge_one = FALSE;
				emsg("FATAL fi faulty in purge %d\n",fi);
				return;
		}
		diskuse_add(-dbn[fi].used); 
		patch_add(fi);
		in_purge_one = FALSE;
		dbn[fi].free = 0;
		dbn[fi].used = 1;
/*
		idx_dopatch();
		dbn[fi].free = 0;
		dbn[fi].used = 0;
		dbn[fi].inuse = FALSE;

		if (fdbr>=0) if (fdbr_fid==fi) {
				dbf_close(fdbr);
				fdbr = -1;
		}
		imsg("Removing squeezed bucket %s took %d seconds\n",fid_name(fi),lib_gmtime()-xstart);
		if (remove(fid_name(fi))!=0) filemsg("remove %s failed\n",fid_name(fi));
		if (remove(del_name(fi))!=0) filemsg("remove %s failed\n",del_name(fi));
*/
		
fail:
		in_purge_one = FALSE;

}
void patch_add(int fi)
{
		patch_fi[patch_nfi++] = fi;
		if (patch_nfi>100) {idx_dopatch(); return;}
}
void patch_del(void)
{
		int i;
		int fi;
		slave_flush(TRUE); /* Tell it to start again */
		for (i=0; i<patch_nfi; i++) {
				fi = patch_fi[i];
				if (dbn_bounds(fi)) {
						emsg("patch_del bounds %d\n",fi);
						continue;
				}
				if (fdbr>=0) if (fdbr_fid==fi) {
						dbf_close(fdbr);
						fdbr = -1;
				}
				imsg("Removing squeezed bucket %s \n",fid_name(fi));
				dbn[fi].free = 0;
				dbn[fi].used = 0;
				dbn[fi].inuse = FALSE;
		
		dblog("Removed patch_del %s\n",fid_name(fi));
				if (remove(fid_name(fi))!=0) { /*ISCLOSED*/
		  if (errno!=ENOENT) {
						filemsg("Removex {%s} failed\n",fid_name(fi));	
						dbn[fi].inuse = TRUE;
						remove_later(fi);
		  } 
				}
				if (remove(del_name(fi))!=0) filemsg("remove %s failed\n",del_name(fi));

		}
		slave_flush(FALSE); /* Tell it to start again */
		patch_nfi = 0;
}
/*
		if (isrebuild) delete index, add items to index
		else add history records to history file.
*/
static int rebuild_from;
int db_get_from(void)
{
	return rebuild_from;
}
int db_set_rebuild_from(int x)
{
	rebuild_from = x;
	imsg("Start rebuild_index from bucket %d\n",rebuild_from);
	return TRUE;
}
void db_rebuild_start(void)
{
		int i;
		int nfail=0;

	if (rebuild_from>0) {
		return ;
	}
		if (isrebuild) {
				db_flush();
				if (fdbi>=0) if (!dbi_close(fdbi)) filemsg("FATAL Error closing index file\n");
				fdbi = -1;
				slave_flush(TRUE);
				readonly_pause();
				lib_sleep(1);
				errno = 0;
again:			if (remove(FILE_DBI)!=0) {
						if (errno!=ENOENT) {
								nfail++;
								if (nfail>100) dnews_ex("Could not remove dbi.idx file");
								lib_sleep(1);
								goto again;
						}
				}
				remove(FILE_FIRST);
				remove(FILE_LAST);
				for (i=0; i<xmax_groups; i++) {
						gid_last[i] = -1;
						gid_first[i] = -1;
				}
				imsg("Writefirst rebuild_start\n");
				db_write_first();
				db_write_last();
				m_cache_clear();
				g_clear("writefirst");

				hist_delete(); /* Start with a clean history file */
				item_stopfeeds(TRUE);
				slave_flush(FALSE);
				readonly_unpause();
				/*imsg("Removing .del files\n");
				for (i=0; i<MAX_DBN; i++) {
#ifdef Mac
						if (!dbn[i].inuse) continue;
						if (i%20==3) {
								dmsg("db: Removing %d\n",i);
								lib_doevents();
						}
#endif					
						remove(del_name(i));
				}
				imsg("Removing .del files finished\n");
		*/
		if (isrenumber || isregid) {
		 imsg("Removing .xov files\n");
		 for (i=0; i<group_n(); i++) if (group_isxover(i)) remove(file_xover(i));
		}
		if (isregid) slave_killslaves();

		}
		if (isrenumber) {
				/* both moved above as we do it for reindex too */
		}
}
void db_clear_cache(char *reason)
{
	imsg("db_clear_cache(%s)\n",reason);
		m_cache_clear();
		g_clear(reason);
	g_free_all(reason); /* I think this should be here too */
}
void db_set_renumber(int t)
{
		isrebuild = t;
		isrenumber = t;
}
void db_set_regid(int t)
{
		isrebuild = t;
		isregid = t;
		if (isregid) {
				group_regid();
				group_write();
		}
}
void db_set_rebuild(int t)
{
		isrebuild = t;
}
void db_set_rebuild_fast(int t)
{
		isrebuild = t;
		rebuild_fast = t;
}
void db_set_addlost(int t)
{
		isaddlost = t;
}
static int purge_fid = -2;
static FILE *purge_f;
int db_fix_index(String *xitem, int gid, int num, int fid, int pnt);

int db_rebuild_bucket(int fi)
{
	isregid = TRUE;
	for (;TRUE;) {
		if (db_addlost(fi)<0) break;
	}
	isregid = FALSE;
	imsg("Rebuild deleting bucket we have finished processing %s\n",fid_name(fi));
	pile_remove_one(0,fi,TRUE,"Delete after rebuild_bucket");
	return TRUE;
}

void set_ignorehist(int x);
int db_addlost(int fi)	/* Used by rebuild_piles, among others*/
{
		static int f = -1;
		int pnt;
		static int busy;
		int gid,num;
		int grps[MAX_GRPS];
		int nums[MAX_GRPS];
		int n;
		static int nok;
		static int nfail;
		String *xitem=NULL;



		if (dbn_bounds(fi)) {
				emsg("FATAL addlost bounds %d\n",fi);
				return -1;
		}

		if (f<0) {
				f = item_open_read(fid_name(fi));
				if (f<0) f = item_open_read(fid_name_pile(fi));
				imsg("Rebuild open %s\n",fid_name(fi));
				if (f<0) {
						filemsg("Unable to read %s\n",fid_name(fi));
						return -1;
				}
				nok = 0;
				nfail = 0;
				purge_fid = -1;
		del_read(fi,TRUE);
		}
		for (;TRUE;) {
				pnt = dbf_tell(f); /* Get current file position */
				xitem = db_scanitem(fi,f,&gid,&num);
		/*imsg("rebuild %d:%d;%d %p",gid,num,fi,xitem);*/
/*imsg("rebuild %d:%d;%d %p,  isrenumber %d  isregid %d\n",gid,num,fi,xitem,isrenumber,isregid);*/
				if (xitem==NULL) break;
		if (del_this(gid,num)) continue;
				if (isrenumber) {
						db_trimold(xitem);
						if (item_suck(xitem)) {
								nok++;
						} else nfail++;
				} else if (isregid) {
						/* num is valid, but need new gid value */
						db_trimold(xitem);
						n = db_find_ref(xitem,-1, num, grps,nums,TRUE);
						group_xref_ary(n,grps,nums);
						group_use_xref(TRUE);
						set_ignoreage(TRUE);
						set_ignorecontrol(TRUE);
			set_item_rebuild(TRUE);
			if (!rebuild_rules) feed_setnorules(TRUE);
			/*set_ignorehist(TRUE); fails badly as sometimes dups exist then they get stuck in index*/
						if (item_suck(xitem)) {
								nok++;
						} else nfail++;
			set_item_rebuild(FALSE);
						group_use_xref(FALSE);
			feed_setnorules(FALSE);
						set_ignoreage(FALSE);
			/*set_ignorehist(FALSE);*/
						set_ignorecontrol(FALSE);
				} else if (isrebuild) {
						if (db_fix_index(xitem,gid,num,fi,pnt)) {
								db_fix_hist(xitem,gid,num,fi,pnt);
						} else {
								db_purge_lost(xitem,gid,num,fi,pnt);
						}
				} else if (isaddlost) {
						db_purge_lost(xitem,gid,num,fi,pnt);
				} else db_fix_hist(xitem,gid,num,fi,pnt);
				return 0; /* More items to do still */
		}
	imsg("rebuild closing bucket (%s)\n",fid_name(fi));
		dbf_close(f);
		if (isrenumber || isregid) {
				if ((nok==0) || ((nfail>nok) && (nok<100))) {
						if (nfail>0) emsg("PANIC, Leaving bucket %s, failed=%d ok=%d\n",fid_name(fi),nfail,nok);
						imsg("Leaving bucket %s, failed=%d ok=%d\n",fid_name(fi),nfail,nok);
				} else {
						imsg("Removing empty bucket %s\n",fid_name(fi));
			dblog("Removed empty_bucket %s\n",fid_name(fi));
			dblog("Removed empty_  Pile %s\n",fid_name_pile(fi));
						diskuse_add(-dbn[fi].used); 
						remove(fid_name_pile(fi));
						remove(fid_name(fi));
						remove(del_name(fi));
						dbn[fi].used = 0;
						dbn[fi].free = 0;
						dbn[fi].inuse = FALSE;
				} 
		}
		if (purge_f!=NULL) fclosep(purge_f);
		purge_f = NULL;
		f = -1;
		return -1;
}
static int fix_control;
void db_fix_control(void)
{
		fix_control = TRUE;
}
int db_fix_index(String *xitem, int gid, int num, int fid, int pnt)
{
		int i;
		int grps[MAX_GRPS];
		int nums[MAX_GRPS];
		int n;
		FILE *f;		

		n = db_find_ref(xitem,gid,num,grps,nums,FALSE);
		if (n==0) return FALSE;
		if (fix_control) {	/* To untangle bug in v4 beta */
				if (ref_control) {
						f = fopen(del_name(fid),"a");
						if (f==NULL) {
								filemsg("Fatal: couldn't append to {%s} \n",del_name(fid));
								return -1;
						}
						if (n>0) fprintf(f,"%d %d %d\n",group_gid(grps[0]),num,str_len(xitem));
						fclosep(f); 							 
						return TRUE;
				}
		}

		dmsg("db: fix_index, num=%d fid=%d len=%d pnt=%d\n",num,fid,str_len(xitem),pnt);
		for (i=0; i<n; i++) {
				db_add_idx(fid,grps[i],nums[i],pnt,str_len(xitem));
		}
		return TRUE;
}
static int log_fixhist;
int db_rebuild_ahist(int fid)
{
		int r = 0;
		log_fixhist = TRUE;
		isrenumber = isregid = isrebuild = isaddlost = rebuild_fast = FALSE;
	rebuild_rules = FALSE;
		for (;r>=0;) r = db_addlost(fid);
		log_fixhist = FALSE;
		return TRUE;
}
void db_check_item(int gid,int xnum,int fid,char *gname,int thisnum)
{
		Mrec *mrec;
		int grp;
		int c;

		grp = group_find(gname);
		if (grp<0) {
				emsg("remove_check: group doesnt exist %d:%d fid=%d (%s:%d)\n", gid, xnum, fid,gname,thisnum);
				return;
		}
		mrec = m_cache_find(group_gid(grp),thisnum);
		if (mrec==NULL) {
		 c = g_cache_find(grp);
		 if (c<0) {
				mrec = db_item_read(grp,thisnum);
		 } else {
				mrec = g_get_mrec_cache(grp,thisnum); /* Might be worth caching last one ? */
		 }
		}
		if (mrec==NULL) {
				emsg("remove_check: no index entry %d:%d fid=%d (%s:%d)\n", gid, xnum, fid,gname,thisnum);
				return;
		}
		return;
}
void db_purge_lost(String *xitem, int gid, int num,int fid, int pnt)
{
		char grps[BFSZ];
		char *ng;
		char *s;
		char *msgid;
		char *h;
		int thisnum;
		int isok=FALSE;
		msgid = get_aheadraw(xitem,"Message-ID:");
		if (msgid==NULL) goto killit;
		if (strlen(msgid)==0) goto killit;
		
		h = hist_exists(msgid);
		if (h!=NULL) if (remove_check) {
				if (purge_fid!=fid) {
						del_read(fid,TRUE);
						purge_fid = fid;
				}
				if (del_this(gid,num)) return;
				strcpy(grps,"None");
				ng = get_aheadraw(xitem,"Newsgroups:");
				if (ng!=NULL) ncpy(grps,ng,400);
				if (h==NULL) goto killit;
				s = hist_groupfirst(h);
				for (;s!=NULL;) {
						thisnum = hist_groupnextnum();
						db_check_item(gid,num,fid,s,thisnum);
						s = hist_groupnext();
						isok = TRUE;
				}
				if (!isok) goto killit;
				return;
		}

		if (h!=NULL) return;
killit:
		strcpy(grps,"None");
		ng = get_aheadraw(xitem,"Newsgroups:");
		if (ng!=NULL) ncpy(grps,ng,50);

		if (purge_f==NULL || purge_fid!=fid) {
				if (purge_f!=NULL) fclosep(purge_f); purge_f = NULL;
				del_read(fid,TRUE);
				purge_f = myfopen(del_name(fid),"a");
				purge_fid = fid;
		}
		if (purge_f!=NULL) {
		  if (!del_this(gid,num)) {
				if (!remove_check) {
						imsg("Removing lost item %d:%d fid=%d pnt=%d (%s)\n", gid, num, fid, pnt,grps);
						fprintf(purge_f,"%d %d %d\n",gid,num,str_len(xitem));
						if (fid>=0) dbn[fid].free += str_len(xitem);
				} else {
						emsg("remove_check: not in history %d:%d fid=%d pnt=%d (%s)\n", gid, num, fid, pnt,grps);
				}
		  }
		} 
		return ;
}

int db_fix_hist(String *xitem, int gid, int num,int fid, int pnt)
{
		int grps[MAX_GRPS];
		int nums[MAX_GRPS];
		char msg[BFSZ];
		char *msgid;
		char *h;
		int n;
		int g;
		char bf[BFSZ];
		static String *hline;
		char post_date[BFSZ];
		char expire_date[BFSZ];
 
		msgid = get_aheadraw(xitem,"Message-ID:");
		if (msgid==NULL) return FALSE;
		if (strlen(msgid)==0) return FALSE;
		if (!rebuild_fast) {
		 h = hist_exists(msgid);
		 if (h!=NULL) {
				dmsg("fixhist: ok from fid=%d pnt=%d %s\n",fid,pnt,msgid);
				return FALSE; /* Already got it */
		 }
		} 
		ncpy(msg,msgid,BFSZ-1);
		if (log_fixhist) imsg("db: Adding missing (%s)\n",msg);
		n = db_find_ref(xitem,gid,num,grps,nums,TRUE);
		if (hline==NULL) hline = str_new(); 	str_clear(hline);

		for (g=0; g<n; g++) {
				if (g!=0) str_adds(hline," ",1);
				lprintf(bf,BFSZ-1,"%s/%d",group_name(grps[g]),nums[g]);
				str_adds(hline,bf,strlen(bf));
				if (str_len(hline)>max_strbf()) {
						emsg("FATAL, hline is too long\n");
						break;
				}
		}

		ncpy(post_date, get_aheadraw(xitem,"Date:"),100 );
		ncpy(expire_date, get_aheadraw(xitem,"Expires:"),100 );
		if (str_len(hline)==0) {
				str_copysn(hline," ");
				*str_s(hline) = 0;
		}
		str_lock(hline);
		hist_add_fix(msg,post_date,expire_date,str_s(hline));
		str_unlock(hline);
		dmsg("db: Added lost message %s:%d from fid=%d pnt=%d\n",group_name(grps[0]),nums[0],fid,pnt);
		/* Possible bug, can strings move during hist add ? */ 
		return TRUE;
}
/*
Xref: cplap.netwin.co.nz alt.college.food:1438 alt.college.fraternities:22090
*/

int db_unpurge2(int gid, int num, String *xitem)
{		
		char xref[BIGBF];
		int i;
		char bf[BFSZ];
		int grp;
		Mrec *m;
		char *s;
		char group[BFSZ];
		char *xpoint[MAX_GRPS];
		int nx,xnum=num;

		if (dbn_bounds(fdbp_fid)) {
				emsg("fatal: unpurge 2 fdbp_fid %d\n",fdbp_fid);
				return TRUE;
		}
		ncpy(xref, get_ahead(xitem,"Xref:"),BIGBF-1);
		nx = token_split(xref,xpoint,", \t\015\012",MAX_GRPS-3);
		for (i=2; i<nx; i++) {
				ncpy(bf,xpoint[i],BFSZ-1);
				s = strtok(bf,":"); if (s==NULL) break;
				ncpy(group,s,BFSZ-1);
				s = strtok(NULL,":"); if (s==NULL) break;
				num = atoi(s);
				grp = group_find(group); 
				ex_start("GetMrec");
				m = g_get_mrec(grp,num);
				ex_end("GetMrec");
				if (m==NULL) { /* Lost item, add it back in*/ 
						imsg("Lost link %s:%d\n",group,num);
				} else {
						m->pnt = fdbp_len;
						m->fid = fdbp_fid;
						ex_start("IDX Update");
						m_update(m);
						ex_end("IDX Update");
						imsg("db: Updating xref %s:%d to fid %d pnt %d\n",group,num,m->fid,m->pnt);
				}
		}
		num = xnum;
		m = g_get_mrec(gid_to_group(gid),num);
		if (m!=NULL) {
again:			m->pnt = fdbp_len;
				m->fid = fdbp_fid;
				imsg("db: Updating item %s:%d to fid %d pnt %d\n",group,num,m->fid,m->pnt);
				m_update(m);
				if (dbf_write(fdbp,str_s(xitem),str_len(xitem))!=str_len(xitem)) {
						filemsg("Fatal error during purge.  You must leave enough space for dnews to rewrite it's files, decrease diskspace setting\n");
				}
				diskuse_add(str_len(xitem));

				dbn[fdbp_fid].used += str_len(xitem);
				fdbp_len += str_len(xitem);
		} else {
				emsg("Removing lost item %s:%d\n",group_name(gid_to_group(gid)),num);
				g_free_all("lost item");
				m = g_get_mrec(gid_to_group(gid),num);
				if (m!=NULL) {
						emsg("Found it again :-) \n");
						goto again;
				}
		}
		return TRUE;
}
int db_unpurge(int gid, int num, String *xitem)
{		
		char xref[BIGBF];
		int i;
		char bf[BFSZ];
		int grp;
		char *s;
		char group[BFSZ];
		char *xpoint[MAX_GRPS];
		int nx,xnum=num;

		ncpy(xref, get_ahead(xitem,"Xref:"),BIGBF-1);
		nx = token_split(xref,xpoint,", \t\015\012",MAX_GRPS-3);
		for (i=2; i<nx; i++) {
				ncpy(bf,xpoint[i],BFSZ-1);
				s = strtok(bf,":"); if (s==NULL) break;
				ncpy(group,s,BFSZ-1);
				s = strtok(NULL,":"); if (s==NULL) break;
				num = atoi(s);
				grp = group_find(group); 
				if (grp<0) continue;
				if (num<0) continue; 
				idx_patch(grp,num,fdbp_fid,fdbp_len);
		}
		num = xnum;
		idx_patch(gid_to_group(gid),num,fdbp_fid,fdbp_len);
		dbf_seek(fdbp,0,SEEK_END);
		if (dbf_write(fdbp,str_s(xitem),str_len(xitem))!=str_len(xitem)) {
				filemsg("Fatal error during purge.  You must leave enough space for dnews to rewrite it's files, decrease diskspace setting\n");
		}
		diskuse_add(str_len(xitem));
		if (dbn_bounds(fdbp_fid) ) {
				emsg("fatal fdbp_fid unpurge %d\n",fdbp_fid);
				return TRUE;
		}
		dbn[fdbp_fid].used += str_len(xitem);
		fdbp_len += str_len(xitem);
		return TRUE;
}
#define PREC_NHASH 8000
#define patch_hash(g,n) (abs((g+n) % PREC_NHASH))
static Prec *prec[PREC_NHASH];


void idx_patch(int grp, int num, int fid, int pnt)
{
		Prec *p;
		int h;
		if (grp<0) return;
		if (nprec>14000) idx_dopatch();
		nprec++;
		dmsg("dbk: idx_patch %s:%d fid=%d\n",group_name(grp),num,fid);


		p = mymalloc2(sizeof(Prec),M_DB);
		p->gid = group_gid(grp);
		p->num = num;
		p->fid = fid;
		p->pnt = pnt;
		p->next = NULL;
		h = patch_hash(p->gid,p->num);
		if (prec[h]!=NULL) {
				p->next = prec[h];
		}
		prec[h] = p;
}
void patch_free_one(Prec *p);
void patch_free(void)
{
		int i;
		nprec = 0;
		for (i=0; i<PREC_NHASH; i++) {
				patch_free_one(prec[i]);
				prec[i] = NULL;
		}
}
void patch_free_one(Prec *p)
{
		Prec *n;
		int z;
		for (z=0;p!=NULL;p=n,z++) {
				if (z>9000000) {emsg("loop detected patch \n"); dnews_ex("patch_free");}
				n = p->next;
				myfree2(p,M_DB);
		}
}
/* read thru entire index, if item matches, correct it,  */
#define PN 400
#define PSZ (PN*IDXSIZE)
void idx_dopatch(void)
{
		for (;!idx_dopatch_part();) ;
}
int idx_dopatch_part_part(void);
void idx_dopatch_normal(void);
int idx_dopatch_part_fake(void) /* returns TRUE when finished */ 
{
/*		for (;!idx_dopatch_part_part();) ;*/
		return TRUE;
}

int idx_dopatch_part(void)
{
		Prec *p;
		Mrec m;
		int h;
		char bf[IDXSIZE];
		int n;
		int isslave = is_slave();
		int len;
		int start = lib_gmtime();
		static int upto;

		if (is_readonly()) return TRUE;
		if (fdbi<0) db_openi();
		if (fdbi<0) return TRUE ;
		ex_start("PatchMoved");
		dbi_seek(fdbi,upto*IDXSIZE,SEEK_SET);
		imsg("idx: dopatch upto %d/%d\n",upto,fdbi_len/IDXSIZE);
		for (n=upto;!dbi_eof(fdbi);n++) {
				if ((n%5000) ==1000) if ((lib_gmtime()-start)>expire_quanta()) {
						imsg("idx: dopatch returning\n");
						upto = n;
						ex_end("PatchMoved");
						return FALSE;
				}
				len = dbi_read(fdbi,bf,IDXSIZE);
				if (len<IDXSIZE) break;
				if (!frec_to_mrec(bf,n*IDXSIZE,&m)) {
						continue;
				}
				h = patch_hash(m.gid,m.num);
				if (h<0) continue;
				if (prec[h]==NULL) continue;
				for (p=prec[h];p!=NULL;p=p->next) {
						if (p->gid == m.gid) if (p->num == m.num) {
								/* imsg("Patch: %d:%d at %d\n",m.gid,m.num,m.ipnt);*/
								m.pnt = p->pnt;
								m.fid = p->fid;
								if ((m.ipnt>fdbi_len) && isslave) {
										emsg("moved seek past end %d %d\n",m.ipnt,fdbi_len);
								} else {
										dbi_seek(fdbi,m.ipnt,SEEK_SET);
										mrec_to_frec(&m,bf);
										if (dbi_write(fdbi,bf,IDXSIZE)!=IDXSIZE) filemsg("m_update, rebuild_index needed\n");
								}
								dbi_seek(fdbi,m.ipnt+IDXSIZE,SEEK_SET);
						}
				}
		}
		upto = 0;
		ex_end("PatchMoved");
		ex_start("PatchDel");
		patch_free();
		imsg("idx: dopatch delete and slave sync\n");
		patch_del();
		db_clear_cache("dopatch");
		slave_dbflush();
		ex_end("PatchDel");
		imsg("idx: dopatch took %d seconds %d items\n",lib_gmtime()-start,nprec);

		return TRUE;
}

void idx_dopatch_normal(void)
{
		Prec *p;
		Mrec m;
		int h;
		char bf[IDXSIZE];
		int n;
		int isslave = is_slave();
		int len;
		int start = lib_gmtime();

		if (is_readonly()) return;
		if (fdbi<0) db_openi();
		if (fdbi<0) return ;
		ex_start("PatchMoved");

		dbi_seek(fdbi,0,SEEK_SET);
		for (n=0;!dbi_eof(fdbi);n++) {
				len = dbi_read(fdbi,bf,IDXSIZE);
				if (len<IDXSIZE) break;
				if (!frec_to_mrec(bf,n*IDXSIZE,&m)) {
						continue;
				}
				h = patch_hash(m.gid,m.num);
				if (h<0) continue;
				if (prec[h]==NULL) continue;
				for (p=prec[h];p!=NULL;p=p->next) {
						if (p->gid == m.gid) if (p->num == m.num) {
								/* imsg("Patch: %d:%d at %d\n",m.gid,m.num,m.ipnt);*/
								m.pnt = p->pnt;
								m.fid = p->fid;
								if ((m.ipnt>fdbi_len) && isslave) {
										emsg("moved seek past end %d %d\n",m.ipnt,fdbi_len);
								} else {
										dbi_seek(fdbi,m.ipnt,SEEK_SET);
										mrec_to_frec(&m,bf);
										if (dbi_write(fdbi,bf,IDXSIZE)!=IDXSIZE) filemsg("m_update, rebuild_index needed\n");
								}
								dbi_seek(fdbi,m.ipnt+IDXSIZE,SEEK_SET);
						}
				}
		}

		ex_end("PatchMoved");
		ex_start("PatchDel");
		patch_free();
		imsg("idx: dopatch delete and slave sync\n");
		patch_del();
		db_clear_cache("dopatch2");
		slave_dbflush();
		ex_end("PatchDel");
		imsg("idx: dopatch took %d seconds %d items\n",lib_gmtime()-start,nprec);
}

/*
		Alternative to above technique, which would be faster but also more
		complex.
				Remeber list of corrections.  
				gid, num, newpnt, newfid
				Once we have 50,000  reprocess index, and correct
				all that need correcting.

 */
/*-------------------------------------------------------*/
/* check for any ways bad data can break this routine */		
String *db_scanitem(int fi, int f, int *gid, int *xnum)
{
		int i;
		char *s;
		char *s2;
		char bf[BFSZ];
		char bf2[BFSZ];
		int num;
		static String *xitem;
		int tlen;
		int n;
		if (xitem==NULL) xitem = str_new();
		str_clearbig(xitem);
		*gid = -1;
again:
		n = dbf_read(f,bf,200);   bf[200]=0; /* increase after testing small */
		if (n<=0) return NULL;
		bf[n] = 0;
		ncpy(bf2,bf,30);
		dmsg("db: Read sample {%s}\n",bf2);
		if (bf[0]!='~') {
				bf[50] = 0;
				emsg("Item file corrupt or out of sync %d {%s}\n",fi,bf);
				goto find_start;
		} 
		dbf_seek(f,-n,SEEK_CUR);
		
		/* Get the item length */
		tlen = atoi(bf+1);	/* Probably need correction here */
		if ((tlen<10) || (tlen > 7000000)) {emsg("Lost sync3\n"); goto find_start2;} 
		s = strchr(bf,':');
		if (s==NULL) {emsg("Lost sync\n"); goto find_start2;}
		s++;
		/* Get the item number */
		num = atoi(s); *xnum = num;
		s = strchr(s,':');
		if (s==NULL) {emsg("Lost sync2\n"); goto find_start2;}
		*gid = -1;
		/* Get the GID field if this is the new format */
		s2 = strchr(s+1,':');	
		if (s2!=NULL) if ( (s2-s) <10) if (isdigit(s[1])){
				*gid = atoi(s+1);
				s = s2;
		}

		if (num<0 || num>999999999) {emsg("purge_one, invalid item number %d bucket=%d\n",num,fi); goto find_start2;}
		if (!isold) if (*gid<0 || *gid>xmax_groups) {emsg("purge_one, invalid gid number %d bucket=%d\n",num,fi); goto find_start2;}

		tlen += s-bf+2;
		if (s[1]=='@') { /* Old style link, skip it  TEST THIS */
				dbf_seek(f,tlen,SEEK_CUR);
				goto again;
		}
		
/* check for any ways bad data can break this routine */		

		if (xitem==NULL) xitem = str_new();
		str_clearbig(xitem);
		if (tlen>(MAX_STRLEN)) {
				emsg("db_scanitem: cannot get item, too big %d, skipping item\n",tlen); 
				n = tlen;
				goto find_start2;
		}
		str_setlen(xitem,tlen+10);
		n = dbf_read(f,str_s(xitem),tlen);
		if (n<tlen) {filemsg("Could not read entire item. got %d of %d, dbf handle %d fi=%d\n",n,tlen,f,fi); return NULL;}
		str_setlen(xitem,tlen);
		ncpy(bf2,str_s(xitem),10);
		dmsg("db: Start 10 characters {%s} \n",bf2);
		ncpy(bf2,str_s(xitem)+tlen-10,12);
		dmsg("db: End 10 characters {%s} len %d \n",bf2,tlen);
		/*	If end char is not ^, then search back 10 to find it */
		if (scan_fix) if (tlen>10) {
				for (i=9;i>=0;i--) {
						if (bf2[i]=='^') break;
				}
				if (i>0) { 
						dmsg("db: scanfix backstep %d bytes\n",9-i);
						tlen = tlen - (9 - i);
						str_setlen(xitem,tlen);
						if (i!=9) dbf_seek(f,- (9 - i),SEEK_CUR);
				}
		}
		
		
		/* ~1213:2100:From: Brett Meehan  ~len:num:Message */
		return xitem;
find_start2:;
		dbf_seek(f,n,SEEK_CUR); /* Skip the problem area */
find_start:;
		for (;!dbf_eof(f); ) {
				n = dbf_read(f,bf,200);
				if (n<=0) break;
				s = strchr(bf,'~');
				if (s!=NULL) {
						dbf_seek(f,-n+(s-bf),SEEK_CUR);
						goto again;
				}
		}
		return NULL;
}
/* Sample mesage header 

From: chrisp@test.site
Subject: Test message number 2333
Message-ID: <2333@test.netwin.co.nz>
Distribution: netwin
Date: 19 Jul 1996 21:22:23 GMT
Newsgroups: test.one,test.two,test.three
Path: 161.29.2.7!grunt.marc.cri.nz!news.palm.cri.nz!iconz.co.nz!news.massey.ac.nz!waikato!news.uoregon.edu!news.kreonet.re.kr!usenet.kornet.nm.kr!news.sprintlink.net!tank.news.pipex.net!pipex!oleane
Xref: 161.29.2.7 test.one:3 test.two:3 test.three:3
*/
/*----------------- Database recovery -------------------*/
/*
Re-write index
		re-build the active ranges and items from the index
		re-link all the pointers in the index
		Sort by group
Re-create the index from the .itm files
Re-create the history file from the .itm files
Re-process an external .itm file
Correct diskuse figure
If unclean shutdown, then on startup, 
		reread index, and use bitfield to detect lost ones
		and then add them in or remove if duplicate article number.
*/
/*----------------- Database recovery -------------------*/
void db_flush2(void)
{
	idx_write_flush("db_flush2");
		db_flush();
		if (fdbi>=0) dbi_close(fdbi);
		fdbi = -1;
}
void db_exit(void)
{
		int r;
		int failed = FALSE;
	idx_write_flush("db_flush");
		db_flush();
		if (fdbi>=0) if (!dbi_close(fdbi)) filemsg("FATAL Error closing index file\n");
		fdbi = -1;

/* r = db_write_first();		if (!r) failed = TRUE;	*/
		r = db_write_last();	if (!r) failed = TRUE; 
		r = db_write_fid(); 	if (!r) failed = TRUE; 
		if (!is_readonly()) {
				if (!failed) remove(FILE_CHK);
		}
}
void db_flush_quick(void)
{
		out_flush();
		if (fdbn>=0) dbf_flush(fdbn);
		if (fdbi>=0) dbi_flush(fdbi);
}

int db_flush(void)
{
		if (fdbi>=0) if (!dbi_flush(fdbi)) filemsg("FATAL Error flushing index file\n");
		if (fdbp>=0) if (!dbf_close(fdbp)) filemsg("FATAL Error closing purge file\n");
		if (fdbg>=0) if (!dbf_close(fdbg)) filemsg("FATAL Error closing gid_last index file\n");
		if (db_ispile()) {
				out_closeall();
		} else {
				if (fdbn>=0) if (!dbf_close(fdbn)) {filemsg("FATAL Error closing item file\n"); lib_setpaused();}
		}
		if (fdbr>=0) dbf_close(fdbr);
		fdbr = fdbn = fdbg = fdbp = -1;
		return TRUE;
}
void db_idle_slow(void)
{
		if (!is_readonly()) return;
		if (fdbi<0) return;
		if (dbi_changed(fdbi)) {
				emsg("Slave detected master index changed, reloading\n");
				db_flush2();
				db_clear_cache("idle_slow");
		}
}
int db_rerun_next(void);
static int rerun_bkt = -1;
static int rerun_at = -1;
static int rerun_tot,rerun_done;
static int rerun_nok = 0;
static int rerun_nfail = 0;
char *db_rerun_status(void)
{
	static char bf[BFSZ];
	if ((rerun_bkt<0) && (rerun_nok==0)) return "";
	lprintf(bf,400,"Rerun_rules ok=%d del=%d, Upto bucket %d of %d\r\n",rerun_nok,rerun_nfail,rerun_done,rerun_tot);
	return bf;
}	
int db_rerun_rules_start(int pile)
{
	/* Copy list of bucket numbers to process 
	and init counter of which one we are doing */
	int i;
	rerun_tot = 0;
	rerun_done = 0;
		for (i=0; i<MAX_DBN; i++) {
		dbn_rerun[i] = FALSE;
		if (pile>0) if (dbn[i].pile!=pile) continue;
			if (dbn[i].inuse) {
			dbn_rerun[i] = TRUE;
			rerun_tot++;
		}
	}
	rerun_nfail = rerun_nok = 0;
	db_rerun_next();
	return TRUE;
}
int db_rerun_next(void)
{
	int i;
		for (i=0; i<MAX_DBN; i++) if (dbn_rerun[i]) break;
	if (i>=MAX_DBN) {
		rerun_bkt = -1;
		rerun_at = -1;
		return TRUE;
	}
	rerun_done++;
	rerun_bkt = i;
	dbn_rerun[i] = FALSE;
	rerun_at = 0;
	return TRUE;
}
static int is_rerun_filter;
int db_is_rerun_filter(void)
{
	return is_rerun_filter;
}

int db_rerun_dosome(void)
{
		static int f = -1;
		int pnt;
	int ngrps;
		int gid,num;
	int xstart = lib_gmtime();
		int grps[MAX_GRPS];
		int nums[MAX_GRPS];
		String *xitem=NULL;
	int i;

	if (rerun_bkt<0) return FALSE;
		if (dbn_bounds(rerun_bkt)) {
				emsg("FATAL addlost bounds %d\n",rerun_bkt);
				return -1;
		}


		if (f<0) {
				f = item_open_read(fid_name_pile(rerun_bkt));
				imsg("Rerun rules open %s\n",fid_name(rerun_bkt));
				if (f<0) {
						filemsg("Unable to read %s\n",fid_name_pile(rerun_bkt));
			db_rerun_next();
			return FALSE;
				}
		}
		
		if (rerun_at>0) dbf_seek(f,rerun_at,SEEK_SET);	  
	for (;TRUE;) {
				pnt = dbf_tell(f); /* Get current file position */
				xitem = db_scanitem(rerun_bkt,f,&gid,&num);
				if (xitem==NULL) break;
		if (del_this(gid,num)) continue;			  
		db_trimold(xitem);
				ngrps = db_find_ref(xitem,-1, num, grps,nums,TRUE);
				set_ignoreage(TRUE);
				set_ignorecontrol(TRUE);
		set_ignorehist(TRUE); 
		is_rerun_filter = TRUE;
				if (item_suck(xitem)) {
			rerun_nok++;
				} else {
			rerun_nfail++;
			for (i=0;i<ngrps;i++) { 
				db_cancel_fast(grps[i],nums[i],"rurun_rules",i!=0);
			}
		}
		is_rerun_filter = FALSE;
				set_ignoreage(FALSE);
		set_ignorehist(FALSE);
				set_ignorecontrol(FALSE);
		
				if ((lib_gmtime()-xstart)>=expire_quanta()) {
						rerun_at = dbf_tell(f);
						dbf_close(f);	
			f = -1; 		
						return TRUE;
				}
		}
	imsg("rebuild closing bucket (%s)\n",fid_name(rerun_bkt));
		dbf_close(f);
		f = -1;
	db_rerun_next();
		return TRUE;
}
int db_idle(void)  /* Called if DNEWS is idle, close some files etc. */
{
		static int last;
		static int last10;
	static int last_free= -1;

	if (rerun_bkt>=0) db_rerun_dosome();

	if (next_grp_free>0) {
		if (next_grp_free != last_free) {
			last_free = next_grp_free;
			imsg("Freeing group dir  cache for group with expired items in it %s\n",group_name(next_grp_free));
			db_free_cache(next_grp_free,"idle_expired");
		}
	}
	next_grp_free = -1;


		if ((lib_gmtime() - last)>60) {
				last = lib_gmtime();
				if (is_readonly()) {
						dmsg("pile: readonly idle \n");
						dbn_trim_load();
						return TRUE;
				} else {
			db_write_fid();
		}
		if (is_expiring()) {
			if (config_true(C_SPAWN_EXPIRE)) return TRUE;
		}
		stat_start(X_DBTRIM);
				db_piles_trim();
		stat_end(X_DBTRIM);
		}
	if (lib_gmtime()-last10>10) {
		stat_start(X_CHANGES);
		if (config_true(C_INDEX_CHANGES)) {
			db_flush_quick(); /* Flush index file occasionally */
				if (fdbi>=0) dbi_close(fdbi);
				fdbi = -1;
		}
		last10 = lib_gmtime();
		db_idle_slave();
		stat_end(X_CHANGES);
	}

		return TRUE;
}
int db_last_len(void)
{
		return last_read_len;  /* Check what this is used for */
}
String *db_showdbn(void)
{
		static String *out;
		int i;
		char bf[BFSZ];
		if (out==NULL) out = str_new();
		str_clear(out);
		for (i=0; i<MAX_DBN; i++) if (dbn[i].inuse || dbn_istrim[i]) {
		lprintf(bf,BFSZ-1,"db[%d] used %d free %d, disk %d, age=%d pile %d, %s%s%s%s \015\012",i,dbn[i].used,dbn[i].free,dbn[i].disk,(time(NULL) - dbn[i].cdate)/(60*60*24),dbn[i].pile
		,dbn_piledel[i] ? "piledel " : ""		
		,dbn_wasused[i] ? "wasused " : ""		
		,dbn_istrim[i] ? "istrim " : "" 		
		,dbn_isbusy[i] ? "isbusy " : "" 		
		); 
				str_addsn(out,bf);
		}
		return out;
}
int db_choose_next(void)
{
		int i;
		if (db_ispile()) {
				out_switch(); /* Choose new buckets for all the out handles */
				return TRUE;
		}
		if (fdbn>=0) dbf_close(fdbn); fdbn = -1;
		i = db_next_fid();
		if (i<0) return FALSE;
		fdbn_fid = i;
		fdbn_len = 0;
		strcpy(fdbn_name,fid_name(i));
		if (config_true(C_BUCKET_NOFILL))  {
		dblog("Removed choose_next %s\n",fdbn_name);
				remove(fdbn_name);
		}
	dblog("Choose next bucket %s\n",fdbn_name);
		fdbn = item_open_append(fdbn_name);
		imsg("fdbn_name aaa %s %d\n",fdbn_name,fdbn);
		if (fdbn<0) {filemsg("Could not append to {%s} \n",fdbn_name); return FALSE;}
		fdbn_len = dbf_len(fdbn);
		return TRUE;
}
char *dbf_status(void);
char *db_status(void)
{
		static char bf[BFSZ];
		int i;
		double xused,xfree;
		xused = xfree = 0;
		for (i=0; i<MAX_DBN; i++) if (dbn[i].inuse) {
				xused += dbn[i].used;
				xfree += dbn[i].free;
		}
		if (xused==0) xused = 1;
		lprintf(bf,BFSZ-1,"DB used %.2fMB, Slack %.2fMB %.2f%%, %s h=%d lost=%d\r\n",xused/1000000,xfree/1000000,xfree*100.0/(xused),dbf_status(),dbf_nfree(),total_lost);
		if (insrt) strcat(bf,"Sorting last mb of index\r\n");
		return bf;
}
static char *spool_dir[100];
static int spool_space[100];  /* Stored in K */
static int spool_used[100];
String *db_showdisks(void)
{
		int nxt = next_disk();
		int f;
		int xov = 0;
		int best_free,z;
		static String *out;
		int i;
		char bf[BFSZ];
		if (out==NULL) out = str_new();
		str_clear(out);

		z  = hist_diskspace() * 3 / 2;
		z += (fdbi_len*3 / 2)/1000;
		if (config_true(C_HISTORY_MOVED)) {
				z = 0;
		}
		

		if (!config_true(C_XOVER_MOVED)) {
				xov = xov_total_get();			
		}		

		best_free = spool_space[0] - spool_used[0] - z - xov;
		i = 0;
		lprintf(bf,BFSZ-1,"spool[%d] used=%d free=%d hist=%d xov=%d\n",i,spool_used[i],best_free,z,xov);
		str_addsn(out,bf);

		for (i=1; i<nspool; i++) {
				f = spool_space[i] - spool_used[i];
				lprintf(bf,BFSZ-1,"spool[%d] used=%d free=%d \n",i+1,spool_used[i],f);
				str_addsn(out,bf);
		}
		lprintf(bf,BFSZ-1,"Next disk chosen [%d]\n",nxt+1);
		str_addsn(out,bf);
		return out;
}
static int noerror=TRUE;
int db_check_buckets_noerror(void)
{
	noerror = TRUE;
	db_check_buckets();
	noerror = FALSE;
	return TRUE;
}
int db_check_bucket_age(int i);
int db_check_bucket_ages(void)
{
	int i;
	db_check_buckets();
		for (i=0; i<MAX_DBN; i++) {
				if (dbn[i].inuse) db_check_bucket_age(i);
	}
		db_write_fid();
	return TRUE;
}
int db_check_bucket_age(int i)
{
	/* Open bucket, find first Date: header, and use it
	
	*/	
	char bf[BFSZ];
	char *s;
	FILE *f;
	int d=0;
	f = fopen(fid_name(i),"r");
	if (f==NULL) {
		filemsg("Failed to open (%s) to check dates\n",fid_name(i));
		return FALSE;
	}
	for (;!feof(f);) {
		if (fgets(bf,200,f)==NULL) break;
		s = strchr(bf,'\n');
		if (s!=NULL) *s = 0;
		s = strchr(bf,'\r');
		if (s!=NULL) *s = 0;
		if (strncmp(bf,"Date: ",6)==0) {
			d = parsedate(bf+6);
			imsg("Check age %d, is %d (%s)\n",i,(time(NULL)-d)/(60*60*24),bf+6);
			if (d>0) break;
		}
	}
	fclosep(f);
	if (d>0) {
		dbn[i].cdate = d;
	}
	return TRUE;
}
int db_check_buckets(void)
{
		int j;
		static Sary *ls;		
		int i;
		int h;
		if (is_readonly()) return TRUE;
		db_flush2();
	out_reset();
		if (ls==NULL) {
				ls = s_alloc();
		}
		for (i=0; i<MAX_DBN; i++) dbn[i].found = FALSE;
		for (i=0; i<nspool; i++) {
				if (config_true(C_SPOOL_HASH)) {
						for (h=0; h<100; h++) {
						 s_clear(ls);
						 s_wild(ls,db_spool_dirh(i,h),"db_*.itm");	/* Returns list without paths */
						 for (j=0; j<s_len(ls); j++) {
								check_bucket(i,s_line(ls,j),h);
						 }
						}
				} else {
						s_clear(ls);
#ifdef NETWARE
			s_wild(ls,db_spool_dir(i),"d*.itm");  /* Returns list without paths */
#else					   
			s_wild(ls,db_spool_dir(i),"db_*.itm");	/* Returns list without paths */
#endif
						if (s_len(ls)<=0) imsg("spool_dir = (%s) \n",db_spool_dir(i));
						for (j=0; j<s_len(ls); j++) {
								check_bucket(i,s_line(ls,j),0);
						}
				}		
		}
		for (i=0; i<MAX_DBN; i++) {
				if (dbn[i].inuse) if (!dbn[i].found) {
						if (!noerror) emsg("Item bucket disappeared? {%s} \n",fid_name(i));
			if (dbn[i].used>0) if (!is_readonly()) {
				if (config_true(C_EXPIRE_LOST_BUCKETS)) {
							dbn_trim(i); /* Record and don't reuse */
					imsg("Recording bucket as expired and not reusing till expire bkt=%d\n",i);
				} else {
					imsg("Missing bucket, bkt=%d\n",i);
				}
			}
						dbn[i].inuse = FALSE;
						dbn[i].used = 0;
				}
		}
		s_free(ls); ls = NULL;
	imsg("db_write_fid\n");
		db_write_fid();
	imsg("db_correct_used() \n");
		db_correct_used();
	imsg("db_correct_used() - done \n");
		return TRUE;
}
void diskuse_set(int x);
void db_correct_used(void)
{
		int i;
		int tot=0;
	if (get_expire_mode()) return;
		diskuse_set(0);
		for (i=0; i<MAX_DBN; i++) {
				if (dbn[i].inuse) {
						diskuse_add(dbn[i].used);
				}
		}
		if (!config_true(C_XOVER_MOVED)) {
				diskuse_add(xov_total_get()*1000);				
		}
		lib_recheck_space();
}
void check_bucket(int disk, char *fname, int h)
{
		char *s;
		int i;
		char *s2;
		int fz;
		int fid;
		int pile=0;
	int realused;
		int cdate;
		imsg("Bucket check {%s} \n",fname);
		s = strchr(fname,'_');
		if (s==NULL) return;
		s++;
		if (!isdigit(*s)) return;
		fid = atoi(s);
		s2 = strchr(s,'_');
		if (s2!=NULL) pile = atoi(s2+1); 
		if (dbn_bounds(fid)) return ;
		realused = fz = lib_filesize(file_spoolh(disk,fname,h));						   
		cdate = lib_cdate(file_spoolh(disk,fname,h));							
		dbn[fid].found = TRUE;
		if (dbn[fid].inuse) {
	  if (dbn[fid].pile != pile) {
		imsg("Bucket in wrong pile, moving %s from %d to %d\n",fname,dbn[fid].pile,pile);
		dbn[fid].pile = pile;
	  }
		  if (dbn[fid].disk != disk) {
				dbn[fid].used = fz;
				dbn[fid].pile = pile;
				imsg("Bucket moved disks, correcting index {%s} was=%d is=%d\n",fid_name(fid),dbn[fid].disk,disk);
				dbn[fid].disk = disk;
		  } else {
				if (fz>=0) {
				 if (dbn[fid].used != fz) {
						emsg("Correcting usage for %s from %d to %d\n",fid_name(fid),dbn[fid].used,fz);
						dbn[fid].used = fz;
				 }
				}
				if (dbn[fid].cdate == 0) {
						emsg("Correcting creation date [%d] to %d\n",fid,cdate);
						dbn[fid].cdate = cdate;
				}
		  } 
		} else { /* unknown one */
				dbn[fid].inuse = TRUE;
				dbn[fid].disk = disk;
				dbn[fid].used = fz;
				dbn[fid].cur = FALSE;
				dbn[fid].fid = fid;
				dbn[fid].pile = pile;
				dbn[fid].cdate = cdate;
				emsg("Found lost fid %s size %d pile %d\n",fid_name(fid),fz,pile);
		}
		/* Check for deleted items */
		del_read(fid,FALSE);
		fz = 0;
		for (i=0; i<ndel; i++) {
				fz += del_sz[i];
		}
		if (dbn[fid].free != fz) {
				imsg("Correcting free space in file from %d to %d\n",dbn[fid].free,fz);
				dbn[fid].free = fz;
		}

		if ((dbn[fid].used - dbn[fid].free) <=0) {
		if (dbn[fid].used==0)  imsg("Removing empty lost bucket [%d] free %d of %d\n",fid,dbn[fid].free,dbn[fid].used);
		else  emsg("Removing empty lost bucket [%d] free %d of %d\n",fid,dbn[fid].free,dbn[fid].used);
		dblog("Removing empty lost %s\n",fid_name(fid));
				if (remove(fid_name(fid))!=0) {
			if (errno!=ENOENT) {
							filemsg("Couldn't remove empty bucket\n");
			} else {
							remove(del_name(fid));
							dbn[fid].inuse = FALSE;
							dbn[fid].used = 0;
							dbn[fid].free = 0;
							dbn[fid].pile = 0;
			}
				} else {
						remove(del_name(fid));
						dbn[fid].inuse = FALSE;
						dbn[fid].used = 0;
						dbn[fid].free = 0;
						dbn[fid].pile = 0;
				}
		}
}
void db_pile_move(int chan,char *xp1,char *xp2)
{
	char bf[BFSZ];
	int i;
	char oldname[BFSZ];
	char newname[BFSZ];
	int p1,p2;
	lprintf(bf,BFSZ-1,"  Moving buckets in pile (%s) to (%s) \r\n",xp1,xp2);
	chan_write_s(chan,bf);
	if (strlen(xp1)==0) return;
	if (strlen(xp2)==0) return;
	p1 = atoi(xp1); p2 = atoi(xp2);
	db_flush();
	out_reset();
		for (i=0; i<MAX_DBN; i++) {
		if (!dbn[i].inuse) continue;
		if (dbn[i].pile != p1) continue;
		ncpy(oldname,fid_name(i),200);
				dbn[i].pile = p2;
		ncpy(newname,fid_name(i),200);
		if (rename(oldname,newname)!=0) {
			lprintf(bf,BFSZ-1,"  Rename failed (%s) (%s) (%s) \r\n",oldname,newname,strerror(errno));
					dbn[i].pile = p1;
		} else {
			lprintf(bf,BFSZ-1,"  Renamed (%s) -> (%s) \r\n",oldname,newname);
		}
		chan_write_s(chan,bf);
	}
}


int next_disk_free(void); /* Choose disk based on free space */
int hist_diskspace();


int next_disk(void)
{
		int d;
		int i;
		int xov = 0;
		int z;
	static int last,last2;
	int f;
		int best,best_free;
		if (nspool==1) return 0; /* No choice */
		if (config_true(C_SPOOL_FREE)) return next_disk_free();
		for (i=0; i<nspool; i++) spool_used[i] = 0;
		for (i=0; i<MAX_DBN; i++) {
				if (dbn[i].inuse) {
						d = dbn[i].disk;
						if (d<0 || d>nspool) continue;
						spool_used[d] += dbn[i].used/1000;
				}
		}
		z  = hist_diskspace() * 3 / 2;
		z += (fdbi_len*3 / 2)/1000;
		if (config_true(C_HISTORY_MOVED)) {
				z = 0;
		}

		
		if (!config_true(C_XOVER_MOVED)) {
				xov = xov_total_get();			
		}		
		best = 0; best_free = spool_space[0] - spool_used[0] - z - xov;
		if (config_true(C_PILE_SEPARATE)) {
		if (next_wanted>0) {
				i = next_wanted-1;
				f = spool_space[i] - spool_used[i];
				if (i==0) f = f - xov  - z;
				if (f>(item_min/1000)) return i;
		}
	}
		i = 0;
		imsg("spool[%d] used=%d free=%d hist=%d xov=%d\n",i,spool_used[i],best_free,z,xov);
	
		for (i=1; i<nspool; i++) {
		/* Avoid choosing the same disk twice if you can */
		if (config_true(C_PILE_NOPEN)) {
			if (last==i) if (nspool>2) continue;
			if (last2==i) if (nspool>3) continue;
		}
				f = spool_space[i] - spool_used[i];
				if (f>best_free) {best = i; best_free = f;}
				imsg("spool[%d] used=%d free=%d \n",i,spool_used[i],f);
		}
	if (best_free<0) {
			for (i=1; i<nspool; i++) {
			/* recheck, maybe we can find one if we do use the same one */
					f = spool_space[i] - spool_used[i];
					if (f>best_free) {best = i; best_free = f;}
					imsg("spool[%d] used=%d free=%d \n",i,spool_used[i],f);
			}
	}
		imsg("Chosen next disk spool[%d] used=%d \n",best,spool_used[best]);
	last2 = last;
	last = best;
		return best;
}
int next_disk_free(void) /* Choose disk based on free space */
{
		int f;
		int best,besti,i;
		besti = 0; best = 0;
		for (i=0; i<nspool; i++) {
				f = lib_freespace(spool_dir[i]); 
				if (f>best) { besti = i; best = f;}
		}		
		imsg("Chose spool area %s with %dMB's free\n",spool_dir[besti],best/1000);
		return besti;
}
char *db_spool_dirh(int d, int h)
{
		static char bf[BFSZ];
		if (config_true(C_SPOOL_HASH)) {
				lprintf(bf,BFSZ-1,"%s%d",db_spool_dir(d),h);
				return bf;
		}
		return db_spool_dir(d);

}
char *db_spool_dir(int d)
{
#ifdef VMS
		static char bf[BFSZ];
		lprintf(bf,BFSZ-1,"%s[000000]",spool_dir[d]);
		return bf;
#else
		if ((d<0) || (d>=nspool)) return "CORRUPT_IDX_ENTRY";
		return spool_dir[d];
#endif
}
int db_nspool(void)
{
	return nspool;
}
void db_init_spool(void)
{
	int i;
	imsg("db_init_spool again nspool %d\n",nspool);
	for (i=0; i<nspool; i++) {
		
	}
		nspool = 0;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE));
		if (strlen(config_getu(C_SPOOL2))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL2),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE2));
		if (strlen(config_getu(C_SPOOL3))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL3),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE3));
		if (strlen(config_getu(C_SPOOL4))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL4),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE4));
		if (strlen(config_getu(C_SPOOL5))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL5),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE5));
		if (strlen(config_getu(C_SPOOL6))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL6),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE6));
		if (strlen(config_getu(C_SPOOL7))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL7),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE7));
		if (strlen(config_getu(C_SPOOL8))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL8),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE8));

		if (strlen(config_getu(C_SPOOL9))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL9),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE9));

		if (strlen(config_getu(C_SPOOL10))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL10),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE10));

		if (strlen(config_getu(C_SPOOL11))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL11),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE11));
		if (strlen(config_getu(C_SPOOL12))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL12),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE12));
		if (strlen(config_getu(C_SPOOL13))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL13),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE13));
		if (strlen(config_getu(C_SPOOL14))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL14),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE14));
		if (strlen(config_getu(C_SPOOL15))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL15),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE15));
		if (strlen(config_getu(C_SPOOL16))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL16),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE16));
		if (strlen(config_getu(C_SPOOL17))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL17),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE17));
		if (strlen(config_getu(C_SPOOL18))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL18),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE18));
		if (strlen(config_getu(C_SPOOL19))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL19),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE19));
		if (strlen(config_getu(C_SPOOL20))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL20),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE20));

	if (strlen(config_getu(C_SPOOL21))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL21),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE21));
		if (strlen(config_getu(C_SPOOL22))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL22),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE22));
		if (strlen(config_getu(C_SPOOL23))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL23),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE23));
		if (strlen(config_getu(C_SPOOL24))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL24),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE24));
		if (strlen(config_getu(C_SPOOL25))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL25),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE25));
		if (strlen(config_getu(C_SPOOL26))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL26),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE26));
		if (strlen(config_getu(C_SPOOL27))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL27),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE27));
		if (strlen(config_getu(C_SPOOL28))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL28),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE28));
		if (strlen(config_getu(C_SPOOL29))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL29),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE29));
		if (strlen(config_getu(C_SPOOL30))==0) return;
		spool_dir[nspool] = mystrdup(config_getu(C_SPOOL30),SD_DB);
		spool_space[nspool++] = 1000*atoi(config_get(C_DISKSPACE30));
}
static int total_space;
int db_total_space(void)
{
		return total_space; /* in MB */
}
int db_total_space_left(void)
{
	static int last_was,last_time;
		int gap;

	if (last_time==lib_gmtime()) return last_was;
	last_time = lib_gmtime();
		gap  = hist_diskspace() * 2;
		gap += (fdbi_len*2)/1000;
		if (config_true(C_HISTORY_MOVED)) {
				gap = 0;
		}
		if (!config_true(C_XOVER_MOVED)) {
				gap += xov_total_get(); 		 
		}		
	last_was = total_space-(gap/1000);
	return last_was; /* in MB */
}
void db_reload(void)
{
		int i;
		int b;
		db_init_spool();

		total_space = 0;
		for (i=0; i<nspool; i++) {
				lib_createdir(fid_name2(i,0)); /* Create spool directories */
				total_space += (spool_space[i]/1000);
		}
		
		item_min = 200000000;
		if (total_space<20000) item_min = 50000000; /* reduced, it was 60gig, now it's 20gig*/
		if (total_space<100) item_min = 1000000;
		if (total_space<10) item_min =	100000;
		b = atoi(config_get(C_BUCKET_SIZE));
		if (b>0) item_min = b;
		imsg("db_init: Bucket size %d total space %d (%d/10000 when full)\n",item_min,total_space, (((total_space*1000)/(item_min/1000)))  );
	if (( (((total_space*1000.0)/(item_min/1000.0))) )> 9500.0 ) {
		if (b>0 && b<10000000) {
			int xx = (int) (((total_space*1000.0)/5000.0)*1000.0);
				emsg("db: WARNING BUCKET_SIZE IS TOO LOW but ok if days setting used, consider %d\n",xx);
		} else {
				item_min = (int) (((total_space*1000.0)/5000.0)*1000.0);
			emsg("db: WARNING BUCKET_SIZE IS TOO LOW, INCREASing TO %d\n",item_min);
		}
	}
}
void db_move_hash(int spl, char *fname);
void db_move_xov(char *fname);
void db_init_hash(void)
{
		int j;
		static Sary *ls;		
		int i;
		char bf[BFSZ];

		if (is_readonly()) return ;
		if (!config_true(C_SPOOL_HASH)) goto next_bit;
				
		ls = s_alloc();
		for (i=0; i<nspool; i++) {
				for (j=0; j<100; j++) {
						lprintf(bf,BFSZ-1,"%s/test.file",db_spool_dirh(i,j));
						lib_createdir(bf);
				}
				s_clear(ls);
				s_wild(ls,db_spool_dir(i),"db_*.*");  /* Returns list without paths */
				if (s_len(ls)<=0) return;
				wmsg("Moving files to hashed structure\n");
				for (j=0; j<s_len(ls); j++) {
						db_move_hash(i,s_line(ls,j));
				}		
		}
		s_free(ls); ls = NULL;

next_bit:
		if (!config_true(C_XOVER_HASH)) return; 
		ls = s_alloc();
		for (j=0; j<100; j++) lib_createdir(file_xwork("",j));
		s_clear(ls);
		s_wild(ls,file_xwork("",-1),"*.xov");  /* Returns list without paths */
		if (s_len(ls)<=0) return;
		wmsg("Moving files to hashed structure\n");
		for (j=0; j<s_len(ls); j++) {
				db_move_xov(s_line(ls,j));
		}		
		s_free(ls); ls = NULL;
}
void db_move_xov(char *fname)
{
		char xold[BFSZ];
		char xnew[BFSZ];
		int h;

		h = atoi(fname+2);
		ncpy(xold,file_xwork(fname,-1),BFSZ-1);
		ncpy(xnew,file_xwork(fname,h),BFSZ-1);
		imsg("Moving (%s) to (%s) \n",xold,xnew);
		if (rename(xold,xnew)!=0) {
				filemsg("Failed moving (%s) to (%s) \n",xold,xnew);
				dnews_ex("Fatal error, cannot continue");
		}
}
void db_move_hash(int spl, char *fname)
{
		char xold[BFSZ];
		char xnew[BFSZ];
		int h;

		h = atoi(fname+4);
		ncpy(xold,file_spool(spl,fname),BFSZ-1);
		ncpy(xnew,file_spoolh(spl,fname,h),BFSZ-1);
		imsg("Moving (%s) to (%s) \n",xold,xnew);
		if (rename(xold,xnew)!=0) {
				filemsg("Failed moving (%s) to (%s) \n",xold,xnew);
				dnews_ex("Fatal error, cannot continue");
		}
}
void db_delete_old(void)
{
		int j;
		static Sary *ls;		

		ls = s_alloc();
	s_wild(ls,file_work(""),"big_*.tmp"); 
		for (j=0; j<s_len(ls); j++) {
		imsg("Remove old file %s\n",file_work(s_line(ls,j)));
		remove(file_work(s_line(ls,j)));
		}
	s_wild(ls,file_work(""),"local_*.itm"); 
		for (j=0; j<s_len(ls); j++) {
		imsg("Remove old file %s\n",file_work(s_line(ls,j)));
		remove(file_work(s_line(ls,j)));
		}
		s_free(ls); ls = NULL;
}
void db_init(void)		/* xmax_groups is the offset size */		  
{
		FILE *f;
#ifdef OS2
		static int max_os2;
		max_os2 = _grow_handles(60);  /* ActuHdl contains number of handles actually disponibles */
		imsg("OS2 file handles increased to %d\n",max_os2);
#endif

	db_delete_old();
	
		if (lib_exists(NEED_RESORT)) {
				remove(NEED_RESORT);
				emsg("Shutdown occurred while index was sorting do a tellnews resort_index soon\n");
				cmdq_add("resort_index"); 
		}
		del_init(atoi(config_get(C_BUCKET_ITEMS)));
	db_bucket_ranges_clear();

		db_cancel_read();
		db_reload();
		db_init_hash();
		db_init_fid();
	imsg("Init_idx()\n");
		db_init_idx();
	imsg("db_init_idx done\n");
		if (!(is_readonly()||get_expire_mode())) {
		  if (lib_exists(FILE_CHK)) {
				emsg("Database not shutdown cleanly, doing consistency check\n");
				db_check_buckets();
		emsg("db_check_index\n");
				db_check_index();
				emsg("Database consistency check completed\n");
		  } else {
		  imsg("Checking buckets on startup, not strictly necessary\n");
				db_check_buckets();
		  imsg("Checking buckets completed\n");
	  }
		  f = fopen(FILE_CHK,"w"); 
		  if (f==NULL) {filemsg("Cannot create DB check file \n"); return;}
		  fprintf(f,"Database Check File\n"); fclosep(f);
		}
	imsg("out_init() nspool=%d\n",nspool);
		out_init();
	imsg("db_correct_used()\n");
		db_correct_used();
	imsg("db_init_done()\n");
}

void db_scan_gids(char gid_isused[],int maxgid)
{
		Mrec m;
		char bf[1000];
		int f;
		int fz;
		int n;
		int len;

		if (fdbi<0) db_openi();
		if (fdbi<0) return;
		f = fdbi;
		fz = dbi_len(f);
		dbi_seek(f,0,SEEK_SET);
		for (n=0;!dbi_eof(f);n++) {
				len = dbi_read(f,bf,IDXSIZE);
				if (len==0) break;
				if (len<IDXSIZE) {
						return;
				}
				if (!frec_to_mrec(bf,n*IDXSIZE,&m)) {
						continue;
				}
				if (m.len==0) continue;
				if ((m.gid>=0) && (m.gid<(unsigned int)maxgid)) gid_isused[m.gid] = TRUE;
				else wmsg("Bad gid in database %d\n",m.gid);
		}
}
				
int db_check_index(void)
{
		/* Fixrange, set last and first files, etc ... */
		int i;
		Mrec m;
		int f;
		int g;
		char bf[IDXSIZE];
		int fz;
		int n;
		int islink;
		int ncorrupt=0;
		int lastfid = -1;
		int len;
		int *grp_end,*grp_start,*grp_n;
		int dont_fixrange = FALSE;
		int dosetend;
		int gend;

		imsg("db_check_index started\n");
	for (i=0; i<GROUP_CACHE; i++) group_grp[i] = -1;
		for (i=0; i<xmax_groups; i++) gid_last[i] = -1;
		for (i=0; i<xmax_groups; i++) gid_first[i] = -1;
		grp_end = mymalloc2(sizeof(int) * (group_n()+1),M_DB);
		grp_start = mymalloc2(sizeof(int) * (group_n()+1),M_DB);
		grp_n = mymalloc2(sizeof(int) * (group_n()+1),M_DB);


		/* Scan the DBI file, setup pointers to first and last for every group*/
		if (lib_exists(UPGRADE_DAT)) dont_fixrange = TRUE;

		if (fdbi<0) db_openi();
		if (fdbi<0) return TRUE;
		f = fdbi;
		fz = dbi_len(f);
		dbi_seek(f,0,SEEK_SET);
		for (i=0; i<group_n(); i++) {
				grp_start[i] = 0x7fffffff;
				grp_end[i] = 0;
				grp_n[i] = 0;
		}
		for (n=0;!dbi_eof(f);n++) {
				len = dbi_read(f,bf,IDXSIZE);
				if (len==0) break;
				if (len<IDXSIZE) {
						emsg("Correcting last record in index, was %d \n",len);
						dbi_seek(f,-len,SEEK_END);
						memset(bf,0,IDXSIZE);
						dbi_write(f,bf,IDXSIZE);
						break; 
				}
				if (!frec_to_mrec(bf,n*IDXSIZE,&m)) {
						ncorrupt++;
						continue;
				}
				if (m.len==0) continue;

				g = gid_to_group(m.gid);
				if ((g>=0) && (g<group_n())) {
						if (db_cancel_find(g,m.num,&islink)) continue;
						if (m.num < grp_start[g]) grp_start[g] = m.num;
						if (m.num > grp_end[g]) grp_end[g] = m.num;
						grp_n[g]++;
				}
				if (!last_boundsu(m.gid)) {
						gid_last[m.gid] = n * IDXSIZE;
						if (gid_first[m.gid]<0) gid_first[m.gid] = n * IDXSIZE;
				}
		}
		if (n==0) filemsg("check_index: Read no records from index file eof=%d last rec len=%d\n",dbi_eof(f),len);
#ifdef VMS
		if (n==0) {
				emsg("Vax, so skipping check_index\n");
				goto fail;
		}
#endif

		
		imsg("db_check_index part way\n");
	
	if (ncorrupt>0) emsg("Skipped %d corrupt records, a rebuild_index might be needed\n",ncorrupt);
		if (dont_fixrange) {
				emsg("Not fixing group ranges because upgrade is in progress\n");
		} else {
		  for (i=0; i<group_n(); i++) {
				if (grp_n[i] > 0) {
/*						if ((grp_end[i] - grp_start[i]) > 200000) {
								emsg("Range of items for group %s is too large %d-%d\n",group_name(i),grp_start[i],grp_end[i]);
								grp_start[i] = grp_end[i] - 100000;
								emsg("You probably need to do a 'rebuild_index' or trim_range overnight\n");
						}
*/
						group_setstart(i,grp_start[i]);
						dosetend = config_true(C_GROUP_SETEND);
						gend = group_getend(i);
						if (!dosetend) if (grp_end[i]<gend) if (grp_end[i]+900>gend) goto dontsetend; 
						group_setend(i,grp_end[i]);
 dontsetend:
						group_setrange(i,grp_n[i]);
				} else {
						group_setrange(i,0);
						group_setstart(i,group_getend(i)+1);
				}
		  }
		}

		imsg("db_check_index part2\n");
		group_write();
		db_write_first(); 
		db_write_last();
#ifdef VMS
fail:
#endif
		if (grp_start!=NULL) {
				myfree2(grp_start,M_DB);
				myfree2(grp_end,M_DB);
				myfree2(grp_n,M_DB);
		}
		imsg("db_check_index started-done\n");
		return TRUE;
}
int db_check_group(char *gname)
{
		Mrec m;
		int f;
		char bf[IDXSIZE];
		int fz;
		char dups[10000];
		int base = 0;
		int n;
		int k,ndup=0;
		int ncorrupt=0;
		int lastfid = -1;
		int len,i;
		int ntot,nok,match;

		for (i=0; i<10000; i++) dups[i] = FALSE;
		ntot=nok=0;
		match = group_find(gname);
		if (match<0) return TRUE;
		g_get_mrec(match,1); /* get group into cache */
		if (fdbi<0) db_openi();
		if (fdbi<0) return TRUE;
		f = fdbi;
		fz = dbi_len(f);
		dbi_seek(f,0,SEEK_SET);
		for (n=0;!dbi_eof(f);n++) {
				len = dbi_read(f,bf,IDXSIZE);
				if (len==0) break;
				if (len<IDXSIZE) break;
				if (!frec_to_mrec(bf,n*IDXSIZE,&m)) {
						ncorrupt++;
						continue;
				}
				if (match==gid_to_group(m.gid)) {
						if (base==0) base = m.num;
						k = m.num - base;
						if ((k>0) && (k<10000)) {
								if (dups[k]) {emsg("Duplicate %d\n",m.num); ndup++;}
								dups[k] = TRUE;
						}
						ntot++;
						if (m.len>0) nok++;
						if (g_get_mrec(match,m.num)==NULL) {
						  if (!db_iscancel(match,m.num)) {
								if (m.len>0) {
								  emsg("Missing  %x %x [%d/%d] %s:%d len %d\n",n*IDXSIZE,m.jump,nok,ntot,gname,m.num,m.len);
								} else {
								  imsg("CanDone %x %x [%d/%d] %s:%d len %d\n",n*IDXSIZE,m.jump,nok,ntot,gname,m.num,m.len);
								}
						  } else {
								imsg("CanPend %x %x [%d/%d] %s:%d len %d\n",n*IDXSIZE,m.jump,nok,ntot,gname,m.num,m.len);
						  }
						} else 
						  imsg("OK  %x %x [%d/%d] %s:%d len %d\n",n*IDXSIZE,m.jump,nok,ntot,gname,m.num,m.len);
				}
		}
		emsg("nduplicates %d\n",ndup);
		return TRUE;
}
void db_trim_range(void)
{
		/* Fixrange, set last and first files, etc ... */
		Mrec m;
		char bf[IDXSIZE];
		int n;
		int g;
		int len;

		if (fdbi<0) db_openi();
		if (fdbi<0) return ;
		dbi_seek(fdbi,0,SEEK_SET);
		for (n=0;!dbi_eof(fdbi);n++) {
				len = dbi_read(fdbi,bf,IDXSIZE);
				if (len==0) break;
				if (len<IDXSIZE) break;
				if (!frec_to_mrec(bf,n*IDXSIZE,&m)) continue;
				if (m.len==0) continue;
				g = gid_to_group(m.gid);
				if ((g>=0) && (g<group_n())) {
				  if (m.num < group_getend(g)-15000) {
						imsg("Removing item %s:%d\n",group_name(g),m.num);
						m.len = 0;
						m_update(&m);							
						dbi_seek(fdbi,(n+1)*IDXSIZE,SEEK_SET);
				  }
				}
		}
}
void db_sort_lastmb(void);
int db_sortsz(void)
{
		static int sz;
		if (sz>0) return sz;
		if (sz==0) sz = atoi(config_get(C_SORT_SIZE));
		if (sz>0) return sz;
		if (fdbi<0) db_openi();
		if (sz==0) {	
				sz = (fdbi_len/IDXSIZE)/15;
		} 
		if (sz<10000) sz = 10000;
		imsg("Continuous sort size is %dk\n",(sz*IDXSIZE)/1024);
		return sz;

}
void db_sort_bk(void)
{
		static int xtime;
		static int lastlen;
		int freq = atoi(config_get(C_SORT_FREQ));

		if (insrt) { db_srt_finish(); return;}
		if (freq==0) freq = 60;
		if (is_readonly()) return;
		if (!config_true(C_SORT_LASTMB)) return;
		if ((lib_gmtime() - xtime)>freq*60) {
				xtime = lib_gmtime();
				db_sort_lastmb();
				if (fdbi_len>0) lastlen = fdbi_len;
		}
		if ((fdbi_len-lastlen)>db_sortsz()*IDXSIZE) 
		  if ((lib_gmtime()-xtime)) {
				xtime = lib_gmtime();
				db_sort_lastmb();
				if (fdbi_len>0) lastlen = fdbi_len;
		}
}
int db_sort_start(void)
{
		if (insrt) return TRUE;
		db_sort_lastmb();
		return TRUE;
}
static int srtbase;
static int *srtgid;
static int srtupto;
void db_sort_lastmb(void)
{
		Mrec m;
		char bf[IDXSIZE];
		int n,blockn;
		int i,gid;
		int len;
		int gap;
		int max=db_sortsz();
		Mrec *mb;
		FILE *f;
		int nmb;
		int srtend;

		if (insrt) db_srt_finish();
		if (fdbi<0) db_openi();
		if (fdbi<0) return ;

		imsg("Starting sort lastmb\n");
		mb = mymalloc2(sizeof(*mb)*max,M_DB);
		if (srtgid==NULL) srtgid = mymalloc2(xmax_groups * sizeof(int),M_DB);
		srtbase = (fdbi_len/IDXSIZE) - max;
		if (srtbase<0) srtbase = 0;
		imsg("Starting sort lastmb from %d\n",srtbase*IDXSIZE);
		basep = srtbase*IDXSIZE;
		dbi_seek(fdbi,srtbase*IDXSIZE,SEEK_SET);
		nmb=0;
		srtend = 0;
		for (n=0;!dbi_eof(fdbi) && (nmb<max);n++) {
				len = dbi_read(fdbi,bf,IDXSIZE);
				if (len==0) break;
				if (len<IDXSIZE) break;
				srtend++;
				if (!frec_to_mrec(bf,nmb*IDXSIZE+basep,&m)) continue;
				if (m.len==0) continue;
				mb[nmb++] = m;
		}
		gap = srtend - nmb;
		if (nmb>2) qsort(mb,nmb,sizeof(Mrec),resort_cmp);
		imsg("Writing new data %d records\n",nmb);
		/* Write the sorted data back to this block*/
		blockn = n;
		baseend = basep + blockn*IDXSIZE;
		
		dbi_seek(fdbi,srtbase*IDXSIZE,SEEK_SET);
		for (i=0; i<nmb; i++) {
				if (mb[i].gid>=(unsigned int) xmax_groups) continue;
				if (mb[i].gid<0) continue;
				if (i+1>=blockn) goto endgrp;
				if (mb[i+1].gid==mb[i].gid) {
						mb[i].jump = (i+1)*IDXSIZE+basep;
				} else {
endgrp: 				mb[i].jump = -1;
						/* imsg("Moved %d to %d\n",resort[i].gid,fdbs_len);*/
				}
				mrec_to_frec(&mb[i],bf);
				if (dbi_write(fdbi,bf,IDXSIZE)!=IDXSIZE) filemsg("lastmb update failed\n");
		}
		memset(&m,0,sizeof(m));
		m.len = 0;
		m.jump = -1;
		m.gid = 0;
		m.num = 0;
		imsg("Filling in gap of %d records\n",gap);
		mrec_to_frec(&m,bf);
		for (i=0; i<gap; i++) {
				if (dbi_write(fdbi,bf,IDXSIZE)!=IDXSIZE) filemsg("lastmb update failed\n");
		}
		
		imsg("Finding first gid's\n");
		/* Scan and find the first entry for each GID.*/
		gid = -2;
		for (i=0; i<xmax_groups; i++) srtgid[i] = -1;

		for (i=0; i<nmb; i++) {  
		  if (mb[i].gid!=(unsigned int) gid) {
				if (gid>=0) {
						gid_last[gid] = (i-1)*IDXSIZE+basep;
				}
				gid = mb[i].gid;
				if ((gid<0) || (gid>=xmax_groups)) gid = 0;
				if ((gid>=0) && (gid<xmax_groups))
						srtgid[gid] = i*IDXSIZE+basep;
		  }
		  if (i+1==nmb) {
				gid_last[gid] = i*IDXSIZE+basep;
		  }
		}
		imsg("Now scan any still left and set gid_first correctly\n");
		f = fopen(NEED_RESORT,"w");
		if (f!=NULL) {
				fprintf(f,"Need resort");
				fclosep(f);
		}
		insrt = TRUE;
		srtupto = 0;
		myfree2(mb,M_DB);
		db_clear_cache("sortlastmb");
}
int db_jumpin(int gid, int jump, int lastjump)
{
		if (lastjump>=basep) return jump;
		if ((jump>=basep) && (jump<baseend)) {
				if ((gid<0) || (gid>=xmax_groups)) return jump;
				imsg("Sort in progress, patching jump %d:%d->%d\n",gid,jump,srtgid[gid]);
				return srtgid[gid];
		}
		return jump;
}
void db_srt_flush(void)
{
		if (insrt) for (;db_srt_finish();) ;
}
int db_srt_finish(void)
{
		int n,j=0;
		int len,i;
		char bf[IDXSIZE];
		Mrec m;

		imsg("Scanning for inbound pointers %d/%d \n",srtupto,srtbase);
		if (fdbi<0) db_openi();
		if (fdbi<0) return FALSE;
		dbi_seek(fdbi,(srtupto)*IDXSIZE,SEEK_SET);
		for (n=srtupto;!dbi_eof(fdbi) && (n<srtbase) ;n++,j++) {
				if (j>50000) {
						srtupto = n;
						return TRUE;
				}
				len = dbi_read(fdbi,bf,IDXSIZE);
				if (len==0) break;
				if (len<IDXSIZE) break;
				if (!frec_to_mrec(bf,n*IDXSIZE,&m)) continue;
				if (m.gid>=(unsigned int) xmax_groups) continue;
				if ((m.jump>=basep) && (m.jump<baseend)) {
				  if (srtgid[m.gid]>=0) {
						m.jump = srtgid[m.gid];
						srtgid[m.gid] = -1;
						m_update(&m);							
						dbi_seek(fdbi,(n+1)*IDXSIZE,SEEK_SET);
				  } 
				}
		}
		insrt = FALSE;
		if (!dbg_open()) goto skip;
		if (fdbg<0) goto skip;
		/* set gid_first for any that do start in sorted block */
		for (i=0; i<xmax_groups; i++) {
				if (srtgid[i]>=0) {
						gid_first[i] = srtgid[i];
/*						imsg("gid_first [%d] = %x\n",i,srtgid[i]);*/
						dbf_seek(fdbg,i*4,SEEK_SET);
						dbf_write(fdbg,&gid_first[i],4);
				}
		}
		dbf_close(fdbg);
		fdbg = -1;
skip:
		db_write_last();
		myfree2(srtgid,M_DB); srtgid = NULL;
		remove(NEED_RESORT);
		db_clear_cache("resort_Finish");
		slave_dbflush();
		imsg("lastmb sort completed\n");
		return FALSE;
}
void db_init_idx(void)	/* xmax_groups is the offset size */		  
{
		int i;
		for (i=0; i<GROUP_CACHE; i++) group_grp[i] = -1;
		for (i=0; i<xmax_groups; i++) gid_last[i] = -1;
		imsg("db_init_idx: clearing gid_first[]\n");
		for (i=0; i<xmax_groups; i++) {
				gid_first[i] = -1;
		}

		/* Read dblast.idx */ 
		imsg("db_read_last()\n");
		db_read_last();

		/* Find which is the current bucket */
		imsg("db_openi()\n");		 
		db_openi();
}						
void db_init_fid(void)
{
		DBN tmp;
		FILE *f;
		int lastfid = -1;
		char fname[BFSZ];
		int i;
		strcpy(fname,FILE_FID);
		for (i=0; i<MAX_DBN; i++) {
				dbn[i].fid = i;
				dbn[i].inuse = FALSE;
				dbn[i].used = 0;
				dbn[i].free = 0;
				dbn[i].disk = 0;
				dbn[i].cur = FALSE;
		}
		f = fopen(fname,F_READ);
		if (f==NULL) {
				filemsg("Couldn't read (%s), searching for existing buckets\n",fname); 
				db_check_buckets();
				goto skip_read;
		}
		for (;!feof(f);) {
				if (fread(&tmp,sizeof(DBN),1,f)<1) break;
				dmsg("db: Read dbn[%d] data used %d\n",tmp.fid,tmp.used);
				if (tmp.fid<0) break;
				if (tmp.fid>=MAX_DBN) break;
				dbn[tmp.fid] = tmp;
				if (tmp.cur) lastfid = tmp.fid;
		}
		fclosep(f);
skip_read:;
		if (lastfid<0) {
				lastfid = db_next_fid();
		}
		strcpy(fdbn_name,fid_name(lastfid));
		fdbn_fid = lastfid;
}
int db_check_fid(void)
{
		DBN tmp;
		FILE *f;
		int lastfid = -1;
		char fname[BFSZ];
		int i;
		static int last_len;

		if (!is_readonly()) return TRUE;
		strcpy(fname,FILE_FID);

		f = fopen(fname,F_READ);
		if (f==NULL) {
				filemsg("Couldn't read (%s), will try later\n",fname); 
				return TRUE;
		}

		if ((int) file_length(fileno(f))==last_len) {
				fclosep(f);
				return TRUE;
		}
		last_len = file_length(fileno(f));
		imsg("Reading new FID data from master\n");
		for (i=0; i<MAX_DBN; i++) {
				dbn[i].fid = i;
				dbn[i].inuse = FALSE;
				dbn[i].used = 0;
				dbn[i].free = 0;
				dbn[i].disk = 0;
				dbn[i].cur = FALSE;
		}
		for (;!feof(f);) {
				if (fread(&tmp,sizeof(DBN),1,f)<1) break;
				if (tmp.fid<0) break;
				if (tmp.fid>=MAX_DBN) break;
				dbn[tmp.fid] = tmp;
				if (tmp.cur) lastfid = tmp.fid;
		}
		fclosep(f);
	if (is_reader()) slave_changes_read();
		return TRUE;
}
int db_write_fid(void)
{
		int i;
		FILE *f;
		int nfail=0;
		if (is_readonly()) {
				imsg("slave: not writing FID\n");
				return TRUE;
		}
again:;
	remove(FILE_FID);
		f = fopen(FILE_FID,F_WRITE);
		if (f==NULL) {
				nfail++;
				if (nfail<10) {
						lib_sleep(1);
						goto again;
				}
				filemsg("FATAL Cannot append to {%s} \n",FILE_FID); 

				return FALSE;
		}
		for (i=0; i<MAX_DBN; i++) {
				if (dbn[i].inuse) {
				  if (fwrite(&dbn[i],sizeof(DBN),1,f)<1) {
						filemsg("Cannot write %s\n",FILE_FID);
						fclosep(f);
						return FALSE; 
				  }
				}
		}
		fclosep(f);
	slave_bucket_clear();
		return TRUE;
}
int item_open_append(char *fname)
{
	int r;
		if (is_readonly()) {
				return dbf_open(fname,"r",10,8000);
		}
		r = dbf_open(fname,"a+",10,8000);
	if (config_true(C_THREAD_WRITE)) {
		if (r>=0) dbf_usethread(r);
	}
	return r;
}
int item_open_read(char *fname)
{
		int r;
		stat_start(X_DBOPEN);
		r = dbf_open(fname,"r",10,8000);
		stat_end(X_DBOPEN);
		return r;
}
/* test caching routines */
int dbg_open(void)
{
		int i;
		int r;
		int pnt = -1;
		if (is_readonly()) {
				if (fdbg<0) fdbg = dbf_open(FILE_FIRST,"r",5,4000);
		} else {
				if (fdbg<0) fdbg = dbf_open(FILE_FIRST,"r+",5,4000);
		}

		if (fdbg<0) {
				if (errno==ENOENT) {
						if (is_readonly()) return FALSE;
						r = dbf_open(FILE_FIRST,"w+",5,4000); 
						if (r<0) {
								filemsg("FATAL: Could not create groups file\n");
								return FALSE;
						}
						for (i=0; i<xmax_groups; i++) {
								dbf_write(r,&pnt,4);
						}
						fdbg = r;		
				} else {
						filemsg("Cannot open %s\n",FILE_FIRST);
						return FALSE;
				}
		}
		return TRUE;
}
int idx_open(char *fname)
{
		int r;
		int pnt=0;
		char *mde="r+";
		if (is_readonly()) mde="r";
		if (stopdbi) {
				mde="r";
		}
		errno = 0;
		r = dbi_open(fname,mde,200,2000);
		if (r<0) {
				if (errno==ENOENT) {
						if (is_readonly()) {emsg("slave: Can't find index\n"); return -1;}
						r = dbi_open(fname,"w+",200,2000); 
				}
		}
		return r;
}
int frec_to_mrec(char *bf, int pnt, Mrec *m)
{
		static int nerr;
		unsigned int xlen;
	int r;
		unsigned short xgid;
		unsigned int add;
		m->ipnt = pnt;
		memcpy(&m->fid,bf,2);
		memcpy(&xgid,bf+2,2);
		memcpy(&m->num,bf+4,4);
		memcpy(&m->pnt,bf+8,4);
		memcpy(&xlen,bf+12,4);
		memcpy(&m->jump,bf+16,4);

		m->len = xlen & 0x0fffffff;
		if (m->len==0x0fffffff) m->len = -1;
		add = xlen & 0xf0000000;
		if (add==0xf0000000) add = 0;
		m->gid	= ((unsigned int) xgid) | (add >> 12);
		if (m->fid < 0) {r=1; goto failed;}
		if (m->fid >= MAX_DBN) {r=2; goto failed;}
		if (m->gid < 0) {r=3; goto failed;}
		if (m->gid >= (unsigned int) xmax_groups) {r=4; goto failed;}
		if (m->num < 0) {r=5; goto failed;}
		if (m->pnt < 0) {r=6; goto failed;}
		if (m->len < 0) {r=7; goto failed;}
		if (m->len > 10000000) {r=8; goto failed;}
		return TRUE;
failed: 
		if (nerr<100) {
				imsg("Corrupt [%x] fid=%x gid=%x num=%x pnt=%x len=%x r=%d\n"
						,pnt,m->fid,m->gid,m->num,m->pnt,m->len,r);
		}		
		nerr++;
		return FALSE;
}
void mrec_to_frec(Mrec *m,char *bf)
{
		int g = gid_to_group(m->gid);
		if (g<0) {
				memcpy(bf,to_idx(
				m->fid
				,0
				,m->num, m->pnt, 0, m->jump),IDXSIZE);
				return;
		} 
				memcpy(bf,to_idx(
				m->fid
				,gid_to_group(m->gid)
				,m->num, m->pnt, m->len, m->jump),IDXSIZE);
}
int g_show_dir(int chan)
{
	int i;
	char bf[BFSZ];

	for (i=0; i<GROUP_CACHE; i++) {
		if (group_dir[i]==NULL) continue; 
		if (group_grp[i]>=0) {
			lprintf(bf,200,"[%d] lastused=%d delay=%d nitems=%d %.100s\r\n",
			i,xtime_now-group_lastused[i],group_delay[i],group_nitems[i],group_name(group_grp[i]));
			chan_write_s(chan,bf);
		}
	}
	return TRUE;
}
int g_cache_lru(int emptyok, int skip)
{
		static int upto;
	int i;
	int oldest = -1;
	int oldest_age = xtime_now;

	if (emptyok) {
		for (i=0; i<GROUP_CACHE; i++) {
			if (group_dir[i]==NULL) return i;
		}
	}
	
	/* Find the oldest entry, if it's older than 10 minutes use it */
	oldest_age = xtime_now; oldest = -1;
	for (i=0; i<GROUP_CACHE; i++) {
		if (i==skip) continue;
		if (group_dir[i]==NULL) continue;
		if (group_lastused[i] < oldest_age) {
			oldest_age = group_lastused[i];
			oldest = i;
		}
	}
	if (oldest>=0) if ((xtime_now - oldest_age)>60*10) return oldest; 

	/* Find the oldest entry that took 0 seconds to load and has less than 10,000 items  */
	oldest_age = xtime_now; oldest = -1;
	for (i=0; i<GROUP_CACHE; i++) {
		if (i==skip) continue;
		if (group_dir[i]==NULL) continue;
		if (group_delay[i]==0) if (group_nitems[i]<6000) if (group_lastused[i] < oldest_age) {
			oldest_age = group_lastused[i];
			oldest = i;
		}
	}
	if (oldest>=0)	return oldest; 
	
	/* If all else fails pick one at random */
	upto++;
	if (upto==skip) upto++;
		if (upto>=GROUP_CACHE) upto=0;

	if (!emptyok) {
		for (i=upto; i<GROUP_CACHE; i++) if (group_dir[i]!=NULL) return i;
		for (i=0; i<GROUP_CACHE; i++) if (group_dir[i]!=NULL) return i;
	}
		return upto;
}
int g_cache_lru_old(void)
{
		static int upto;
	int i;
	int oldest = -1;
	int oldest_age = xtime_now;
		upto++;
		if (upto>=GROUP_CACHE) upto=0;

	/* Find any unused entry  */
	for (i=0; i<GROUP_CACHE; i++) {
		if (group_dir[i]==NULL) return i;
	}

	/* Find the oldest one that took 0 seconds to load */
	for (i=0; i<GROUP_CACHE; i++) {
		if (group_delay[i]==0) {
			if (group_lastused[i] < oldest_age) {
				oldest_age = group_lastused[i];
				oldest = i;
			}
		}
	}
	if (oldest>=0) {
		imsg("group_read [%d] entry was %d seconds old\n",oldest,xtime_now-oldest_age);
		return oldest;
	}

	/* Find a group index that took 0 seconds to load to free */
	for (i=upto; i<GROUP_CACHE; i++) {
		if (group_delay[i]==0) {
			upto = i;
			return upto;
		}
	}
	for (i=0; i<GROUP_CACHE; i++) {
		if (group_delay[i]==0) {
			upto = i;
			return upto;
		}
	}
		return upto;
}
/* find first, find num, list all items (including downloading msg)*/
int g_clear(char *reason)
{		
		int i;
		int z;
		Mrec *m,*mnext;
		imsg("Group clear cache\n");
		for (i=0; i<GROUP_CACHE; i++) {
		if (group_grp[i]>=0) imsg("g_free (%d) age=%d %s g_clear %s\n",i,xtime_now-group_lastused[i],reason,group_name(group_grp[i]));
				m = group_dir[i];
			group_dir_next[i] = NULL;
				for (z=0;m!=NULL;m=mnext,z++) {
						if (z>9000000) {emsg("loop detected \n"); dnews_ex("g_clear");}
						mnext = m->next;
						myfree2(m,M_DIR); nm--;
				}
				group_dir[i] = NULL;
				group_grp[i] = -1;
		}
		nm = 0;
		imsg("Group clear cache - done\n");
		g_list_clear(-1);
		return TRUE;
}
int g_find_next(int grp, int num)
{
		int i;
		Mrec *m;
	int zmax;
		int c = g_get_group(grp);
		if (c<0) return -1;
		m = group_dir_next[c];
	zmax = max_items();

		if (m!=NULL) if (m->num<=num) for (i=0;m!=NULL && i<zmax; m=m->next,i++) {
						if (m->num>num) return m->num;
		}
		m = group_dir[c];
		for (i=0;m!=NULL && i<zmax; m=m->next,i++) {
						if (m->num>num) return m->num;
		}
		return -1;
}
int g_find_last(int grp, int num)
{
		int i;
		Mrec *m;
		int last = -1;
	int zmax;
		int c = g_get_group(grp);
		if (c<0) return -1;
		m = group_dir[c];
		last = -1;
	zmax = max_items();
		for (i=0;m!=NULL && i<zmax; m=m->next,i++) {
						if (m->num>=num) return last;
						last = m->num;
		}
		return last;
}
int g_list_findnum(int grp, int *xfirst, int *xlast)
{
		int i;
		int n=0;
		int first,last;
		Mrec *m;
	int zmax;
		int c = g_get_group(grp);
		first = 0xfffffff;
		last = 0;
		if (c<0) return 0;
		m = group_dir[c];
	zmax = max_items();
		for (i=0;m!=NULL && i<zmax; m=m->next,i++) {
				if (m->num < first) first = m->num;
				if (m->num > last) last = m->num;
				n++;
		}
		if (n>0) {
				*xfirst = first;
				*xlast = last;
		}
		return n;
}
static Mrec *g_list;
static int g_donefirst;
static int g_list_c;
static int list_save;
static Mrec *list_m;
void g_list_save(void)
{
	list_m = g_list;
	list_save = g_donefirst;
}
void g_list_restore(void)
{
	g_list =  list_m ;
	g_donefirst = list_save; 
}
void g_list_clear(int x)
{
		if (g_list_c==x) g_list = NULL;
		if (x<0) g_list = NULL;
	list_m = NULL;
}
int g_list_start(int grp)
{
		Mrec *m;
		int n=0;
		int c = g_get_group(grp);
		g_list_c = c;
		if (c<0) {
				g_list = NULL;
		} else {
				g_list = m = group_dir[c];
		}
		if (!group_iscached2(grp)) {
				g_donefirst = FALSE;
				return group_getstart(grp);
		}
		g_donefirst = TRUE;
		if (g_list==NULL) return -1;
		return m->num;
}
int g_list_search(int from)
{
	int n=0;
		if (g_list==NULL) return -1;

		g_donefirst = TRUE;
	for (; (g_list!=NULL) ;) {
		if (g_list->num>=from) break;
		g_list = g_list->next;	
	}
		if (g_list==NULL) return -1;
	return g_list->num;
}

int g_list_next(void)
{
		if (g_list==NULL) return -1;
		if (g_donefirst) {
				g_list = g_list->next;
		}
		g_donefirst = TRUE;
		if (g_list==NULL) return -1;
		return g_list->num;
}
int cmd_du(int chan, int *ngrps, char *wild, int *nitems)
{
/*		Find list of 'gid's that match this wild card,
		Scan thru index, and add up size of each item.
*/
		int ng = group_n();
		int i,n;
		char *matched;
		Mrec m;
		char bf[IDXSIZE];
	char xx[BFSZ];
		int g;
	int pitems[20];
	double pnibs[20];
		int len;
		int tot=0;
	int p;

	for (i=0; i<20; i++) {
		pitems[i] = 0;
		pnibs[i] = 0;
	}
		*nitems = 0;
		*ngrps = 0;
		matched = mymalloc2(ng,M_DB);
		for (i=0; i<ng; i++) matched[i] = FALSE;
		
		for (i=0; i<ng; i++) {
		   if (group_match(wild,group_namelc(i))) {
				matched[i] = TRUE;				
				(*ngrps)++;
		   }
		}

		if (fdbi<0) db_openi();
		if (fdbi<0) goto done;
		dbi_seek(fdbi,0,SEEK_SET);
		for (n=0;!dbi_eof(fdbi);n++) {
				len = dbi_read(fdbi,bf,IDXSIZE);
				if (len==0) break;
				if (len<IDXSIZE) break;
				if (!frec_to_mrec(bf,n*IDXSIZE,&m)) {
						continue;
				}
				if (m.len==0) continue;
				g = gid_to_group(m.gid);
				if ((g<0) || (g>=ng)) continue;
				if (matched[g]) {
						tot += ((m.len+50)/100);
						(*nitems)++;
			p = dbn[m.fid].pile;
			if ((p>=0) && (p<20)) {
				pitems[p] ++;
				pnibs[p] += ((m.len+50)/100);
			}
				}
		}
done:;
		myfree2(matched,M_DB);

	for (i=0; i<20; i++) {
		if (pitems[i]>0) {
			lprintf(xx,BFSZ-1,"Pile %d, %d items, %d MB\r\n",i,pitems[i],(int) (pnibs[i]/10000));
			chan_write_s(chan,xx);
		}
	}
		return tot;
}

int cmd_du_all(int chan)
{
	FILE *f;
	char bf[BFSZ];
	int i;
	int n;
		Mrec m;
	int g;
	int len;
	int max = group_n();
	int ng=max;
	double *tot_bytes;
	int *tot_items;

	tot_bytes = mymalloc2(max*sizeof(double),M_DB);
	tot_items = mymalloc2(max*sizeof(int),M_DB);
	memset(tot_bytes,0,max*sizeof(double));
	memset(tot_items,0,max*sizeof(int));

		if (fdbi<0) db_openi();
		if (fdbi<0) goto done;
		dbi_seek(fdbi,0,SEEK_SET);
		for (n=0;!dbi_eof(fdbi);n++) {
				len = dbi_read(fdbi,bf,IDXSIZE);
				if (len==0) break;
				if (len<IDXSIZE) break;
				if (!frec_to_mrec(bf,n*IDXSIZE,&m)) {
						continue;
				}
				if (m.len==0) continue;
				g = gid_to_group(m.gid);
				if ((g<0) || (g>=ng)) continue;
		tot_bytes[g] += m.len;
		tot_items[g]++;
		}
done:;

	
	f = fopen(file_work("diskuse.all"),"w");
	if (f==NULL) goto ret;
	for (i=0; i<ng; i++) {
		fprintf(f,"%8.2fmb %8d %s\n",tot_bytes[i]/1000000,tot_items[i],group_name(i));
	}
	fclose(f);
	lprintf(bf,BFSZ-1,"Written diskuse summary to (%s)\r\n",file_work("diskuse.all"));
	chan_write_s(chan,bf);
ret:
		myfree2(tot_bytes,M_DB);
	myfree2(tot_items,M_DB);
	return TRUE;
}
/*-------------------------------------------------------------------*/
/*
Xstatic DBN dbn[MAX_DBN];
Xstatic char dbn_isdel[MAX_DBN];
Xstatic int dbn_isbusy[MAX_DBN];
Xstatic int dbn_purged[MAX_DBN];
Xstatic int dbn_pending[MAX_DBN];
typedef struct {int fid,disk,used,free,inuse,cur,found,pile,date,reserved[5];} DBN;
*/
/*-------------------------------*/
/*enum {MSTATE_FAIL = -1, MSTATE_NORMAL, MSTATE_DONE};*/
static int npfound;
double db_pile_used(int p)
{
		int i;
		double tot=0;
		npfound = 0;
		for (i=0; i<MAX_DBN;i++) {
		if (dbn_istrim[i]) continue;
		if (dbn_piledel[i]) continue;
				if (dbn[i].pile==p) {
						tot += dbn[i].used; 			
						npfound++;
				}
		}
		return tot;
}
int db_pile_over(int p)
{
		double share,used,actual;
		share = expire_pile_get(p);
		if (share<0) return FALSE;
		used = db_pile_used(p);
		actual = used/((double) db_total_space_left()*1000000.0);
		dmsg("pile: pile_over Share %g actual %g\n",share,actual);
		if (npfound==1) if (share>0) return FALSE;
		if (share<actual) {
		dmsg("pile: pile_over true %g<%g used=%g \n",share,actual,used);
		return TRUE;
	}
		return FALSE;
}
int expire_pile_sort(int x);
void pile_remove_days(int p);
char *db_pile_status(void);
int db_show_pile(int chan)
{
	chan_write_s(chan,db_pile_status());
	return TRUE;
}
int expire_pile_unread(int i);
int expire_unread_pile(void);
char *db_pile_status(void)
{
		int p;
		char buse[BFSZ];
		char bdays[BFSZ];
	char nopenstr[BFSZ];
		double days;
		char bf[BFSZ];
	static char outbf[5000];
	int isunread;
		double share,used,actual;
	int u_hist,u_xov,u_dbi;
	int issort;
	strcpy(outbf,"");
		u_hist = hist_diskspace() * 2;
		u_dbi = (fdbi_len*2)/1000;
		if (config_true(C_HISTORY_MOVED)) {
				u_hist = u_dbi = 0;
		}
	u_xov = 0;
		if (!config_true(C_XOVER_MOVED)) {
		u_xov = xov_total_get();		  
	}
	
	lprintf(bf,BFSZ-1,"Disk space %dMB - history*2=%dMB,xover=%dMB,dbi.idx*2=%dMB = %dMB for piles\r\n"
		,db_total_space()
		,u_hist/1000,u_xov/1000,u_dbi/1000
		,db_total_space_left());
	ncat(outbf,bf,4000);
		for (p=0; p<expire_npiles(); p++) {
		dmsg("pile_status %d\n",p);
				share = expire_pile_get(p);
				days = expire_pile_days(p);
				used = db_pile_used(p);
				actual = used/((double) db_total_space_left()*1000000.0);
				if (share<0) {
						share=1;
						lprintf(buse,BFSZ-1,"%3.1f/(nolimit) ",actual*share*100.0/(share+0.00001));
				} else {
/*culprit*/ 			lprintf(buse,BFSZ-1,"%3.1f/%3.1f%% ",actual*share*100.0/(share+0.00001),share*100.0);
				}
		issort = expire_pile_sort(p);
		isunread = expire_pile_unread(p);
				lprintf(bdays,BFSZ-1,"days=%g ",days);
				lprintf(nopenstr,BFSZ-1,"nopen=%d ",expire_pile_nopen(p));
				if (days<0) strcpy(bdays,"      ");
				lprintf(bf,BFSZ-1,"  Pile %d, %s%8.0fMB, %s%s%s%s%s\n",
						p,buse
						,used/1000000.0
			,issort ? "Sort " : ""
			,isunread ? "Unread " : ""
			,expire_pile_nopen(p)>1 ? nopenstr : ""
						,bdays,expire_pile_groups(p));
		ncat(outbf,bf,4000);
		if (strlen(outbf)>2000) break;
		}
	dmsg("Pile status finished\n");
		return outbf;
}
int pile_remove_oldest(int p, int istrim)
{
		int i;
		int cdate=lib_gmtime();
		int oldest= -1;
		/* Find oldest entry, delete it, remember it for later expiring */
		/* don't reuse it until index has been processed */
		for (i=0; i<MAX_DBN;i++) {
				if (dbn[i].inuse) if (dbn[i].pile==p) {
						if (dbusy(i)) continue;
						if (dbn[i].cdate<cdate) {
								oldest = i;
								cdate = dbn[i].cdate;
						}
				}
		}
		i = oldest;
		if (i<0) {
				if (p>0) {
			if (isregid) emsg("Couldn't find an oldest one to remove %d\n",p);
			else emsg("Couldn't find an oldest one to remove PILE %d %d\n",p,cdate);
		}
				return FALSE;
		}
		dblog("Expire remove oldest age %d bkt=%d pile=%d\n",dbn[i].cdate,i,dbn[i].pile);
		pile_remove_one(p,i,istrim,"Deleting oldest bucket");
		return TRUE;
}
int pile_remove_one(int p, int i, int istrim, char *reason)
{
	int days;
		if (istrim) {
				out_closeall();
				dbn_trim(i); /* Record and don't reuse */
				db_clear_cache("remove_one");
		} else {
				dbn_piledel[i] = TRUE; /* Record list of buckets we are deleting */
		}

		days = (time(NULL)-dbn[i].cdate)/(60*60*24);
		/* Delete entry 'i' */ 
		imsg("Cyclic, deleting oldest bucket %d for pile %d \n",i,p);
		dbn[i].inuse = FALSE;
		dbn_pending[i] = FALSE;
		diskuse_add(-dbn[i].used); 
		dbn[i].used = 0;
		dbn[i].free = 0;
		dbn[i].cdate = 0; /* lib_gmtime(); */
		errno = 0;
		remove(del_name(i));
	stat_start(X_DBDEL);
	dblog("Pile remove one %s age=%d %s\n",fid_name(i),days,reason);
		if (remove(fid_name(i))!=0) { /*ISCLOSED*/
		stat_end(X_DBDEL);
				if (errno!=ENOENT) {
						filemsg("Remove {%s} failed\n",fid_name(i));
						remove_later(i);
						dbn_pending[i] = TRUE;
						dbn[i].inuse = TRUE;
						return TRUE;
				} else {
						fimsg("Can't remove (%s)\n",fid_name(i));
						return TRUE;
				}
		} else stat_end(X_DBDEL);
		imsg("Successfully deleted (%s) %d\n",fid_name(i),lib_exists(fid_name(i)));
		return TRUE;

}
int db_get_age(int bkt)
{
		if (dbn_piledel[bkt]) return 0;
		if (dbn_wasused[bkt]) return 0;
		if (dbn_istrim[bkt]) return 0;
	if (dbn[bkt].inuse) {
		return dbn[bkt].cdate;
	}
	return 0;
}
void pile_remove_days(int p)
{
		int i;
	int young=0,youngi= -1;
		int cdate;
		int oldest= -1;
		int n=0;
		for (i=0; i<MAX_DBN;i++) {
				if (dbn[i].inuse) if (dbn[i].pile==p) n++;
		} 
		if (n<3) return; /* Not enough piles for date to be accurate */
		cdate = (int) (lib_gmtime() - expire_pile_days(p)*3600*24);

		for (i=0; i<MAX_DBN;i++) {
				if (dbn[i].inuse) if (dbn[i].pile==p) {
			if (dbn[i].cdate!=0) if (dbn[i].cdate>young) {
				young = dbn[i].cdate;
				youngi = i;
			}
		}
	}
		for (i=0; i<MAX_DBN;i++) {
				if (dbn[i].inuse) if (dbn[i].pile==p) {
			if (i==youngi) {
				imsg("Not deleting youngest bucket for this pile %d\n",i);
				continue;
			}
						if (dbusy(i)) continue;
						if (dbn[i].cdate!=0) {
				if (((cdate-dbn[i].cdate)/(3600*24)) > 4000) {
					imsg("Not deleting bucket of 'odd' age [%d] %d\n",i,((cdate-dbn[i].cdate)/(3600*24)));	
					continue;
				}
				if (dbn[i].cdate<cdate) {
									imsg("Deleting pile which is older than %g days (%d days)\n",expire_pile_days(p),((cdate-dbn[i].cdate)/(3600*24)));
								dblog("Expire based on age %d %d bkt=%d pile=%d\n",dbn[i].cdate,cdate,i,dbn[i].pile);
								pile_remove_one(p,i,FALSE,"cdate");
							}
			}
				}
		}
		
}

#define FILE_DB_TRIM file_work("dbtrim.dat")
void db_reuse_buckets()
{
	int i;
	int r=0;

	for (i=0; i<MAX_DBN; i++) dbn_piledel[i] = FALSE;
	/* also delete the ones that we have already deleted on the fly */
again:
	if (remove(FILE_DB_TRIM)!=0) {
		if (errno!=ENOENT) {
			r++;
			lib_sleep(1);
			if (r>10) {
				dnews_ex("Unable to delete dbtrim.dat, report to support@netwinsite.com");
			}
			goto again;
		}
	}
	for (i=0; i<MAX_DBN; i++) dbn_istrim[i] = FALSE;
}

int db_expire_pile(void) /* return -1=end, 0 = continue, 1 = done */
{
		int i;
		int p;

		out_closeall();
		for (i=0; i<MAX_DBN; i++) dbn_piledel[i] = FALSE;

		/* also delete the ones that we have already delete on the fly */
		if (get_expire_mode()) dbn_trim_load();
		for (i=0; i<MAX_DBN; i++) if (dbn_istrim[i]) dbn_piledel[i] = TRUE; /* previously deleted items */
		if (remove(FILE_DB_TRIM)==0) {
			imsg("db_expire_pile - dbtrim.idx removed\n");
		}
		else {
			if (errno!=ENOENT) imsg("db_expire_pile - dbtrim.idx couldn't be removed? check reason (%s)\n",strerror(errno));
		}
		for (i=0; i<MAX_DBN; i++) dbn_istrim[i] = FALSE;

		for (p=1; p<expire_npiles(); p++) {
				for (;db_pile_over(p);) {
						if (!pile_remove_oldest(p,FALSE)) break;
				}
		}
		for (p=1; p<expire_npiles(); p++) {
				if (expire_pile_days(p)>=0)  {
						pile_remove_days(p);
				}
		}

		return MSTATE_DONE;
}

static char *e_groups;
void set_egroups(char *xlist)
{
	int i;
	char del_list[BFSZ];
	ncpy(del_list,xlist,BFSZ-1);
	strlwr(del_list);
	e_groups = (char *) mymalloc2(xmax_groups,M_DB);
	for (i=0; i<group_n(); i++) {
		e_groups[group_getgroupid(i)] = group_match(del_list,group_namelc(i));
	}

}
int db_expire_index(void) /* return -1=end, 0 = continue, 1 = done */
{
		/* Pass thru index, and wipe entries for deleted items */
		/* While and at the same time, record entire list of items 
		  to use with history re-write */
		Mrec m;
		char bf[IDXSIZE];
		int n;
	int npatched=0;
	char tmp[BFSZ];
		int len;
	int dodb = config_true(C_INDEX_DEBUG);

		if (is_readonly()) return -1;
		if (fdbi<0) db_openi();
		if (fdbi<0) return TRUE ;
		ex_start("PatchIndex");
		dbi_seek(fdbi,0*IDXSIZE,SEEK_SET);
	ex_cancel_read();
		for (n=0;!dbi_eof(fdbi);n++) {
				len = dbi_read(fdbi,bf,IDXSIZE);
				if (len<IDXSIZE) break;
				if (!frec_to_mrec(bf,n*IDXSIZE,&m)) {
						continue;
				}
				if (m.fid<0) continue;
		if (e_groups!=NULL) if ((int) m.gid<(int) xmax_groups) if (e_groups[m.gid]) goto removeit;
				if (dbn_piledel[m.fid]) {
removeit:
			if (dodb) {
				lprintf(tmp,BFSZ-1,"del fid[%d] gid=%d num=%d len=%d pnt=%d\n"
					,m.fid,m.gid,m.num,m.len,m.ipnt);
				index2_log(tmp);	

			}
			expire_pile_del(m.gid,m.num);
						m.len = 0;
						dbi_seek(fdbi,m.ipnt,SEEK_SET);
						mrec_to_frec(&m,bf);
						if (dbi_write(fdbi,bf,IDXSIZE)!=IDXSIZE) filemsg("m_update, rebuild_index needed\n");
			npatched++;
						dbi_seek(fdbi,m.ipnt+IDXSIZE,SEEK_SET);
				}
		}
	if (npatched>0) imsg("db: Patched %d records in dbi.idx, handle %d\n",npatched,fdbi);
	if (e_groups!=NULL) myfree2(e_groups,M_DB); e_groups = NULL;
		ex_end("PatchIndex");
		db_clear_cache("patchindex");
		slave_dbflush();
		ex_end("PatchDel");
		if (get_expire_mode()) tell_master("reuse_buckets");
		return MSTATE_DONE;
}
int db_ispile(void)
{
		if (expire_ispile()) return TRUE;
		return config_true(C_DB_SORT);
}
int db_piles_trim(void)
{
		/* Scan for over full buckets */
		int p;
	if (!config_true(C_DISKIO_MIN)) {
		imsg("db_piles_trim called\n");
	}
		dmsg("pile: piles trim called\n");
		for (p=1; p<expire_npiles(); p++) {
				dmsg("pile: pile trim called %d %d\n",p,db_pile_over(p));
				for (;db_pile_over(p);) {
						dmsg("pile: Removing oldest  %d %d\n",p,db_pile_over(p));
						if (!pile_remove_oldest(p,TRUE)) break;
				}
		}

		return MSTATE_DONE;

}
int dbn_trim(int x)
{
		FILE *f;
		f = fopen(FILE_DB_TRIM,"a");
		if (f==NULL) return TRUE;
		fprintf(f,"%d\n",x);
		fclosep(f);
		dbn_istrim[x] = TRUE;
		return TRUE;
}
int dbn_trim_load(void)
{
		FILE *f;
		int i;
		int x;
		char bf[BFSZ];
		static int last;

		if ((lib_gmtime() - last)<60) return TRUE;
		last = lib_gmtime();
		

		for (i=0; i<MAX_DBN; i++) {
				dbn_istrim[i] = FALSE;
		}
		f = fopen(FILE_DB_TRIM,"r");
		if (f==NULL) return TRUE;
		for (;!feof(f);) {
				if (fgets(bf,BFSZ,f)==NULL) break;
				if (!isdigit(bf[0])) break;
				x = atoi(bf);
				if (x<0) continue;
				if (x>=MAX_DBN) continue;
				dmsg("db: Cyclic remembered to delete %d\n",x);
				dbn_istrim[x] = TRUE;
				if (strlen(bf)==0) break;
		}
		fclosep(f);
		return TRUE;
}
int db_expired(int i)
{
	if (dbn_isdel[i]) return TRUE;
		if (dbn_piledel[i]) return TRUE;
		if (dbn_istrim[i]) return TRUE;
	return FALSE;
}
int dbusy(int i)
{
		if (dbn_piledel[i]) return TRUE;
		if (dbn_wasused[i]) return TRUE;
		if (dbn_istrim[i]) return TRUE;
		if (dbn_isbusy[i]) return TRUE;
		return FALSE;
}
void db_trimgroup(char *gname, int listid[], int nlist)
{
		int n;
		int i;
		int k;
	int zmax;
		int grp = group_find(gname);
		if (grp<0) return;

	if (ishdr_new()) hdr_translate(grp,listid,nlist);
		n = g_list_start(grp);
	zmax = max_items();

		for (i=0; (i<zmax) && (n>=0); i++)	{
				for (k=0; k<nlist; k++) if (listid[k]==n) goto nextone;
				imsg("header: Expire %s:%d\n",gname,n);
		if (config_true(C_HEADER_EXPIRE_CAREFUL)) 
			item_delete_all(-1,grp,n);		
		else	item_delete(grp,n);

nextone:
				n = g_list_next();
		}
		db_clear_cache("trimgroups");
}
#define CANCEL_EXP file_work("cancel.exp")
static	FILE *exf;
void ex_cancel_add(int gid, int num)
{
	if (exf==NULL) {
		exf = myfopen(CANCEL_EXP,"w");
		if (exf==NULL) {
			filemsg("Could no write %s\n",CANCEL_EXP);
			return;
		}
	}
	fprintf(exf,"%d %d\n",gid,num);
}
void ex_cancel_read(void)
{
	char bf[BFSZ];
	char *s;
	int gid,num;
	if (exf!=NULL) fclosep(exf);
	exf = myfopen(CANCEL_EXP,"r");
	if (exf==NULL) {
		imsg("No cancel.exp data waiting\n");
		return;
	}
	for (;!feof(exf);) {
		if (fgets(bf,BFSZ-10,exf)==NULL) break;
		s = strtok(bf," \r\n"); 	if (s==NULL) continue;
		gid = atoi(s);
		s = strtok(bf," \r\n"); 	if (s==NULL) continue;
		num = atoi(s);
		/* imsg("Remebered to delete %d:%d\n",gid,num); */
		expire_pile_del(gid,num);
	}	
	fclosep(exf);	
	remove(CANCEL_EXP);
	exf = NULL;
}

/*---------- Bucket Sorting Stuff ---------------------------*/
#include "dnewssrt.h"
static char sort_mask[MAX_DBN];
static int nqued = 0;
static int bsort_busy;
int bsort_hash(int gid, int it);
int bsort_patch(PHead *h);
int bsort_que_add(int bkt)
{
	if (!db_ispile()) return TRUE;
	if (is_readonly()) return TRUE;
	if (feed_isfeeder()) {
		emsg("Cannot sort buckets with outgoing live/dfeed feeds\n");
		return TRUE;
	}

	if (bkt<0) return TRUE;
	if (bkt>=MAX_DBN) return TRUE;
	nqued++;
	if (nqued<20) imsg("bsort: Qued sort of bucket %d\n",bkt);
	sort_mask[bkt] = TRUE;
	return TRUE;
}
static char bsort_xstatus[BFSZ];
static int bsort_cur;
int bsort_idle(void)
{
	static int lastt;
	static int started;
	int i;

	if (is_readonly()) return TRUE;
	if (lib_gmtime()-lastt<3) return TRUE;
	if (is_expiring()) {
		if (bsort_busy) {
			cancel_bsort = TRUE;
			imsg("Cancelling current sort as we are expiring\n");
		}
	}

	lastt = lib_gmtime();
	if (bsort_busy) {
		if (lib_exists(file_work(DNEWSSRT_PAT))) {
			imsg("Yes it does exist\n");
			if (!cancel_bsort) {		
				stat_start(X_BSORT);
				bsort_process(file_work(DNEWSSRT_PAT));
				stat_end(X_BSORT);
			} else imsg("Cancelled bsort during expire\n");
			cancel_bsort = FALSE;
			if (remove(file_work(DNEWSSRT_PAT))!=0) {
				filemsg("Remove of (%s) failed \n",file_work(DNEWSSRT_PAT));
			}
			bsort_spawn_release();
			bsort_busy = FALSE;
		} 
		if (lib_gmtime()-started>1800) {
			imsg("bsort: timeout\n");
			bsort_spawn_release();
			if (bsort_cur>0) lprintf(bsort_xstatus,BFSZ-1,"Killed dnewssrt process as it didn't finish %d %s\n",bsort_cur,fid_name(bsort_cur));
			bsort_busy = FALSE;
		}
		return TRUE;		
	}
	if (is_expiring()) return TRUE;

	if (nqued<=0) return TRUE;
	for (i=0; i<MAX_DBN; i++) {
		if (sort_mask[i]) {
			remove(file_work(DNEWSSRT_PAT));
			imsg("bsort: spawn bucket sort %d\n",i);
			bsort_spawn(i);
			bsort_cur = i;
			started = lib_gmtime();
			bsort_busy = TRUE;
			sort_mask[i] = 0;
			nqued--;
			return TRUE;
		}
	}
	nqued = 0;
	return TRUE;
}
/*
		Read patch file into hash table.
		Seek to start of index entries.
		Pause slaves - flush indexes.
		Scan for bucket matches, and apply
		corrections.
		rename File
		Unpause slaves.
*/
#define BHASH_SIZE 2000
int bsort_hash(int gid, int item)
{
	int h;
	h = (gid + item) % BHASH_SIZE; 
	return h;
}
int bsort_hashat(int at)
{
	int h;
	h = (at) % BHASH_SIZE; 
	return h;
}
static Item *bhashed[BHASH_SIZE];
static Item *bhashedat[BHASH_SIZE];
Item *bsort_find(int gid, int num)
{
	Item *m;
	int h;
	h = bsort_hash(gid,num);
	for (m=bhashed[h];m!=NULL;m=m->next) {
		if (m->gid!=gid) continue;
		if (m->item==num) return m;
	}
	return NULL;
}
Item *bsort_findat(int at)
{
	Item *m;
	int h;
	h = bsort_hashat(at);
	for (m=bhashedat[h];m!=NULL;m=m->next) {
		if (m->wasat==at) return m;
	}
	return NULL;
}
char *bsort_status(void)
{
	return bsort_xstatus;
}
int bsort_process(char *fname)
{
	FILE *f;
	int i;
	int h;
	PHead head;
	char bname[BFSZ];
	Item rec,*m,*mnext;
	int r;

	imsg("bsort: Processing patch file from dnewssrt for\n");
	f = myfopen(fname,"rb");
	if (f==NULL) {
		filemsg("Can't open %s %s\n",fname);	
		return FALSE;
	}
	r = fread(&head,sizeof(head),1,f);	
	if (r!=1) {
		filemsg("Failed to read header %s %s\n",fname); 
		fclosep(f);
		return FALSE;
	}
	if (head.version!=100) {
		emsg("bsort mismatch version with dnewssrt program %d\n",head.version);
		fclosep(f);
		return FALSE;
	}
	if ((head.ibkt<0) || (head.ibkt>=MAX_DBN))	{
		emsg("bsort mismatch version with dnewssrt program %d\n",head.version);
		fclosep(f);
		return FALSE;
	}
	if (!head.isok) {
		emsg("dnewssrt failed, examine dnewssrt.log %s\n",head.msg);
		fclosep(f);
		return FALSE;
	}
	lprintf(bsort_xstatus,BFSZ-1,"BSort %d qued, %.80s",nqued,head.msg);

	strcpy(bname,fid_name(head.ibkt));
	imsg("bsort: Patching %d entries\n",head.nitem);
	/* Now rename the file */
	out_close_id(head.ibkt);

	if (lib_filesize(bname)!=lib_filesize(lib_samepath(bname,DNEWSSRT_ITEM))) {
		filemsg("bsort: sorted bucket is wrong size, ignoring it %d!=%d %s\n",lib_filesize(lib_samepath(bname,DNEWSSRT_ITEM)),lib_filesize(bname),bname);
		fclosep(f);
		return FALSE;
	}
	if (remove(bname)!=0) {
		filemsg("bsort: remove of bucket failed bkt %d %s\n",head.ibkt,bname);
		fclosep(f);
		return FALSE;
	}
	if (rename(lib_samepath(bname,DNEWSSRT_ITEM),bname)!=0) {
		filemsg("bsort: rename of bucket failed %s\n",bname);
		fclosep(f);
		return FALSE;
	}

	/* Read patch file into hashed index */
	for (i=0; i<head.nitem; i++) {
		r = fread(&rec,sizeof(Item),1,f); 
		if (r!=1) { filemsg("Failed reading patch %d\n",i); fclosep(f);return FALSE;}
		h = bsort_hash(rec.gid,rec.item);
		m = mymalloc2(sizeof(Item),M_DB);
		*m = rec;
		m->next = bhashed[h];
		bhashed[h] = m;
		/* And put it into 'at' hashed index */
		m = mymalloc2(sizeof(Item),M_DB);
		*m = rec;
		h = bsort_hashat(rec.wasat);
		m->next = bhashedat[h];
		bhashedat[h] = m;
	}
	fclosep(f);
	/* Pause slaves now */
	db_slave_flush_pause();


	/* Process and patch the dbi.idx file */
	bsort_patch(&head); 
	
	
	/* UnPause slaves now */
	db_slave_flush_unpause();

	

	/* Free hash index */
	for (h=0; h<BHASH_SIZE; h++) {
		for (m=bhashed[h]; m!=NULL; m=mnext) {
			mnext = m->next;
			myfree2(m,M_DB);
		}
		bhashed[h] = NULL;
		for (m=bhashedat[h]; m!=NULL; m=mnext) {
			mnext = m->next;
			myfree2(m,M_DB);
		}
		bhashedat[h] = NULL;
	}
	imsg("bsort: Patch file processed for dnewssrt\n");
	return TRUE;
}
/*
Result of bug, item in wrong group or totally screwewd index
but only slightly ? 

*/
int bsort_patch(PHead *h)
{

		Mrec m;
	Item *prec;
	char bf[IDXSIZE];
		int n;
	int nnorm=0;
		int tstart;
		int len;
	int npat = 0;

		if (fdbi<0) db_openi();
		tstart = lib_gmtime();
		if (fdbi<0) {
				return FALSE;
		}
	if (h->istart<0) h->istart = 0;
		dbi_seek(fdbi,h->istart*IDXSIZE,SEEK_SET);
	if (h->iend<=0) h->iend = 1000000000;
		imsg("bsort: patching from %d to %d\n",h->istart,h->iend);
		for (n=h->istart;!dbi_eof(fdbi);n++) {
		if (n>h->iend) break;
				len = dbi_read(fdbi,bf,IDXSIZE);
				if (len==0) break;
				if (len<IDXSIZE) break;
				if (!frec_to_mrec(bf,n*IDXSIZE,&m)) continue;
				if (m.len==0) continue;
		if (m.fid!=h->ibkt) continue;
		
		prec = bsort_find(m.gid,m.num);
		if (prec==NULL) {
			prec = bsort_findat(m.pnt);
			if (prec==NULL) {
				emsg("bsort: Patched item missing %d:%d\n",m.gid,m.num);	
				continue;
			} 
		} else nnorm++;
		m.pnt = prec->start;
				m_update(&m);							
		npat++;
				dbi_seek(fdbi,(n+1)*IDXSIZE,SEEK_SET);				  
		}
	imsg("bsort: patch done, took %d seconds, patched %d total %d\n",lib_gmtime()-tstart,nnorm,npat);
	if (nnorm != h->nitem) {
		imsg("bsort: must be some deleted items %d!=%d\n",nnorm,h->nitem);	
	}
	db_clear_cache("bsort_patch");
	g_free_all("bsort_patch");
	return TRUE;

}
/*		if (any) if (!spawn_isalive(dfeed_pid)) {*/


static int bsort_pid;
#include "lib_spawn.h"
int bsort_spawn(int bkt)
{
	char cmd[BFSZ];
	if (dbn_start[bkt]<0) {
		imsgf("bsort: Finding bucket ranges for sorting...\n");
		db_bucket_ranges();
	}

	lprintf(cmd,BFSZ-1,"%s %s %d %s %d %d %d",file_exe("dnewssrt")
		,fid_name(bkt),bkt,file_work("dnewssrt.pat"),dbn_start[bkt],dbn_end[bkt]
		,atoi(config_get(C_BUCKET_SORT_MAX)));
	bsort_pid = lib_spawn_cmd(cmd);

	imsg("Spawned bucket sorter %d %s\n",bsort_pid,cmd);
	return TRUE;
}
int bsort_spawn_release(void)
{
	if (bsort_pid==0) return TRUE;
	if (spawn_isalive(bsort_pid)) {
#ifndef WIN32
		imsgf("Waiting for sort process ...\n");
		lib_spawn_wait_finished(bsort_pid); 	
		imsgf("Finished waiting it has finished\n");
#endif
	}
	/* spawn_isalive(bsort_pid);*/
	bsort_pid = 0;
	return TRUE;
}
int bsort_que_add_all(void)
{
	int i;
	for (i=0;i<MAX_DBN; i++) {
				if (dbn[i].inuse) bsort_que_add(i);
		} 
	return TRUE;
}
int bsort_que_add_pile(int p)
{
	int i;
	for (i=0;i<MAX_DBN; i++) {
				if (dbn[i].pile==p) if (dbn[i].inuse) bsort_que_add(i);
		} 
	return TRUE;
}

int db_bucket_ranges_clear(void)
{
	int i;
		for (i=0; i<MAX_DBN; i++) dbn_start[i] = -1;
		for (i=0; i<MAX_DBN; i++) dbn_end[i] = 0;
	return TRUE;
}
int db_bucket_ranges(void)
{
		/* Fixrange, set last and first files, etc ... */
		int i;
		Mrec m;
		int f;
	int len;
		char bf[IDXSIZE];
		int fz;
		int n;
		int ncorrupt=0;

		for (i=0; i<MAX_DBN; i++) dbn_start[i] = 1000000000;
		for (i=0; i<MAX_DBN; i++) dbn_end[i] = 0;
	
		if (fdbi<0) db_openi();
		if (fdbi<0) return TRUE;
		f = fdbi;
		fz = dbi_len(f);
		dbi_seek(f,0,SEEK_SET);
		for (n=0;!dbi_eof(f);n++) {
				len = dbi_read(f,bf,IDXSIZE);
				if (len==0) break;
				if (len<IDXSIZE) break;
				if (!frec_to_mrec(bf,n*IDXSIZE,&m)) {
						ncorrupt++;
						continue;
				}
		if (m.fid<0) continue;
		if (m.fid>=MAX_DBN) continue;
		if (dbn_start[m.fid]>n) dbn_start[m.fid] = n;
		if (dbn_end[m.fid]<n) dbn_end[m.fid] = n;
		}
		return TRUE;
}
int bsort_que_add_ifpile(int bkt)
{
	if (expire_pile_sort(dbn[bkt].pile)) {
		imsg("bsort: Sorting as pile is set to sort, bkt %d, pile %d\n",bkt,dbn[bkt].pile);
		bsort_que_add(bkt);
		return TRUE;
	}
	return TRUE;
}

/* New REINDEX sorting mechanism goes here 

Step 1
	Assign each group to a reindex stack
	Create 'nstack' buffers and clear
Step 2 ...
	Scan main index (in parts) and write each record
	to stacks, flushing stacks as they become full.
Step 3 ...
	Read one stack, sort it, write it to new index
	When finished switch_index.
		And free stacks

*/

/* returns -1 when completed
 Assume cancel and check index are 'done' already in real db_reindex() 
*/
static int nstk;
#define STACK_LEN 10000
#define MAX_STACK 15000
static String *stack[MAX_STACK];
static int stack_nrec[MAX_STACK];
/* Returns 'true' when finished */
int db_reindex_new(int start) 
{
	int tot=0;
	static int upto;
	static int state;
	int g,r,i;
	int ng=0;
		if (resort==NULL) resort_grow(rechunk()+1000);

	if (start) {
		dblog("stack: resort: dbi size is %d \n",fdbi_len);
		tot = ng = 0;
		for (g=0; g<group_n(); g++) re_grps[g] = -1;
			for (g=0;(g<group_n()) /* && (tot<rechunk()) */ ;g++) {
					r = group_rawrange(g);									
					tot += r;
					if (tot>5000) if ((tot>(rechunk()-5000)) && (ng>0)) {
				tot = 0;
				nstk++;
				ng = 0;
			}
					re_grps[g] = nstk;
					ng++;
			}
		nstk++;
		imsgf("Sorting dbi.idx into %d stacks\n",nstk);
		if (nstk>=MAX_STACK) {dnews_ex("sort_chunk is too small dieing"); abort();}
		for (i=0; i<nstk; i++) {
			stack[i] = str_new();
			stack_nrec[i] = 0;
			remove(db_stack_file(i));
			dblog("stack: Remove at start of resort %s\n",db_stack_file(i));
		}
		upto = 0; /* Position in index */
		state = 1;
		return FALSE;
	} 
	if (state==1) {
imsg("db_reindex_new_scan %d\n",upto);
		if (db_reindex_new_scan(&upto)) {
			upto = 0;
			state = 2;
			for (i=0; i<nstk; i++) {
				db_reindex_stack_flush(i);
				str_free(stack[i]);
				stack[i] = NULL;
			}
		}
		return FALSE;
	}
	if (state==2) {
		imsg("db_reindex_new_sort %d\n",upto);
		if (!db_reindex_new_sort(&upto)) return 0;
	}
	/* Finished, switch index, free strings etc */ 

	for (i=0; i<nstk; i++) {
		dblog("stack: resort: finished removing %s \n",db_stack_file(i));
		remove(db_stack_file(i));
	}

		wmsg("Finished reindex_new procedure\n");
	if (expire_isgiveup()) return TRUE;

		reindex_switch();
		inresort = FALSE;
		if (!get_expire_mode()) db_write_idx_now(); /* Write any buffered index entries*/
		g_free_all("reindex_new");
		reindex_free();
		db_clear_cache("reindex_new");
		upto = 0;
		return TRUE;
}
int db_reindex_new_scan(int *upto) /* Return true when finished */
{
		char bf[IDXSIZE];
		int n;
		int xstart = lib_gmtime();
		Mrec m;
		int nexp=0;
		int ncorrupt = 0;
		int len;
	int g;
	static int lastprune;
	int maxrange = max_items()-1000;
	int ishistmax = atoi(config_get(C_HIST_MAX));
	if (ishistmax!=0) ishistmax = TRUE;
	if (maxrange>30000) maxrange = maxrange - (maxrange /10);


		if (fdbi<0) db_openi(); if (fdbi<0) return TRUE;
		dbi_seek(fdbi,*upto * IDXSIZE,SEEK_SET);
		for (n= *upto;(n*IDXSIZE)<fdbi_len;n++) {
				len = dbi_read(fdbi,bf,IDXSIZE);
				if (len==0) break;
				if (len<IDXSIZE) {
						emsg("FATAL, idx is corrupt length, panic, got %d bytes\n",len);
						break;
				}
				if (!frec_to_mrec(bf,n*IDXSIZE,&m)) {
						ncorrupt++;
						continue;
				}
				if (m.len==0) nexp++;
				
		g = gid_to_group(m.gid);
				if (g<0) imsg("gidtogroup lost %d %d\n",m.gid,gid_to_group(m.gid));
				else if (m.len>0)  {
			if (m.num > group_getend(g)-maxrange) {
				if ((!ishistmax) && (m.num<min_get(g))) {
					imsg("Triming item not in history index %s:%d\n",group_name(g),m.num);
				} else {
					db_reindex_stack(g,&m);
				}
			} else {
				if (g!=lastprune) imsg("Pruning large group %s:%d\n",group_name(g),m.num);
				lastprune = g;
			}
		}
				
				if ((n%5000)==500) if (lib_gmtime()-xstart>expire_quanta()) {
						*upto = n;
						return FALSE;
				}
		}
		if (ncorrupt>0) emsg("Removed %d corrupt lines, a rebuild_index might be needed\n",ncorrupt);
		return TRUE;
}
int db_reindex_stack_flush(int h)
{
	FILE *f;	
	if (stack[h]==NULL) return TRUE;
	if (str_len(stack[h])==0) return TRUE;

	f = fopen(db_stack_file(h),"ab");
	if (f==NULL) {
		filemsg("%s\n",db_stack_file(h));
					   expire_failed("Writing new index file 2 ");
		return FALSE;
	}
	dmsg("db: Stack flush %d bytes to %s\n",str_len(stack[h]),db_stack_file(h));
	if (fwrite(str_s(stack[h]),1,str_len(stack[h]),f)<1) {
		filemsg("expire writing new index failed\n");
					   expire_failed("Writing new index files");
		fclosep(f);
		return FALSE;
	}
	fclosep(f);
	str_clear(stack[h]);
	return TRUE;
}
int db_reindex_stack(int g, Mrec *m)
{
	int h;
	h = re_grps[g];
	if (h<0) return FALSE;
	if (h>=nstk) return FALSE;
	str_adds(stack[h],(char *)m,sizeof(Mrec));
	stack_nrec[h]++;
	if (str_len(stack[h])>STACK_LEN) {
		db_reindex_stack_flush(h);
	}
	return TRUE;
}
char *db_stack_file(i)
{
	char bf[BFSZ];
	lprintf(bf,BFSZ-1,"dbsrt_%d.tmp",i);
	return file_work(bf);
}
int db_reindex_new_sort(int *upto) 
{
	/* For each stack, read it in, sort it, write it out to new index file */		
	char bf[BFSZ];	
	FILE *f;
	int h = *upto;
	int i;
	Mrec m;
	int xstart = lib_gmtime();

	if (*upto>=nstk) return TRUE;
	f = fopen(db_stack_file(h),"rb");
	if (f==NULL) { 
		filemsg("No stack %s found, %d written\n",db_stack_file(h),stack_nrec[h]);
		(*upto)++;
		return FALSE; 
	}
	dblog("stack: Process file %s %d\n",db_stack_file(h),file_length(fileno(f)));
	for (;!feof(f);) {
		if (fread(&m,sizeof(Mrec),1,f)<1) break;
		readd(&m);
	}
	fclosep(f);
	(*upto)++;

		imsg("new_reindex_sort %d / %d\n",nresort,stack_nrec[h]);
		if (nresort>1) qsort(resort,nresort,sizeof(Mrec),resort_cmp);
	dblog("stack: Sorted %d  %d\n",h,nresort);
		for (i=0; i<nresort; i++) {
				if (last_boundsu(resort[i].gid)) {
						imsg("resort gid funny %d %d \n",i,resort[i].gid);
						continue;
				}
				if (i==0) {
						gid_first[resort[i].gid] = fdbs_len;
				} else {
						if (resort[i].gid!=resort[i-1].gid) {
								gid_first[resort[i].gid] = fdbs_len;
						}
				}
				if (i+1>=nresort) goto endgrp;
				if (resort[i+1].gid==resort[i].gid) {
						if (resort[i+1].num==resort[i].num) {
								imsg("db: Removing duplicate item from group gid=%d:%d\n",resort[i].gid,resort[i].num);
								goto skipdup;
						}
						resort[i].jump = fdbs_len+IDXSIZE;
				} else {
endgrp: 				resort[i].jump = -1;
						gid_last[resort[i].gid] = fdbs_len;
						/* imsg("Moved %d to %d\n",resort[i].gid,fdbs_len);*/
				}
				mrec_to_frec(&resort[i],bf);
				m = resort[i];
				if (dbf_write(fdbs,bf,IDXSIZE)<IDXSIZE) {
						filemsg("Failed writing new index file \n");
						expire_failed("Writing new index file");
						return FALSE;
				}
				fdbs_len += IDXSIZE;
skipdup:;
		}
		imsg("new_reindex_done %d seconds \n",lib_gmtime()-xstart);
		resort_clear();
		g_clear("newreindexdone");	
	return FALSE;
}

int db_idle_slave(void)
{
	static int lastlen;
	int newlen;
	int n;
	if (!config_true(C_INDEX_CHANGES)) return TRUE; 
	if (is_mastermaster()) {
		/* imsg("ismastermaster so not checking for index changes\n");*/
		return TRUE;
	}
	newlen = lib_length_file(FILE_DBI);
	n = newlen;
	if (lastlen>0) if (newlen>lastlen) {
		n = db_idle_addnew(lastlen,newlen);
	}
	lastlen = n;	
	return TRUE;
}
int db_idle_addnew(int start, int xend)
{
		Mrec mm;
	int pnt;
	int nin=0;
	int nout=0;
	int c;
	int grp;
	int lastpnt=start;
		char rec[IDXSIZE];
	int nexp=0;
	int nfree=0;
	time_t tstart = time(NULL);

		if (fdbi>=0) dbi_close(fdbi);
		fdbi = -1;
		if (!db_openi()) return lastpnt;
	if (!dbi_seek(fdbi,start,SEEK_SET)) {filemsg("dbi: Seek failed %d\n",start); return lastpnt;}
	for (pnt=start; !dbi_eof(fdbi) ; pnt+=IDXSIZE) {
		if (pnt>fdbi_len) break;
				if (dbi_read(fdbi,rec,IDXSIZE)!=IDXSIZE) {break;}
				lastpnt = pnt+IDXSIZE;
				if (!frec_to_mrec(rec,pnt,&mm)) {
			emsg("Idx read bad item from index? :%d\n",mm.num);
			continue;
				}
				if (mm.len==0) {
					   continue;
				}
				if (dbn_istrim[mm.fid]) {
			nexp++; continue;
		}
		grp = gid_to_group(mm.gid);
		if (mm.num>group_getend(grp)) group_setend(grp,mm.num);
		group_addrange(grp,1);

			c = g_cache_find(grp);
		/*imsg("Adding item from index (%.100s %d cache=%d)\n",group_name(grp),mm.num,c);*/
		nout++;
			if (c>=0) {
			
			if (config_true(C_REPLICATE) && group_getrange(grp)>50000)	{
				imsg("index_changes: freeing index (%s)\n",group_name(grp));
				g_free(c,"index_changes");
				nfree++;
			} else {
						m_insert(c,&mm);
						nin++;
			}
		}
		}
	imsg("changes: Cached=%d New=%d start=%d end=%d, freed=%d, In %d seconds\n",nin,nout,start,xend,nfree,time(NULL)-tstart);
	return lastpnt;
}

void dblog(char * arg_list, ...)
{
		va_list arg_ptr;
		char *format;
		int len;
	int del=FALSE;
	char output[BFSZ];
	char fname[BFSZ];
	FILE *f;
		va_start(arg_ptr, arg_list);
		format = arg_list;
		len = vsnprintf(output,BFSZ-1, format, arg_ptr);
	output[BFSZ-1] = 0;

	f = fopen(file_log("db.log"),"a");
	if (f!=NULL) {
		fprintf(f,"%s id=%s s=%d e=%d %s",emsg_time(),config_get(C_READONLY_ID),slave_n(),get_expire_mode(),output);
		if (file_length(fileno(f))>1000000) del=TRUE;
		fclose(f);
	}
	if (del) {
		remove(file_log("db2.log"));
		ncpy(fname,file_log("db.log"),BFSZ-1);
		rename(fname,file_log("db2.log"));
	}
}
/* 
Cacheing, to cache 10 minutes of index entries, so they
can be written in order to disk, to improve disk read performance 

For each group, save the data 

*/
typedef struct WIDX 
{
	int fid,grp,num,pnt,len;
	struct WIDX *next;
} Widx;
static Widx **group_widx;
static int nwidx;
int idx_write_one(int g);
int idx_write_add(int fid, int grp,int num,int pnt, int len)
{
	Widx *w,*k;
	Widx **lastnext;
	if (group_widx==NULL) {
		group_widx = (Widx **) mymalloc2(sizeof(Widx *) * xmax_groups,M_DB);
		memset(group_widx,0,sizeof(Widx *) * xmax_groups);
	}

	w = mymalloc2(sizeof(Widx),M_DB);
	w->fid = fid;
	w->grp = grp;
	w->num = num;
	w->pnt = pnt;
	w->len = len;
	w->next = NULL;
	nwidx++;
	lastnext = &(group_widx[grp]);
	for (k=group_widx[grp];k!=NULL;k=k->next) {
		if (num>=k->num) break;
		lastnext = &(k->next);
	}
	w->next = k;
	*lastnext = w;
	if (nwidx>idx_write_size) idx_write_flush("Buffer size reached");
	return TRUE;
}
static int ncontig;
int idx_write_flush(char *reason)
{
	int g;
	ncontig = 0;
	if (nwidx==0) return TRUE;
	imsg("idx_write_flush %d items %s\n",nwidx, reason);
	for (g=0; g<xmax_groups; g++) {
		if (group_widx[g]!=NULL) idx_write_one(g);
	}
	imsg("idx_write_flush %d/%d contiguous finished %s\n",ncontig,nwidx,reason);
	nwidx = 0;
	return TRUE;
}
int idx_write_one(int g)
{
	Widx *w,*wnext;
	idx_write_cache = FALSE;
	ncontig--;
	for (w=group_widx[g]; w!=NULL; w=wnext) {
		ncontig++; 
		wnext = w->next;
		db_add_idx(w->fid,w->grp,w->num,w->pnt,w->len);
		myfree2(w,M_DB);
	}	
	group_widx[g] = NULL;
	idx_write_cache=TRUE;
	return TRUE;
}


typedef struct SORTCACHE {
	Mrec m;
	int c;
	struct SORTCACHE *next;
} Sortcache;
static Sortcache *sortlist;
int m_insert_sort_start(void)
{
	m_insert_free();
	return TRUE;
}
int m_insert_free(void)
{
	Sortcache *s,*snext;

	for (s = sortlist; s!=NULL; s=snext)  {
		snext = s->next;
		myfree2(s,M_DB);
	}
	sortlist = NULL;
	return TRUE;
}
int m_sort_cmp(const void *e1, const void *e2)
{
	Sortcache **a = (Sortcache **) e1;
	Sortcache **b = (Sortcache **) e2;
	return (**a).m.num -  (**b).m.num;
}
int m_insert_sort_done(void)
{
	int n;
	int i;
	static Sortcache **sortcache=NULL; 
	Sortcache *s;
	if (sortlist==NULL) return FALSE;
	n = 0;
	for (s = sortlist; s!=NULL; s=s->next)	n++;
	if (n==0) return FALSE;
	if (n>max_items()) n = max_items()-1;
	if (sortcache==NULL) sortcache = mymalloc2((max_items())*sizeof(Sortcache *),M_DB);
	memset(sortcache,0,(n)*sizeof(Sortcache *));
	for (i=0,s = sortlist; s!=NULL; s=s->next,i++)	{
		sortcache[i] = s;	
	}
	if (n>1) {
		qsort(sortcache,n,sizeof(Sortcache *),m_sort_cmp);
	}
	for (i=0; i<n; i++) {
		m_insert(sortcache[i]->c,&sortcache[i]->m);
	}
	/* myfree2(sortcache,M_DB); - only kept once so memory doesn't fragment  */ 
	m_insert_free();
	return TRUE;
}
int m_insert_sort(int c, Mrec *mm)
{
	Sortcache *s;
	s = mymalloc2(sizeof(Sortcache),M_DB);
	s->next = sortlist;
	s->m = *mm;
	s->c = c;
	sortlist = s;
	return TRUE;
}

