/*
Tests:
All with and without history page size defined.
	Simple MID lookups
	tellnews showgroup_hist 
	stream of duplicate articles. 
	rebuild_index
	rebuild_hist (twice)
	expire
	spawned expire
	dmulti expire
	Readonly configuration history lookups and expire.
*/

/* Copyright (C) 1995 NetWin World, New Zealand, All rights reserved. */
#include <stdlib.h>
#include <stdio.h>
#include <ctype.h>
#include <string.h>
#include <assert.h>
#include <errno.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>

#ifdef VMS
#include <unixio.h>
#else
/*RHP file stuff for Mac lib will have to deal with these*/
#ifdef __MWERKS__
#include "Mac.h"
#else
#ifndef WIN32
#ifndef NOUNISTD
#include <unistd.h>
#endif
#endif
#endif
#endif

#ifndef NO_FCNTL
#include <fcntl.h>
#else
/*RHP dont want file.h for Mac*/
#ifndef Mac
#include <file.h>
#endif
#endif

#ifdef OS2
#include <io.h>
#endif
#ifdef WIN32
#include <io.h>
#endif

#include "emsg.h"
#include "lib.h"
#include "str.h"
#include "group.h"
#include "hist.h"
#include "mem.h"
#undef mymalloc
#undef myfree
#define mymalloc(x) mymalloc2(x,M_HIST)
#define myfree(x) myfree2(x,M_HIST)

int hist_save_file_order(void);
int hist_raw_close(int x);
int hist_raw_open(char *fname, int mode, int smode);
/* BLOCK and Block is defined in mem.h, so we rename this one to HBLOCK */
int db_total_space(void);
typedef struct HBLOCK {int *data; int width, size, used, upto; struct HBLOCK *next;} Hblock;
int block_init(Hblock *b, int width, int size);
int hist_malloc_init(void);
int hist_idx_new(void);
int *hist_block(int bi);
Hpnt *hist_malloc(int sz);
void hist_dofree(Hblock *b, int *x);
void hist_free(Hpnt *x, int sz);
void hist_free2(Hpnt *h);
void blocks_show(void);
h_type hist_file_length(int x);

static int total_hblock;
static int in_part=FALSE;
static Hpnt **hidx;
int hist_size=20000;        /* 80000; */
/*#include "myraw.h"*/

/* Still to write */

#ifdef HIST64
#define HVER 6
#else
#define HVER 5
#endif

int hist_open_files(int isread);

int hist_find_oldest_file(void);
void hist_idx_clear(void);
int str_hash(char *s, int max);
#define RCHUNK 400 /* should be aboue 300, avg length of 1-2 lines */
static int rchunk=RCHUNK;

/**/
#define HIST_MAX_FILES 256
static char last_hist_file[BFSZ]={"history.dat"};
static int file_count=0; /* Number of history files being used */
static int current_page_id=0; /* Page we are currently working on*/
typedef struct HFile {int file; int ord;} Hfile; /*0=unused 1=oldest n=newest */
static Hfile hfiles[HIST_MAX_FILES];
static unsigned int hist_page_size=0;
static int hver=HVER,hwords=HWORDS;

static int ibf[5001];
static int nibf;

#define DNEWS_HISTORY_ORDER file_hist("hist.ord")
#define DNEWS_EXPIRED_LIST file_hist("hist.exp")

#define DNEWS_HISTORY  file_hist(last_hist_file)
/*#define DNEWS_HISTORY  file_hist("history.dat")*/
#define DNEWS_HISTORY_IDX  file_hist("history.idx")

#ifdef SHORT_NAMES
#define DNEWS_HISTORY_NEW  file_hist("history.dnw")
#define DNEWS_HISTORY_NEW_IDX file_hist("history.inw")
#else
#define DNEWS_HISTORY_NEW  file_hist("history.dat_new")
#define DNEWS_HISTORY_NEW_IDX file_hist("history.idx_new")
#endif

#ifndef WIN32
#define O_BINARY_MISSING TRUE
#endif

#ifdef O_BINARY_MISSING
#ifndef O_BINARY
#define O_BINARY 0
#endif
#endif

void hist_cache_flush(void);
void hist_cache_add(char *bf, char *mid, h_type fz);
char *hist_cache_find(char *mid);
static int hist_isread;
static h_type catchupfz;
/* Message-ID<tab>arrived~expire~posted<tab>group/num group/num ... */
/* If no expirey date, then set to '-' */
/* If article has expired, then set to messagid-expired<tab>'x' */ 
static int total_mem;

#define TESTIT xx
#ifdef TESTITxx
extern char debug_str[];
main()
{
        int num,grp,r,i;
        char bf[200],bf2[200];
        char *s,*rec;

        strcpy(debug_str,"nntp:,err:,debug:,info:,item:,group:,suck:,cmd:,hist:,fatal:,");

        imsg("hist: History init\n");
        hist_init();
#define ADDRECS xx
#ifdef ADDRECS
        for (i=1; i<60000; i++) {
                lprintf(bf,200-1,"<mm%d>",i);
                lprintf(bf2,200-1,"comp.os.vms/%d misc.writing/%d",i,i);
                hist_add(bf,"25 Jan 95 09:54:22 GMT"
                        ,"25 Feb 95 09:54:22 GMT"
                        ,bf2);
        }
#endif
        for (i=1; i<60000; i++)  {
                lprintf(bf,200-1,"<mm%d>",i);
                if (!hist_exists(bf)) printf("Doesn't exist? {%s} \n",bf);
        }
}
int hist_open_files(int isread);
char *chan_getipname() {return "";}
int diskuse_get() {return 0;}
int feed_wanted() {return 0;}
int match_wild(char *a, char *b) {return TRUE;}
char *nntp_hostname() {return "";}
void post_seterr() {}
int suck_update_groups() {return TRUE;}
#endif 

#ifdef TESTITxx
extern char debug_str[];
main()
{
        int num,grp,r,i;
        char bf[200],bf2[200];
        char *s,*rec;
/*test after with odd and even number of records*/
        strcpy(debug_str,"nntp:,err:,debug:,info:,item:,group:,suck:,cmd:,hist:,fatal:,");

        imsg("hist: History init\n");
        hist_init();
#define ADDRECS XX
#ifdef ADDRECS
        for (i=1; i<100000; i++) {
                lprintf(bf,200-1,"<m%d>",i);
                lprintf(bf2,200-1,"comp.os.vms/%d misc.writing/%d",i,i);
                hist_add(bf,"25 Jan 95 09:54:22 GMT"
                        ,"25 Feb 95 09:54:22 GMT"
                        ,bf2);
        }
#endif
        
        imsg("info: Flush total_mem %d \n",total_mem);
        hist_flush();
                printf("I hope you deleted history.dat first !!?\n ");
                printf("Enter message id ? "); gets(bf);
                r = hist_find(bf,&grp,&num);
                printf("Found %d Group %d Number %d ",r,grp,num);
                printf("Also test extract functions and date stuff \n");
                rec = hist_exists(bf);
                if (rec==NULL) printf("Could not find record \n");
                if (rec!=NULL) {
                  printf("Dates date_expires %d \n ",hist_date_expires(rec));
                  printf("Dates date_arrived %d \n ",hist_date_arrived(rec));
                        s = hist_groupfirst(rec);
                        for (i=0;s!=NULL;i++) {
                                 printf("Found Group {%s} \n",s);       
                                printf("num %d \n",hist_num(rec,i));
                                s = hist_groupnext();
                        }
                }
        printf("Enter date for after search "); gets(bf);
        i  = atoi(bf);
        rec = hist_after(i);
        for (;rec!=NULL;) {
                printf("date %d  msg %s \n",hist_date_arrived(rec),rec);
                rec = hist_next();
        }
        /*
        int hist_after(int date);
        */

}
#endif 

static int h_his = -1,nbuffed;
static h_type file_size;
static int notatend;
static h_type upto;
h_type h_seek(int h_his,h_type offset,int s)
{
#ifdef WIN32
	return (h_type) _lseeki64(h_his,offset,s);	
#else
#ifdef HAS_SEEK64
	return lseek64(h_his,(off64_t) offset,s);
#else
	return lseek(h_his,offset,s);
#endif
#endif
}

String *hist_status(void)
{
        static String *x;
        static char bf[1200];
        lprintf(bf,1200-1,"History: disk %dk memory %dk/%dk, Str %s Malloc %dk/%dk/%dk\015\012"
          ,file_size/1000,total_mem/1000,total_hblock/1000,p_status(),mem_total()/1000,mem_biggest()/1000,mem_block_total()/1000);
        if (x==NULL) x = str_new(); 
        str_copysn(x,bf);
        return x;
}
char *hist_status_s(void)
{
        static char bf[200];
        lprintf(bf,200-1,"History: disk %dk memory %dk/%dk, Str %s Malloc %dk/%dk/%dk\015\012"
          ,file_size/1000,total_mem/1000,total_hblock/1000,p_status(),mem_total()/1000,mem_biggest()/1000,mem_block_total()/1000);
	return bf;
}
int hist_diskspace(void)
{
	return (int) (file_size/1000);
}
h_type hist_upto(void)
{
        return upto;
}
static int kill_last;
extern int xtime_now;
static int hist_fixup;
int hist_add_fix(char *mid, char *s_posted, char *s_expires, char *grps)
{
	int r;
	hist_fixup = TRUE;
	r = hist_add(mid,s_posted,s_expires,grps);
	hist_fixup = FALSE;
	return r;
}
int mid_isok(char *s)
{
	int in = FALSE;
	int atend = FALSE;
	int len=0;
	for (;*s!=0;s++,len++) {
		if (atend) {imsg("mid: Embedded right bracket\n"); return FALSE;}
		if (*s=='>') {in = FALSE; atend = TRUE;}
		if (in) {
			if (*s=='<') {imsg("mid: Embedded left bracket\n"); return FALSE;}
			if (*s==' ') {imsg("mid: Embedded space\n"); return FALSE;} 
		}
		if (*s=='<') in = TRUE;
	}
	if (!atend) {imsg("mid: missing right bracket\n"); return FALSE;}
	if (len>97) {imsg("mid: too long\n"); return FALSE;}
	return TRUE;
}
int hist_add(char *mid, char *s_posted, char *s_expires, char *grps)
{
        char bf[11000];
	char tmp[200];
        int expires,posted,arrived;

        kill_last = TRUE;
        
        arrived = xtime_now; /* Convert to an integer :-) */
        expires = parsedate(s_expires);
        posted = parsedate(s_posted);
	if (hist_fixup) {
		arrived = posted;
		if (abs(xtime_now - arrived)>60*60*24*300) arrived = xtime_now;
	}
        if (strlen(mid)>200) {emsg("hist: Message id too long %d \n",strlen(mid)); return FALSE;}

        if (!mid_isok(mid)) {ncpy(tmp,mid,100); emsg("hist: Message id bad format (%s)\n",tmp); return FALSE;}
        if (strlen(grps)>10000) {emsg("hist: Groups too long %d \n",strlen(grps)); return FALSE;}
        lprintf(bf,11000-1,"%s\t%d~%d~%d\t",mid,arrived,expires,posted);
        ncpy(bf+strlen(bf),grps,10000); /* we won't allow more cross posting thatn that*/
        strcat(bf,"\n");
/* As bug in vaxcrtl prevents seeking and reading and writing we must avoid it */
        hist_cache_add(bf,mid,file_size);
/*
        hist_addidx(mid,file_size);
        hist_append(bf);
*/
        return TRUE;
}
#define CACHE_SIZE 800
static char *c_bf[CACHE_SIZE];
static char *c_mid[CACHE_SIZE];
static h_type c_fz[CACHE_SIZE];
static int nc;
void hist_cache_add(char *bf, char *mid, h_type fz)
{
	if (nc>=CACHE_SIZE) emsg("Panic hist cache add\n");
        c_bf[nc] = mystrdup(bf,SD_HIST);
        c_mid[nc] = mystrdup(mid,SD_HIST);
        if ((c_bf[nc]==NULL) || (c_mid[nc]==NULL)) {emsg("hist: memory allocation failure\n"); exit(1); }
        c_fz[nc] = fz;
 /*wmsg("hist: cache add {%s} offset %d  \n",mid,fz); */ 
        nc++;
  /*if (nc>=10) {hist_cache_flush(); imsg("DEBUG: HIST FLUSH\n"); }*/
        if (nc>=(CACHE_SIZE-2)) 
			hist_cache_flush();
		
/*hist_flush();/**/
}
void hist_cache_flush(void)
{
        int i;
        if (nc==0) return;
	hist_close(); 
	hist_open_files(FALSE);

        for (i=0; i<nc; i++) {
                hist_addidx(c_mid[i],file_size); /* The passed I'll was wrong*/
/*if (i==0) imsg("Appending message {%s} \n",c_mid[i]);*/
                hist_append(c_bf[i]);
                mystrfree(c_bf[i],SD_HIST); mystrfree(c_mid[i],SD_HIST);
        }
	hist_close(); 
	hist_open_files(TRUE);
        nc = 0;
        hist_flush(); 
}

char *hist_cache_find(char *mid)
{
        static char tmp[1002];
        int i;
	vtrace();
        for (i=0; i<nc; i++) {
                if (strcmp(mid,c_mid[i])==0) {
                        ncpy(tmp,c_bf[i],1000);
			vtrace();
                        return tmp;
                }
        }
	vtrace();
        return NULL;
}
void hist_flush(void)
{
		hist_cache_flush();
        hist_idx_flush();
        if (nbuffed==0) return; /* Nothing to flush */
        nbuffed = 0;
	if (h_his!= -1) {
#ifdef WIN32
		if (!hist_isread) if (_commit(h_his)<0) filemsg("Cannot flush history file\n");
#endif
	}
}

void hist_close(void)
{
	int i;
	if (!hist_page_size) {
		close(h_his);
		h_his = -1;
		return;
	}
	for(i=0;i<HIST_MAX_FILES;i++) {
		if(hfiles[i].file>=0) {
			close(hfiles[i].file);
			hfiles[i].file = -1;
		}
	}
}


typedef struct EXP {int grp; int num; int used; struct EXP *next;} Exp;
static void *head;
#define EXP_HASH 10000
static Exp *expired_list[EXP_HASH];

int hist_expire_item_hash(int grp, int num)
{
	return abs((grp*13 + num) % EXP_HASH);
}

void hist_expire_item_add(int grp, int num)
{
	int h;
	Exp *m;
	h = hist_expire_item_hash(grp,num);
	m = (Exp *)getmem_malloc(&head,sizeof(Exp));

	m->grp = grp;
	m->num = num;
	m->used = 0;
	m->next = expired_list[h];
	expired_list[h] = m;
}

int hist_open_expired_list(void)
{
	int n;
	FILE *f;
	int i;
	int ex[BFSZ];

	if ((f=fopen(DNEWS_EXPIRED_LIST,"rb"))==NULL) return -1;

	for (;!feof(f);) {
		n = fread(ex,sizeof(int),BFSZ,f);
		for (i=0; i<n; i+=2) {
			hist_expire_item_add(ex[i],ex[i+1]);
		}
	}

	fclose(f); 
	return TRUE;
}

int hist_expire_list_free(void)
{
	int i;

	for(i=0;i<EXP_HASH;i++) {
		expired_list[i] = NULL;
	}

	getmem_free(&head);
	head = NULL;

	return TRUE;
	
}

int hist_save_expired_list(void)
{
	int i,entry[2];
	Exp *exp;
	FILE *f;

	if((f=fopen(DNEWS_EXPIRED_LIST,"wb"))==NULL) return FALSE;

	for(i=0;i<EXP_HASH;i++) {
		exp=expired_list[i];
		for(;exp!=NULL;exp=exp->next) {
			if(!exp->used) {
				entry[0]=exp->grp;
				entry[1]=exp->num;
				fwrite(entry,sizeof(int),2,f);
				/* printf("Test: Group: %d Article: %d not removed\n",exp->grp,exp->num);*/
			} else {
				/* printf("Test: Group: %d Article: %d removed\n",exp->grp,exp->num);*/
			}
		}
	}

	fclose(f);
	return TRUE;
}

int hist_expire_oldest_file(void)
{
	int grp,art,hash;
	int i,minidx,expired,hlcl,idx_entry[4];
	char *s,*delimit,*grpn,mid[BFSZ],localcopy[10000];
	Hpnt h,*hp;
	Exp *exp;
	FILE *h_idx;

	hist_cache_flush();
	readonly_pause();

	h.pnt=0;
	h.page=minidx=hist_find_oldest_file();
	
	/* go through index and find items in this page */
	imsg("Oldest file: hist%0.4d.dat\n",minidx);

	hist_open_expired_list();

	/* add unexpired items again, delete the rest */
	if(minidx==current_page_id) {
		wmsg("expire: Still on first page so no oldest history to expire\n");
		readonly_unpause();		
		return 1;
	}

	h_idx = fopen(DNEWS_HISTORY_IDX,"wb");
	if (h_idx==NULL) {
		filemsg("Cannot write to (%s) FATAL ERROR\n",DNEWS_HISTORY_IDX);
		return 1;
	}

	for(i=0;i<4;i++) fwrite(&hver,sizeof(int),1,h_idx);
	for(i=0;i<4;i++) fwrite(&hist_size,sizeof(int),1,h_idx);

	/* Re-write the index minus items from the file to delete */
	for(i=0;i<hist_size;i++) {
		hp=hidx[i];
		for (;hp!=NULL;hp=hp->next) {
			if(hp->page!=minidx) {
				idx_entry[0]=i; /* initial hash, pnt, match, page */
				idx_entry[1]=hp->pnt;
				idx_entry[2]=hp->match;
				idx_entry[3]=hp->page;
				if (fwrite(idx_entry,sizeof(int),4,h_idx)!=4) {
					emsg("History idx write failed, FATAL ERROR\n");	
				}
			}
		}
	}

	fclose(h_idx); h_idx = NULL;
	hist_idx_read(0);
	hist_idx_new(); /* Create new memory index if needed */

	imsg("hist_expire_oldest_file: Index expire done\n");

	lprintf(last_hist_file,12,"hist%0.4d.dat",minidx);
	hlcl=open(file_hist(last_hist_file),O_RDONLY | O_BINARY,U_RW);
	imsg("hist_expire_oldest_file: Now scanning %s\n",last_hist_file);
	while(s=hist_next_multi(hlcl)) {
		ncpy(localcopy,s,10000-1);
		grpn = hist_groupfirst(s);
		for (;grpn!=NULL;) {
		  grp = gid_find_g(grpn);
		  art = hist_groupnextnum();
		  hash=hist_expire_item_hash(grp,art);
		  for(exp=expired_list[hash],expired=0;exp!=NULL;exp=exp->next) {
			if(exp->grp==grp && exp->num==art) {
				exp->used=1;
				/*printf("REMOVING %s\n",s);*/
				break;
			}
	 	  }
		  if(!expired) {
			i=strlen(localcopy);
			if(localcopy[i-1]!='\n') 
			{ localcopy[i]='\n'; localcopy[++i]='\0'; }
			delimit=strchr(localcopy,'\t');
			if(delimit)	{
				memcpy(mid,localcopy,delimit-localcopy);
				mid[delimit-localcopy]='\0';
				hist_cache_add(localcopy,mid,file_size);
			}
			/*printf("KEEPING %s\n",s);*/
		  }
		  grpn = hist_groupnext();
		}
	}
	close(hlcl); hlcl = -1;

	hist_save_expired_list();
	hist_expire_list_free();

	/*hfiles[minidx].ord=0;*/

	/* Change file order */
	for(i=0;i<HIST_MAX_FILES;i++) {
		if(hfiles[i].ord>0) hfiles[i].ord--;
	}
	file_count--;
	hist_save_file_order();
	
	remove(file_hist(last_hist_file));
	readonly_unpause();
	return TRUE;
}

int hist_new_file(void)
{
	int i;
	for(i=0;i<HIST_MAX_FILES;i++) {
		if(hfiles[i].ord<=0) {
			lprintf(last_hist_file,12,"hist%0.4d.dat",i);
			hfiles[i].ord=(++file_count);
			hist_save_file_order();
			hist_open_files(FALSE);
			return h_his;
		}
	}
	emsg("FATAL ERROR, all history files used up, configuration error likely\n");
	return 0;
}

void hist_append(char *s)
{
        int len = strlen(s);
        nbuffed++;
        if (len>10000) wmsg("hist: very long line in history %d \n",len);
	
	if(hist_page_size) {
		if((long)file_size+len>=(long) hist_page_size) {
			hist_new_file();
		}
	}	
	if ((int) write(h_his,s,len)!=len) filemsg("History write failed\n");
	file_size += len;
}
int hist_open_read(char *fname)
{
        return hist_open(fname, O_RDONLY | O_BINARY,TRUE);
}

int hist_open_write(char *fname)
{
	if (is_readonly()) {
		emsg("slave tried to open history file\n");
		return -1;
	}
        return hist_open(fname,O_WRONLY | O_APPEND | O_BINARY,FALSE);
}
int hist_raw_open(char *fname, int mode, int smode)
{
#ifdef VMS
	char alq[BFSZ];
	int oldsize = 20000000/512;
	sprintf (alq,"alq=%d",oldsize);
	return open (fname,mode,smode,alq,"fop=cbt,tef");
#else
#ifdef HAS_SEEK64
	return open64(file_hist(fname),mode,smode);
#else
	return open_log(file_hist(fname),mode,smode);
#endif
#endif
}
int hist_raw_close(int x)
{
#ifdef HAS_SEEK64
	return close(x);
#else
	return xclose(x);
#endif
}
#ifdef WIN32
void sleep(int);
#endif

int hist_open(char *fname,int mode, int isread)
{
	static int nfail;
	int hlcl;
	nfail=0;
	hist_isread = isread;
	imsg("hist_open: %s %s\n",file_hist(fname),isread ? "Read" : "Write");

again:
	hlcl =  hist_raw_open(fname,mode,U_RW);
	dmsg("hist: History opened, handle = %d\n",hlcl);
	nfail++;
	if (nfail>3600) {
	 	emsg("History open failed for 20 minutes ,exiting\n");
		exit(0);
	}
	if (hlcl<0) {
		if (errno==ENOENT) {
	                filemsg("hist: History file doesn't exist, trying create %s\n",fname);
	                hlcl =  hist_raw_open(fname,O_WRONLY | O_BINARY | O_CREAT,U_RW);
	                if (hlcl>=0) { 
	                        hist_raw_close(hlcl); 
				goto again;
	                }
		}
		filemsg("hist: Open failed, check there is only one DNEWS running %s\n",DNEWS_HISTORY);
		filemsg("hist: Also check if a backup or other process has locked this file \n"); 
		sleep(1);
		goto again;
	}
	file_size=file_length(hlcl);
	return hlcl;
}

int hist_find_oldest_file()
{
	int i;
	for(i=0;i<HIST_MAX_FILES;i++) {
		if(hfiles[i].ord==1) {
			return i;
		}
	}
	emsg("find_oldest_file failed\n");
	dnews_ex("Failed to find oldest history\n");
	return -1;
}

/* To be used during history init only - assumes current page is the newest file */
int hist_load_curr_file(int fileno, int isread)
{
	h_his = -1;
	if(fileno>=HIST_MAX_FILES) return 0;
	if (hfiles[fileno].file>=0) {
		dnews_ex("Failed hfiles handle not closed");
	}
	lprintf(last_hist_file,12,"hist%0.4d.dat",fileno);
	hfiles[fileno].file = hist_open_write(last_hist_file);
	if (hfiles[fileno].file<0) {
		filemsg("Could not open (%s)\n",last_hist_file);
		return FALSE;	
	}
	current_page_id=fileno;
	h_his=hfiles[fileno].file;
	file_size=file_length(h_his);
	return TRUE;
}

int hist_save_file_order(void)
{
	int ord[HIST_MAX_FILES],i,f;

	f=open(DNEWS_HISTORY_ORDER,O_WRONLY | O_BINARY | O_CREAT,U_RW);
	if (f<0) {
		imsg("hist: Could not open file order list for writing\n");
		return FALSE;
	}

	for(i=0;i<file_count;i++) {
		ord[hfiles[i].ord-1]=i;
	}

	i=write(f,ord,file_count*sizeof(int));
	close(f);
	if(i!=(int) (file_count*sizeof(int))) {
		emsg("hist: Failed to write complete file order list\n");
		return FALSE;
	}
	return TRUE;
}

struct fileInfo {int file,date;};

int dateComp(const void *a,const void *b)
{
	int i;
	i=((struct fileInfo *)a)->date - ((struct fileInfo *)b)->date;
	if(!i) {
		return ((struct fileInfo *)a)->file - ((struct fileInfo *)b)->file;
	}
	else return i;
}

void hist_guess_file_order()
{
	struct stat file;
	int i;
	struct fileInfo files[HIST_MAX_FILES];

	imsg("hist: %s not found - looking for existing history files\n",DNEWS_HISTORY_ORDER);

	/* Find all available history files */
	for(i=file_count=0;i<HIST_MAX_FILES;i++) {
		lprintf(last_hist_file,12,"hist%0.4d.dat",i);
		if(!stat(file_hist(last_hist_file),&file)) {
			files[file_count].date=file.st_ctime;
			files[file_count].file=file_count;
			file_count++;
		}
	}

	/* Sort by date */
	qsort((void*)files,file_count,sizeof(struct fileInfo),dateComp);

	/* Set order in file information */
	for(i=0;i<file_count;i++) {
		hfiles[files[i].file].ord = i;
	}

	hist_save_file_order();
	imsg("hist: Found %d history files%s\n",file_count,file_count?"":" - added to file list");
}

void hist_load_file_history(void)
{
	int fileOrder[HIST_MAX_FILES],i;
	h_type c;
	FILE *f;
	imsg("hist_load_file_history\n");
	/* Read in history file order */
	f=fopen(DNEWS_HISTORY_ORDER,"rb");
	if(f==NULL) {
		hist_guess_file_order();
		return;
	}
	c = fread(fileOrder,sizeof(int),HIST_MAX_FILES,f);
	if (c<=0) {
		filemsg("hist: FATAL ERROR READING (%s)\n",DNEWS_HISTORY_ORDER);
		fclose(f);
		hist_guess_file_order();
		return ;
	}
	fclose(f);

	for(i=0;i<file_count;i++) {
		if(fileOrder[i]<HIST_MAX_FILES && fileOrder[i]>-1) {
			hfiles[fileOrder[i]].ord = i;
		}
	}	
	file_count = c;
}
int hist_find_age(int k)
{
	int i;
	for (i=0; i<HIST_MAX_FILES; i++) {
		if (hfiles[k].ord==k) return i;
	}
	return -1;
}
int hist_open_files(int isread)
{
	int i,j,r;
	if (!hist_page_size) {
		if (h_his>=0) {
			dnews_ex("History file was already open");
		}
		if (isread) h_his = hist_open_read(last_hist_file);
		else h_his = hist_open_write(last_hist_file);
		if (h_his<0) {
			filemsg("Unable to open (%s)\n",last_hist_file);
		}
		return TRUE;
	} 
	for(i=1;i<=file_count;i++) {
		j = hist_find_age(i);
		r = TRUE;
		if (!isread) if (i==file_count) r = FALSE;
		if (j>=0) hist_load_curr_file(j,r);
		if (h_his<0) {
			filemsg("Unable to open (%s)\n",last_hist_file);
		}
	}	
	return TRUE;
}


static int seeked;
int hist_rename_old_new(void)
{
	char xold[BFSZ];
	char xnew[BFSZ];
	ncpy(xold,file_hist("history.dat"),BFSZ-1);
	ncpy(xnew,file_hist("hist0000.dat"),BFSZ-1);
	if (lib_exists(xold)) {
		if (rename(xold,xnew)!=0) {
			filemsg("rename failed (%s)\n",xnew);	
		}
	}
	return TRUE;
}
int hist_init(void)
{
        char *rec;
        h_type fz;
        int rlen;
        char *s;
	int rc;
        int nr=0;
        char mid[600]; 

	if (is_slave()) return TRUE;
        hist_malloc_init();

	if(hist_page_size=atoi(config_get(C_HISTORY_PAGE_SIZE))) {
		hist_rename_old_new(); /* history.dat -> hist0000.dat */
		hver=7;
		hwords=HWORDS+1;
		memset(hfiles,-1,HIST_MAX_FILES*sizeof(Hfile *));
		hist_load_file_history();
	}
	hist_open_files(TRUE);

	rc = atoi(config_get(C_HISTORY_CHUNK));
	if (rc>0) rchunk = rc;
	if (rchunk<0) rchunk = 400;
	if (rchunk>3999) rchunk = 3999;
	fz = hist_file_length(h_his);
	file_size = fz;

	if (get_expire_mode()) {
		imsg("Not reading history into memory as in expire mode\n");
		return TRUE;
	}
        if (hist_idx_read(fz)) return TRUE;

        emsg("hist: No index found, building one from scratch\n");
        /* with 40 mega bytes, we want about 80,000 in our hash table*/
        /* file_size += len; */

        hist_size = (int) (fz/500);
        if (hist_size<10000) hist_size = 10000;  /* should be 40000 */
        if (hist_size>600000) hist_size = 600000; /* Should be big enough */    

        hidx = (Hpnt **) mycalloc(hist_size,sizeof(Hpnt *) );
        if (hidx==NULL) {emsg("fatal: Out of Memory in history"); return FALSE;}

        hist_idx_add(hver,hver,hver,hver);
        hist_idx_add(hist_size,hist_size,0,0);

        /* If startup is too slow, we could save the index in a file */
        file_size = 0;
        rec = hist_line(0);
        for (;rec!=NULL;) {
                nr++;
                s = strchr(rec,'\t');
                if (s==NULL) {emsg("fatal: history file corrupt, no tab found \n"); return FALSE;}
                rlen = s-rec;
                if (rlen>500) rlen = 500;
                ncpy(mid,rec,rlen);

                seeked = FALSE;
                hist_addidx(mid,file_size);
                file_size += strlen(rec)+1; /* EOL character is not returned */
                rec = hist_next();
        }
        imsg("hist: Num records %d \n",nr);
        hist_idx_flush();
        return TRUE;
}

/* Rebuild the history file */
static int islogging;
void hist_rebuild(void)
{
        char *rec,*s;
        char mid[600]; 
        int rlen;
        int nd=0,nr=0;
        file_size = 0;

        hist_idx_clear();
        hidx = (Hpnt **) mycalloc(hist_size,sizeof(Hpnt *) ); /* shouldbe int* */
        if (hidx==NULL) emsg("hist: Memory allocation failure hidx\n");
        rec = hist_line(0);
        for (;rec!=NULL;) {
                nr++;
                s = strchr(rec,'\t');
                if (s==NULL) {
                        emsg("fatal: history file corrupt, no tab found \n");
                        goto nextrec;
                }
                rlen = s-rec;
                if (rlen>600) rlen = 600;
                ncpy(mid,rec,rlen);


                seeked = FALSE;
                if (hist_exists(mid)!=NULL) {
                        nd++;
                        if (nd<10) emsg("fatal: Duplicate record found, ignoring (should delete) {%s} \n",mid);
                } else {
                        hist_addidx(mid,file_size);
                        hnew_write(rec);
                }
                if (seeked) {
                        hist_line(file_size);
                }
nextrec:
                file_size += strlen(rec)+1; /* EOL character is not returned */
                rec = hist_next();
        }
        emsg("hist: Num records %d,  deleted %d \n",nr,nd);
        hnew_close();
        hist_close();
        hist_switch();
}
void hist_reload(void)
{
        hist_close();
        hist_open_files(TRUE);
        hist_idx_read(hist_file_length(h_his));
	hist_idx_new(); /* Create new memory index if needed */
}
void hist_addidx(char *msgid, h_type offset) 
{
        int sh,match;
        sh = hist_hash(msgid,hist_size,&match);
 /*wmsg("hist: Hased to %d id {%s} offset %d  \n",sh,msgid,offset);  */
        hist_idx_add(sh,offset,match,current_page_id);
		hist_addone(&hidx[sh],offset,match,current_page_id);
}
void hist_analyze(void)
{
	Hpnt *h;
	int nh;
	int most = 0;
	int i; 
	int tot=0;
	int nany=0;
	for (i=0; i<hist_size; i++) {
		h = hidx[i];
                for (nh = 0;h!=NULL;h = h->next) {
			nh++;
                }
		if (nh>0) nany++;
		if (most<nh) most = nh;
		tot += nh;
		if (i<1000) if (nh>0) imsg("[%d]=%d\n",i,nh);
	}
	if (hist_size==0) return;
	if (nany==0) return;
	emsg("History tot=%d %d/%d  average %d %d worst=%d\n",tot,nany,hist_size
		,tot/hist_size,tot/nany,most);
}

int hist_addone(Hpnt **hh, h_type offset, int match, int page)
{
        Hpnt *h = *hh;
        Hpnt *tmp;

        total_mem += sizeof(Hpnt);
        tmp = hist_malloc(sizeof(Hpnt));
        if (tmp==NULL) {emsg("fatal: Out of Memory in history used %d \n",total_mem); return FALSE; }
        h = tmp;
	if (offset<0) {offset = 0; emsg("History offset faulty\n");}
        h->pnt = offset;
	h->match = match;
        h->next = *hh;
		if(hist_page_size) {
			h->page = page;
		} else {
		}
	*hh = h;       
        return TRUE;
}

int hist_addone_old(Hpnt **hh, int offset, int match)
{
        Hpnt *h = *hh;
        Hpnt *tmp,*last;

        total_mem += sizeof(Hpnt);
        tmp = hist_malloc(sizeof(Hpnt));
        if (tmp==NULL) {emsg("fatal: Out of Memory in history used %d \n",total_mem); return FALSE; }
        if (h==NULL) {
                *hh = tmp;
        } else {
                for (last = h;h!=NULL;h = h->next) {
                        last = h;
                }
                last->next = tmp;
        }
        h = tmp;
        h->pnt = offset;
	h->match = match;
        h->next = NULL;
        return TRUE;
} 

int hist_find(char *msgid, int *grp, int *num)
{
        int sh,match,oldgrp;
        Hpnt *h;
        char *s;
	int n;
	oldgrp = *grp;
	
	if (strcmp(msgid,"<fake@netwin.com.nz>")==0) {
                if (*grp<0) *grp = 0;
                *num = group_getfake(*grp);
                return TRUE;
        }
        *grp = 0; *num = 0;
        sh = hist_hash(msgid,hist_size,&match);
	if (hidx==NULL) return FALSE;
/*      dmsg("hist: hash {%s} to %d %d \n",msgid,sh,hist_size);  */
        h = hidx[sh];
        if (h==NULL) goto fail;
        for (; h!=NULL; h = h->next) {
		if (match==h->match) {
		s=hist_line_multi(h);
                /*dmsg("hist: hist_find %d %p \n",*h,s);
                if (s!=NULL) dmsg("hist: find %d {%s} \n",*h,s);*/
                if (s==NULL) {
                        emsg("hist: Fatal error hist_find NULL ptr EOF \n");
                        goto fail;
                }
                if (strncmp(s,msgid,strlen(msgid))==0) {
                        if (s[strlen(msgid)]=='\t') {
                                *grp = hist_group(s,0);
                                *num = hist_num(s,0);
                                if (*grp<0) {*grp=0; return FALSE;}
                                goto return_ok;
                        }
                }       
		}
        }
	/* Check the cache, it may have just been posted. */
fail:
        s = hist_cache_find(msgid);
	if (s==NULL) return FALSE;
        if (strncmp(s,msgid,strlen(msgid))==0) {
               if (s[strlen(msgid)]=='\t') {
                    *grp = hist_group(s,0);
                    *num = hist_num(s,0);
                    if (*grp<0) {*grp = 0; return FALSE;}
                    goto return_ok;
               }
         }       
	return FALSE;
return_ok:
	/* We need to return the current group if possible (for crossposts) */
	if (oldgrp<0) {
	    	s = hist_groupfirst(s);
	    	for (n=0;s!=NULL;n++) {			
	       		s = hist_groupnext();
		}
		if (n==1) return TRUE; /* No cross posted items so it must be right*/
		group_unknown(TRUE);
		return TRUE;
	}
    	s = hist_groupfirst(s);
    	for (;s!=NULL;) {
		if (strcmp(group_name(oldgrp),s)==0) {
			*grp = oldgrp;
			*num = hist_groupnextnum();
			dmsg("hist: Cross post matched old group :-)   %s %d %d \n",group_name(oldgrp),*grp,*num);
			return TRUE;
		}
       		s = hist_groupnext();
	}
	group_unknown(TRUE);
	return TRUE;
}
static int hfast;
int hist_setfast(void)
{
        hfast = !hfast;
        if (hfast) emsg("History set to FAST lieing mode\n");
        else emsg("History set to normal mode \n");
        return TRUE;
}
/* Bug, should cache last few items so if arrives on different
channel we say 'no thanks' */
#define MAXHIT2 200
static char *hit2_mid[MAXHIT2];
static int hit2_chan[MAXHIT2];
static int hupto;
int hist_freechan(int chan)
{
	int i;
	for (i=0; i<MAXHIT2; i++) {
		if (hit2_chan[i]==chan) {
		 if (hit2_mid[i]!=NULL) {
			mystrfree(hit2_mid[i],SD_HIST);
			hit2_mid[i] = NULL;
		 }
		}
	}
	return TRUE;
}
int hist_gotit2(char *msgid, int chan)
{
	int i,h;

	if (config_true(C_HIST_NEVER)) {
		return FALSE;		
	}

	for (i=0; i<MAXHIT2; i++) {
		if (hit2_chan[i]!=chan) if (hit2_mid[i]!=NULL)
		  if (strcmp(hit2_mid[i],msgid)==0) {
			return TRUE;
		}
	}
	h = hist_gotit(msgid);
	if (!h) {
		hupto++;
		if (hupto>=MAXHIT2) hupto = 0;
		if (hit2_mid[hupto]!=NULL) mystrfree(hit2_mid[hupto],SD_HIST);
		hit2_mid[hupto] = mystrdup(msgid,SD_HIST);
		hit2_chan[hupto] = chan;
	}
	return h;
}
int hist_gotit(char *msgid)
{
        static int lastret;
        static char lastmsgid[400];
	static int lastc;
	char *s;

	if (config_true(C_HIST_NEVER)) {
		return FALSE;		
	}


        if (hfast) return FALSE;
        if (kill_last) strcpy(lastmsgid," "); kill_last = FALSE;
	if (strlen(msgid)>96) {
		imsg("Rejecting message id that is too long %d\n",strlen(msgid));
		return FALSE;
	}
        if (strcmp(msgid,lastmsgid)==0) return lastret;
	
        ncpy(lastmsgid,msgid,399);
	s = strchr(lastmsgid,'>');
	if (s!=NULL) *(s+1) = 0;
        if (hist_exists(lastmsgid)==NULL) {
                lastret = FALSE;
		if (!in_part) if (part_hist(lastmsgid)) lastret = TRUE;
        } else {
                lastret = TRUE;
	}
        return lastret;
}
static int testhist;
void hist_set_test(int x)
{
	testhist = x;
}

int hist_forget(char *msgid)
{
        int sh,match;
        Hpnt *h;
        char *s;

	hist_flush();
        sh = hist_hash(msgid,hist_size,&match);
        if (hidx==NULL) {emsg("hist: hidx is NULL, CRASHING");}
        h = hidx[sh];
        if (h!=NULL) {
          for (; h!=NULL; h = h->next) {
	  	if (h->match==match) {
                 s = hist_line(h->pnt);
                 if (s==NULL) {
                        emsg("hist_forget: Fatal error hist_exists NULL ptr  {%s}\n",msgid);
                        return FALSE;
                 }
                 if (strncmp(s,msgid,strlen(msgid))==0) {
                        if (s[strlen(msgid)]=='\t') {
				h->match++;
				return TRUE;
                        }
                 }
                }       
          }
        }
        return FALSE;
}
void hist_set_part(int x)
{
	in_part = x;
}

char *hist_exists(char *msgid)
{
        int sh,match;
        Hpnt *h;
        char *s;
	vtrace();
        islogging = FALSE;

	if (config_true(C_HIST_NEVER)) {
		return NULL;		
	}
        sh = hist_hash(msgid,hist_size,&match);
        if (hidx==NULL) {emsg("hist: hidx is NULL, CRASHING, hist_exists\n"); dnews_ex("hidx is null, save dnews.log and configs and send to netwin, don't use header sucking with dmulti!\n");}
        h = hidx[sh];
        if (islogging) emsg("hist: exists?, [%d:%d] points=%p {%s} \n",sh,hist_size,h,msgid);
        if (h!=NULL) {
          for (; h!=NULL; h = h->next) {
	  	if (h->match==match) {
		s=hist_line_multi(h);
                 if (s==NULL) {
                        emsg("hist: Fatal error hist_exists NULL ptr  {%s}\n",msgid);
			vtrace();
                        return NULL;
                 }
                 if (islogging) emsg("hist: exists, seek %p, got {%s} \n",h,s);
		/*imsg("hist_compare (%s) (%s) \n",s,msgid);*/
                 if (strncmp(s,msgid,strlen(msgid))==0) {
                        if (s[strlen(msgid)]=='\t') {
				vtrace();
                                return s;
                        }
                 }
                }       
          }
        }
	vtrace();
        /* Maybe it's in our recent cache on vms? */
        return hist_cache_find(msgid);
}

#define MAXHLINE 12000
static char hline[MAXHLINE];
static int blen;

char *hist_line_multi(Hpnt *h)
{
	
	int file;
	char *line;
	if (!hist_page_size) file = h_his;
	else {
		if (h->page<0) return NULL;
		if (h->page>HIST_MAX_FILES) return NULL;
		file=hfiles[h->page].file;
	}

	if (file<0) return NULL;

	/*printf("hist_line_multi page/file: %d(%d) seek: %d\n",h->page,file,h->pnt);*/
	
	seeked=TRUE;
	if (h_seek(file,h->pnt,SEEK_SET)== -1) filemsg("hist: page(handle) %d(%d) seek %d \n",h->page,file,h->pnt); 
	upto = h->pnt;
	notatend = TRUE;

	blen = 0;
	hline[0] = 0; hline[1] = 0;
	line=hist_next_multi(file);
	return line;
}

char *hist_line(h_type h) /* Ugly, reads until EOL */
{	
	if (h_his<0) {
		hist_open_files(TRUE);
	}
	if (h_his<0) return NULL;
        
	seeked = TRUE;
        if (h_seek(h_his,h,SEEK_SET)== -1) filemsg("hist: handle %d seek %d \n",h_his,h); 
        upto = h;
        notatend = TRUE;
        blen = 0;
        hline[0] = 0; hline[1] = 0;
        return hist_next();
}

char *hist_next_multi(int file)  /*Read the next line, and return NULL on EOF */
{
     int nread = -1;
        char *s;
        int len;
	int rest;

        len = strlen(hline); /* Search for dummy NULL */
        if (len>(MAXHLINE-2)) len = MAXHLINE-100; 
	rest = blen-len+1;
	if (rest+len>MAXHLINE+2) {emsg("hist_next error1 %d\n",rest); rest = 2;}
	if (rest<0) {emsg("hist_next error2 %d\n",rest); rest = 2;}

        memmove(hline,hline+len+1,rest); /* Move the rest over */
        blen = blen - (len+1);
        if (blen<0) blen = 0;
        if (hline[0]==0) blen = 0;
again:
        s = strchr(hline,'\n');
        if (s!=NULL) {
                *s = 0;
                upto += strlen(hline) + 1;  /* 2.4q */ 
                return hline;
        }
        if (nread==0) {
                return NULL;
        } /* End of file found */

        if (blen>8000) {
		hline[50] = 0;
                emsg("fatal: Error in history file - long too line - skipping line {%s}\n",hline); 
                blen = 0; hline[0] = 0;
                /* return NULL;  */
        }

        nread = read(file,hline+blen,rchunk);
	if (nread<0) filemsg("Error reading history file, PANIC\n");
        if (nread<0) nread = 0;
	blen = blen + nread;
        hline[blen] = 0;
        goto again;
}

/* Read the next line, and return NULL on EOF */
/* Only valid after a call to hist_line */
char *hist_next(void)  /*Read the next line, and return NULL on EOF */
{

	return hist_next_multi(h_his);
}
char *hist_line2(h_type h) /* Ugly, reads until two EOL's so it's got a whole record*/
{
        char *s;
        int add=0;
        int n;

        notatend = TRUE;
        if (h_seek(h_his,h,SEEK_SET)== -1) {
		filemsg("hist: seek %d \n",h); 
		return NULL;
	}
        
again:
        if (add>8000) {emsg("Error in history file - long line \n"); return NULL;}
        n = read(h_his,hline+add,rchunk);
	if (n<0) filemsg("Error reading history file, PANIC\n");
        if (n==0) return NULL;
        if (n<0) {emsg("fatal: Reading history file, io error"); return NULL;}
        hline[n+add] = 0;
        s = strchr(hline,'\n');
        if (s==NULL) {add += rchunk; goto again;}
        if (strchr(s+1,'\n')==NULL) {add += rchunk; goto again;}
        upto += strlen(hline) + 1; /* 2.4q */ 
        return hline;
}
char *hist_after(int date)
{
        /* Binary search for first message which 'arrived' after 'date' */ 
        h_type lower=0;
        h_type middle;
        char *s,*start;
	h_type upper;

	if (h_his<0) {
		hist_open_files(TRUE);
	}
	if (h_his<0) return NULL;
        upper=hist_file_length(h_his);

        for (;TRUE;) {
                middle = upper/2 + lower/2;
                s = hist_line2(middle);
                if (s==NULL) break;  /* Answer must be lower*/ 
                start = strchr(s,'<');
                if (start==NULL) {emsg("fatal: impossible history error \n"); abort();} 
                middle = middle + start - s;  /*Correct so is start of line*/
                if (middle>=upper) break;
                if (hist_date_arrived(start)>date) upper = middle;
                else if (lower==middle) break;
                else lower = middle;
        }       
        s = hist_line(lower);
        for (;s!=NULL;) {
                if (hist_date_arrived(s)>date) return s;
                s = hist_next();
        }
        return NULL;
}
int hist_date_expires(char *rec)  /*Second date, zero if is '-' */
{
        char *s;
        s = strchr(rec,'\t');
        if (s==NULL) {emsg("fatal: history file corrupt1 {%s}\n",rec); return 0;}       
        s = strchr(s+1,'~');
        if (s==NULL) {emsg("fatal: history file corrupt3 {%s}\n",rec); return 0;}       
        return atoi(s+1);
}
int hist_date_posted(char *rec)  /*Second date, zero if is '-' */
{
        char *s;
        s = strchr(rec,'\t');
        if (s==NULL) {emsg("fatal: history file corrupt1 {%s}\n",rec); return 0;}       
        s = strchr(s+1,'~');
        if (s==NULL) {emsg("fatal: history file corrupt3 {%s}\n",rec); return 0;}       
        s = strchr(s+1,'~');
        if (s==NULL) {emsg("fatal: history file corrupt3 {%s}\n",rec); return 0;}       
        return atoi(s+1);
}
int hist_date_arrived(char *rec)
{
        char *s;
        s = strchr(rec,'\t');
        if (s==NULL) {emsg("fatal: history file corrupt4 {%s}\n",rec); return 0;}       
        return atoi(s+1);
}
static char *gpoint;
char *hist_groupfirst(char *rec)
{
        char *s;

        s = strchr(rec,'\t');
        if (s==NULL) {emsg("fatal: history file corrupt5 {%s}\n",rec); return NULL;}    
        s = strchr(s+1,'\t');
        if (s==NULL) {emsg("fatal: history file corrupt6 {%s}\n",rec); return NULL;}    
        gpoint = s+1;
        return hist_groupnext();
}
static int nextnum;
int hist_groupnextnum(void)
{
        return nextnum;
}
char *hist_groupnext(void)
{
        static char gname[200];
        char *s2;
        int len;

        if (gpoint==NULL) {emsg("Hist_groupnext failed, gpoint null\n"); return NULL;}
        s2 = strchr(gpoint,'/');
        if (s2==NULL) return NULL;
        nextnum = atoi(s2+1);
        len = s2 - gpoint;
        if (len>100) {emsg("Hist groupnext: len too long %d \n",len); return NULL;}
        ncpy(gname,gpoint,len);
        gpoint = strchr(s2,' ');
        if (gpoint==NULL) gpoint = s2 + strlen(s2);
        else gpoint++;

        return gname;
}
char *hist_gname(char *rec, int g) /* Return group and num */
{
        static char gname[200];
        char *s,*s2;
        int i,len;

        s = strchr(rec,'\t');
        if (s==NULL) {emsg("fatal: history file corrupt7 {%s}\n",rec); return NULL;}    
        s = strchr(s+1,'\t');
        if (s==NULL) {emsg("fatal: history file corrupt8 {%s}\n",rec); return NULL;}   
        s++;
        for (i=0; i<g; i++) { /* Skip over 'g' items */ 
                s = strchr(s,' ');
                if (s==NULL) return NULL;
                s++;
        }
        s2 = strchr(s,'/');
        if (s2==NULL) return NULL;
        len = s2 - s;
        if (len>150) {emsg("fatal: history file corrupt, len %d \n",len); return NULL;}
        ncpy(gname,s,len);
        return gname;
}

int hist_group(char *s,int n)
{
        int x;
        char *grp = hist_gname(s,n);
        if (grp==NULL) return -1;
        x = group_find(grp);
        return x;
}
int hist_num(char *rec,int g)
{
        char *s,*s2;
        int i;

        s = strchr(rec,'\t');
        if (s==NULL) {emsg("fatal: history file corrupt9 {%s}\n",rec); return 0;}       
        s = strchr(s+1,'\t');
        if (s==NULL) {emsg("fatal: history file corrupt a {%s}\n",rec); return 0;}     
        s++;

        for (i=0; i<g; i++) { /* Skip over 'g' items */ 
                s = strchr(s,' ');
                if (s==NULL) return 0;
                s++;
        }
        s2 = strchr(s,'/');
        if (s2==NULL) return 0;
        return atoi(s2+1);
}


static int hnew_open = FALSE;
static int n_out = -1;
static FILE *n_outf = NULL;
static int new_size,new_hsize;
void hnew_setfail(void);
void hnew_setfail(void)
{
	if ((n_out>=0) || (n_outf!=NULL)) hnew_close();
	hnew_open = FALSE;
        remove(DNEWS_HISTORY_NEW_IDX);
        remove(DNEWS_HISTORY_NEW);
}
int hnew_write(char *s)
{
        int len,h;
	int match;
        char rec[10000];
        int show=FALSE;
	static char hbuf[32000];
        ncpy(rec,s,9996);
        strcat(rec,"\n");

/*      if (strstr(rec,"3ti8hn")!=NULL) show=TRUE; open */


        if (show) dmsg("hist: addrec %s",rec);
        if (!hnew_open) {
                new_size = 0;
                remove(DNEWS_HISTORY_NEW_IDX);
		if (!config_true(C_HISTORY_SYSOPEN)) {
			n_outf = myfopen(DNEWS_HISTORY_NEW,"wb");
			if (n_outf==NULL) {
	                	emsg("fatal: Could not open/create history file {%s} \n",DNEWS_HISTORY_NEW); 
	                	hnew_setfail();
	                	return FALSE;
			}
#ifndef VMS
			setvbuf( n_outf, hbuf, _IOFBF, sizeof( hbuf ) ); 
#endif
		} else {
	                n_out =  hist_raw_open(DNEWS_HISTORY_NEW,O_TRUNC | O_WRONLY | O_CREAT | O_BINARY,U_RW);
	                if (n_out<0) {
	                	emsg("fatal: Could not open/create history file {%s} \n",DNEWS_HISTORY_NEW); 
	                	hnew_setfail();
	                	return FALSE;
	                }
		}
		hnew_open = TRUE;
                
                new_hsize = (int) (file_size/500);
                if (new_hsize<10000) new_hsize = 10000;  /* should be 40000 */
                if (new_hsize>600000) new_hsize = 600000; /* Should be big enough */    
		imsg("hist: new_hist_hash %d\n",new_hsize);
                hnew_idx_add(HVER,HVER,HVER); /* New format*/
                hnew_idx_add(new_hsize,0,0);      
                dmsg("hist: hsize %d",new_hsize);
        }
        s = strchr(rec,'\t');  if (s==NULL) return TRUE;
        if (s==rec) return TRUE;
        if (*(s-1) != '>') {
                emsg("hist: Message-id contained rubbish, deleting it {%s} \n"
                        ,lib_safe(s));
                return TRUE;
        }

        *s = 0; h = hist_hash(rec,new_hsize,&match);
        if (show) dmsg("hist: str_hash {%s} %d %d \n",rec,h,new_hsize);

         *s = '\t';
        if (show) dmsg("hist: hased it to %d / %d",h,new_hsize);
        if (h>=new_hsize) emsg("hist: fatal: writing new hash error %d %d %d \n",h,new_hsize,new_size);
        hnew_idx_add(h,new_size,match);       
        
        len = strlen(rec);

	if (!config_true(C_HISTORY_SYSOPEN)) {
		if ((int) fwrite(rec,1,len,n_outf) != len) {
			filemsg("FATAL Error writing new history file\n");
			hnew_setfail();
	        	return FALSE;
	        }
	} else {
		if ((int) write(n_out,rec,len) != len) {
			filemsg("FATAL Error writing new history file\n");
			hnew_setfail();
	        	return FALSE;
	        }
	}
        new_size += len;
	return TRUE;
}

void hnew_close(void)
{
        hnew_idx_flush();
        if (n_out!= -1) hist_raw_close(n_out);
        if (n_outf!= NULL) fclosep(n_outf);
        n_out = -1;
        n_outf = NULL;
        hnew_open = FALSE;
}
void hist_delete(void)
{
	emsg("Deleting history file for rebuild, ignore rename warnings\n");
	hist_flush();
 	hist_close();
	remove(DNEWS_HISTORY_NEW);
	remove(DNEWS_HISTORY_NEW_IDX);
	hist_switch();
}
void hist_switch(void)
{
        char bf[500];

        int r;
	int failcount=0;

	imsgf("hist: History Switch\n");
        hnew_close();
        hist_close();
        
	imsgf("hist: Pause readonly servers\n");
	readonly_pause(); /* Tell all reading servers to pause and close files */
	imsgf("hist: Delete old history file\n");
again:  r = remove(DNEWS_HISTORY); 
	imsgf("hist: Delete old history file, completed %d\n",r);
	if (r!=0) if (errno!=ENOENT) {
		lib_sleep(1); /* Wait 20 seconds*/
		failcount++;
		filemsg("Waiting for readonly servers to release history file\n");
		if (failcount<900) goto again;
	}
	if (r!=0) if (errno!=ENOENT) {
		emsg("hist: 1unable to delete %s \n",DNEWS_HISTORY); 
		if (get_expire_mode()) {  /* We are a spawned expire process */
		    emsg("Attempting to tell master that we have finished\n");
		    readonly_unpause();
                    tell_master("history_flush_unpause");
		    expire_write_current_status("FATAL ERROR, COULD NOT DELETE HISTORY.DAT, SAVE DNEWS*.LOG*\n");
		    tell_master("expire_done");
		}
		dnews_ex("History file could not be deleted, readonly server must be locked up");
	}
	imsgf("hist: Delete old history index\n");
        r = remove(DNEWS_HISTORY_IDX); 
        if (r!=0) {
                emsg("hist: 2unable to delete %s \n",DNEWS_HISTORY_IDX); 
                filemsg("hist.idx");
        }
	imsgf("hist: Delete old history index, completed\n");
        ncpy(bf,DNEWS_HISTORY_NEW,400);
        r = rename(bf,DNEWS_HISTORY);  if (r!=0) emsg("hist: 3unable to delete %s \n",DNEWS_HISTORY_NEW); 
        ncpy(bf,DNEWS_HISTORY_NEW_IDX,400);
        r = rename(bf,DNEWS_HISTORY_IDX);  if (r!=0) emsg("hist: unable 4to delete %s \n",DNEWS_HISTORY_NEW_IDX); 
	imsgf("hist: Renames completed\n");
	readonly_unpause();
	imsgf("hist: Unpaused readonly servers\n");
	hist_open_files(TRUE);

	imsgf("hist: About to read index in again\n");
	if (get_expire_mode()) {
		imsgf("hist: Expire mode so not reading history back in either\n");
		return;
	}
        hist_idx_read(hist_file_length(h_his));
	imsgf("hist: History read in now\n");
	hist_idx_new(); /* Create new memory index if needed */
	imsgf("hist: Finished switch\n");
}
int hist_idx_new(void)
{
        if (hidx==NULL) {
                emsg("New history file is empty, must have expired the lot !!!\n");
                hist_size = 10000;  /* should be 40000 */
		if (db_total_space() > 1000) hist_size = 100000;
		if (db_total_space() > 10000) hist_size = 500000;
		imsg("hist: hist_hash set to %d\n",hist_size);
                hidx = (Hpnt **) mycalloc(hist_size,sizeof(Hpnt *) );
                if (hidx==NULL) {emsg("fatal: Out of Memory in history"); return TRUE;}
	        hist_idx_add(hver,hver,hver,hver);
	        hist_idx_add(hist_size,hist_size,0,0);
                file_size = 0;
        }
	return TRUE;
}
/*
void hist_idx_add(int a, h_type b, int c)
{
        ibf[nibf++] = a;        
#ifdef HIST64
	{
		h_both bb;
		bb.whole = b;
		ibf[nibf++] = bb.part[0];
		ibf[nibf++] = bb.part[1];
	}
#else
        ibf[nibf++] = b;             
#endif
        ibf[nibf++] = c;             
        if (nibf>3000) {
		hist_idx_flush();
		nibf = 0;
	}
}*/

void hist_idx_add(int a, h_type b, int c, int d)
{
        ibf[nibf++] = a;        
#ifdef HIST64
	{
		h_both bb;
		bb.whole = b;
		ibf[nibf++] = bb.part[0];
		ibf[nibf++] = bb.part[1];
	}
#else
        ibf[nibf++] = b;             
#endif
        ibf[nibf++] = c;             
        if(hist_page_size) ibf[nibf++] = d;
		if (nibf>3000) {
		hist_idx_flush();
		nibf = 0;
	}
}
int hist_ispage(void)
{
	if (hist_page_size) return TRUE;
	return FALSE;
}
void hist_idx_flush(void)
{		
		int h_idx,n;
        if (nibf==0) return;

		h_idx = open_log(DNEWS_HISTORY_IDX,O_WRONLY | O_APPEND | O_CREAT | O_BINARY,U_RW);
		if (h_idx<0) {
                emsg("fatal: can't write history index file {%s} \n",DNEWS_HISTORY_IDX); 
                filemsg("History index flush:");
                return; 
        }

        n = write(h_idx,(char *) ibf,nibf*sizeof(int));

		if (n<(int)(nibf*sizeof(int))) filemsg("fatal: Error writing hist index \n");
        nibf = 0;
        if (h_idx!= -1) xclose(h_idx);
        h_idx = -1;
}

static int inew[5001];
static int ninew;
void hnew_idx_add(int a, h_type b, int c)
{
        inew[ninew++] = a;      
#ifdef HIST64
	{
		h_both bb;
		bb.whole = b;
		inew[ninew++] = bb.part[0];
		inew[ninew++] = bb.part[1];
	}
#else
        inew[ninew++] = b;      
#endif
        inew[ninew++] = c;      
        if (ninew>2000) hnew_idx_flush();
}
void hnew_idx_flush(void)
{
        int h_idx,n;
        if (ninew==0) return;

        h_idx = open_log(DNEWS_HISTORY_NEW_IDX,O_WRONLY | O_APPEND | O_CREAT | O_BINARY,U_RW);
        if (h_idx<0) {filemsg("fatal: can't write new history index file\n"); return ;}

        n = write(h_idx,(char *) inew,ninew*sizeof(int));
        if (n< (int) (ninew*sizeof(int))) filemsg("fatal: Error writing hist index \n");
        ninew = 0;
        xclose(h_idx);   
}
void hist_idx_clear(void)
{
        int i;
        Hpnt *h,*next;

	imsgf("hist: Freeing old history indexes\n");
        if (hidx!=NULL) {
                for (i=0; i<hist_size; i++) {
                        h = hidx[i];
                        for (;h!=NULL;h=next) {
                                next = h->next;
                                hist_free(h,HSZ);
                        }
                }
                myfree(hidx);
                hidx = NULL;
        }
	imsgf("hist: Freeing old history indexes, completed\n");
        total_mem = 0;
}
int hist_idx_read(h_type sz)
{
        int idx,i,n;
	h_type fz;

        hist_idx_clear();
        dmsg("hist: Just cleared it, should be nothing allocated\n");
        blocks_show();
        idx = open_log(DNEWS_HISTORY_IDX,O_RDONLY | O_BINARY,U_RW);
        if (idx<0) {
		imsg("History index doesn't exist yet, thats ok\n");
                return FALSE;
        }
        
        fz = hist_file_length(idx);
	catchupfz = fz;
        dmsg("hist: idx size %d \n",fz);

        if(hist_page_size) n = read(idx,(char *) ibf,sizeof(int)*4);
	else n = read(idx,(char *) ibf,sizeof(int)*2); /** Originally 2, check for changes */
	
	if (n<2) {
		imsg("History index is empty just now, thats ok\n");
		xclose(idx); return FALSE;
	}

	if ((ibf[0]!=hver) || (ibf[1]!=hver)) { /* Do upgrade to new format */ 
		emsg("Upgrading history.idx version, may take 2-10 minutes \n");
		xclose(idx);
		remove(DNEWS_HISTORY_IDX);
	 	return FALSE;
	}
        n = read(idx,(char *) ibf,sizeof(int)*4); /* Now read new header data */
        if (n<2) {xclose(idx); return FALSE;}

        hist_size = ibf[1];
		imsg("hist: hist_hash %d\n",hist_size);

        hidx = (Hpnt **) mycalloc(hist_size,sizeof(Hpnt *) );
        if (hidx==NULL) {emsg("fatal: Out of Memory in history"); xclose(idx); return FALSE;}

        dmsg("hist: hist_size %d \n",hist_size);
        n = read(idx,(char *) ibf,sizeof(int)*1000*hwords);
        for (;n>0;) {
                n = n/sizeof(int);
                for (i=0; i<n; i+=hwords) {
                        if ((ibf[i]<0) || (ibf[i]>=hist_size) ) {
                                emsg("fatal: History index corrupt [%d] #=%d f=%d \n",i,ibf[i],ibf[i+1]);
                        } else {
#ifdef HIST64
				h_both b;
				b.part[0] = ibf[i+1];
				b.part[1] = ibf[i+2]; 
								hist_addone(&hidx[ibf[i]],b.whole,ibf[i+2],ibf[i+3]);
#else
								hist_addone(&hidx[ibf[i]],ibf[i+1],ibf[i+2],ibf[i+3]);
#endif
                        }
                }
                n = read(idx,(char *) ibf,sizeof(int)*1000*hwords);
#ifdef Mac
		lib_doevents();
#endif

        }
        xclose(idx);

        imsg("hist: Just read in new history index, hash=%d\n",hist_size);
        blocks_show();

        return TRUE;
}
void hist_catchup(void)
{
	int ngot = 0;
        int idx,i,n;
	h_type fz;

	imsg("Catching up with new history entries on master\n");
	if (hidx==NULL) {
		imsg("Can't catchup as hidx is null, trying to re-read\n");
		hist_idx_read(0);
		return;
	}
	if (catchupfz==0) {
		emsg("History size was zero, restart readonly servers\n");
		return;
	}
        idx = open_log(DNEWS_HISTORY_IDX,O_RDONLY | O_BINARY,U_RW);
        if (idx<0) {
               return ;
        }
        
        fz = hist_file_length(idx);

	if (fz<=catchupfz) goto nothing;

	h_seek(idx,catchupfz,SEEK_SET);
	imsg("Catchup from %d to %d hash=%d\n",catchupfz,fz,hist_size);
        n = read(idx,(char *) ibf,sizeof(int)*1000*hwords);
        for (;n>0;) {
                n = n/sizeof(int);
                for (i=0; i<n; i+=hwords) {
                        if ((ibf[i]<0) || (ibf[i]>=hist_size) ) {
                                emsg("fatal: Catchup, History index corrupt [%d] #=%d f=%d \n",i,ibf[i],ibf[i+1]);
                        } else {
#ifdef HIST64
				h_both b;
				b.part[0] = ibf[i+1];
				b.part[1] = ibf[i+2]; 
                                hist_addone(&hidx[ibf[i]],b.whole,ibf[i+2],0);
#else
                                hist_addone(&hidx[ibf[i]],ibf[i+1],ibf[i+2],0);
#endif
				ngot++;
                        }
                }
                n = read(idx,(char *) ibf,sizeof(int)*1000*hwords);
        }
	catchupfz = hist_file_length(idx);

nothing:
        xclose(idx);
	imsg("Catchup completed, got %d records\n",ngot);
}
h_type hist_file_length(int x)
{
#ifdef Mac
	return file_length(mac_fileno(x));
/*	return mac_length(x);*/
#else
	return file_length(x);
#endif
}
/* ------------------------------------------------------ */
/* typedef struct BLOCK {int *data; int width, size, used, upto; struct BLOCK *next;} Block; */

static Hblock blocks[10];
static Hblock *block_last[10];
static int nblocks = 1;

/* 3 * 10000 * 4 = 12 * 10000 120k  */
int block_init(Hblock *b, int width, int size)
{
        int i;
        b->width = width;
        b->size = size;
        b->used = 0;
        total_hblock += sizeof(int)*width*size;
        dmsg("hist: malloc %d total now %d \n",sizeof(int)*width*size,total_hblock);
        b->data = mymalloc(sizeof(int)*width*size);
        if (b->data == NULL) {emsg("Memory allocation failed in history\n"); dnews_ex("Memory"); }
	memset(b->data,0,sizeof(int)*width*size);
        for (i=0; i<b->size; i++) b->data[i*b->width] = -1;
        b->upto = 0;
        b->next = NULL;
        return TRUE;
}
int hist_malloc_init(void)
{
        block_init(&blocks[0],HSZ,10000);
        return TRUE;
}
void hist_corrupt(void)
{
        Hpnt *tmp;

        tmp = hist_malloc(sizeof(Hpnt));
	tmp->next = (Hpnt *)  ((char *) tmp +1);
}

void hist_check(char *reason)
{
        Hblock *b,*jb;
	int *x;
	int i,n;
	char *v;

	if (!config_true(C_HIST_DEBUG)) return;
	imsg("hist_check %s\n",reason);
	b = &blocks[0];
	for (n=0;b!=NULL; b=b->next,n++) {
	 x = b->data+2;
         for (i=0; i<b->size; i++,x+=b->width) {
		v = (char *) *x;
		if (v==NULL) continue;
		for (jb=&blocks[0]; jb!=NULL; jb = jb->next) {
		  if ((v>=(char *)jb->data) && (v<(char *) jb->data+jb->width*jb->size*4)) {
			if ((v - (char *) jb->data)%12 != 0) {
				emsg("History pointer corrupt, block [%d] at %d/%d  %p, remainder %d (%d) \n"
					,n,i,b->size
					,v,(v - (char *) jb->data)%12,v- (char *) jb->data);
				goto die;
			}
			goto isok;
		  }
		}
		emsg("History pointer corrupt, block [%d] at %d/%d  %p  \n"
			,n,i,b->size
			,v)	;
die:
		emsg("offset = %d, match = %d\n",x[-2],x[-1]);
		for (jb=&blocks[0],n=0; jb!=NULL; jb = jb->next,n++) {
		  emsg("Data blocks [%d] %x-%x\n",n,jb->data,(char *) jb->data+jb->width*jb->size*4);
		}		
		dnews_ex("history_invalid2");
isok:;
	 }
	}

}
int *hist_block(int bi)
{
        int i;
        int newsz;
        Hblock *b;

        b = block_last[bi];
        if (b==NULL) b = &blocks[bi];
        for (;(b->used >= b->size) && (b->next!=NULL);) b = b->next;
        if (b->used >= b->size) { b = &blocks[bi]; }
        for (;(b->used >= b->size) && (b->next!=NULL);) b = b->next;
        if (b->used >= b->size) {
                b->next = mymalloc(sizeof(Hblock));
                if (b->next==NULL) {emsg("Memhistory\n"); dnews_ex("Memory");}
                newsz = b->size*2;
                if (newsz > 300000) newsz = 300000;
                block_init(b->next,b->width,newsz);
                b = b->next;
        }
        if (b==NULL) {emsg("Memhistory\n"); dnews_ex("Memory"); abort();}
        for (i=b->upto; i<b->size; i++) {
                if (b->data[i*b->width] == -1) goto found_free;
        }                       
        for (i=0; i<b->upto; i++) {
                if (b->data[i*b->width] == -1) goto found_free;
        }                       
        emsg("Shouldn't ever get here in hist_block\n"); 
        blocks_show();
        dnews_ex("Block logic");

found_free:
        block_last[bi] = b;
        b->used++;
        b->upto = i; 
#ifdef NETWAREX
       dmsg("hist: malloc=%d upto=%d width=%d %p \n",b->used,b->upto,b->width
                ,&(b->data[i*b->width]));              
#endif                
        return &(b->data[i*b->width]);          
}
Hpnt *hist_malloc(int sz)
{
        int i;
        sz = sz/sizeof(int);
        for (i=0; i<nblocks; i++) if (blocks[i].width>=sz) return (Hpnt *) hist_block(i);
        /* dmsg("hist: Real malloc used %d \n",sz); */

        return (Hpnt *) mymalloc(sz*sizeof(int));
}
void hist_dofree(Hblock *b, int *x)
{
        for (;b!=NULL;b = b->next) {
                /* dmsg("hist: free %p %p-%p \n",x,b->data,b->data + b->width*b->size); */
                if ((x>= b->data) && (x< (b->data + b->width*b->size))) goto foundit;
        }
        emsg("hist: free shouldn't happen here\n"); dnews_ex("free logic");
        return;
foundit:
        *x = -1;
/* Funny, this slowed it down
        b->upto = (x - b->data)/b->width; 
        if (b->upto>b->size) imsg("hist: dofree error upto %d \n",b->upto);
*/
        b->used--; 
}
void hist_free(Hpnt *xx, int sz)
{
        int i;
        int *x= (int *) xx;
        /* dmsg("hist: free %p width %d \n",x,sz);*/
        for (i=0; i<nblocks; i++) if (blocks[i].width>=sz) {hist_dofree(&blocks[i],x); return;}
        /* dmsg("hist: free real malloc? %p \n",x); */
	emsg("hist_free, this should never happen %p, %d\n",xx,sz);
	dnews_ex("History Free failure, PLEASE send this log file to NETWIN@NETWINSITE.COM");
}
void hist_free2(Hpnt *h)
{
        if (h==NULL) {emsg("hist: free2=null\n"); return;}
        hist_free(h,HSZ);
}
void blocks_show(void)
{
        int i;
        int j;
        Hblock *b;
        int n,nn,nu;
        for (i=0; i<nblocks; i++) {
                n=0;         nn=0;
                 b = &blocks[i];
                 /* dmsg("hist: Blocks width %d\n",b->width);*/
                 for (;b!=NULL;b=b->next) {
                        n += b->size;
                        nn++;
                        for (nu=b->size,j=0; j<b->size; j++) {
                                if (b->data[j*b->width] == -1) nu--;
                        }
                        imsg("hist:  Block usedmatch size %d, used %d==%d \n",b->size,b->used,nu);
                 }
		imsg("hist: Hist Block[%d] width %d total %d n=%d\n",i,blocks[i].width,n,nn);
        }
}
int hist_stress(void)
{
        int i;
        char bf[BFSZ];
        char bf2[BFSZ];

        for (i=1; i<60000; i++) {
                lprintf(bf,BFSZ-1,"<mm%d>",i);
                lprintf(bf2,BFSZ-1,"comp.os.vms/%d misc.writing/%d",i,i);
                hist_add(bf,"25 Jan 95 09:54:22 GMT"
                        ,"25 Feb 95 09:54:22 GMT"
                        ,bf2);
        }
        return TRUE;
}
int hist_hash(char *s, int max, int *match)
{
        int total=0;
        int i;
	int m=0;
        for (i=0;(*s!=0) && (*s!='>');s++,i++) {
               	m = (m*8+(*s)*i) % max;    
                total = (total*64+(*s)*i) % 98765;    
        }
	*match = total % 1000000;
 	if (m<0) return -m;
        return m;
}

