/* Copyright (C) 1995 NetWin World, New Zealand, All rights reserved. */
#include <stdlib.h>
#include <stdio.h>
#include <ctype.h>
#include <string.h>
#include <assert.h>
#ifdef VMS
#include <unixio.h>
#endif
#ifndef NO_FCNTL
#include <fcntl.h>
#else
#include <file.h>
#endif
#include "emsg.h"
#include "lib.h"
#include "str.h"
#include "group.h"
#include "hist.h"

/* Still to write */
void hist_idx_clear(void);
int str_hash(char *s, int max);

#define DNEWS_HISTORY  file_hist("history.dat")
#define DNEWS_HISTORY_IDX  file_hist("history.idx")
#define DNEWS_HISTORY_NEW  file_hist("history.dat_new")
#define DNEWS_HISTORY_NEW_IDX file_hist("history.idx_new")
#ifndef WIN32
#define O_BINARY_MISSING TRUE
#endif

#ifdef O_BINARY_MISSING
#define O_BINARY 0
#endif

void hist_cache_flush(void);
void hist_cache_add(char *bf, char *mid, int fz);
char *hist_cache_find(char *mid);

/* Message-ID<tab>arrived~expire~posted<tab>group/num group/num ... */
/* If no expirey date, then set to '-' */
/* If article has expired, then set to messagid-expired<tab>'x' */ 
static int total_mem;
#define FLUSH_SIZE 20	/*Increase to 100 */

#define TESTIT xx
#ifdef TESTITxx
extern char debug_str[];
main()
{
	int num,grp,r,i;
	char bf[200],bf2[200];
	char *s,*rec;

	strcpy(debug_str,"nntp:,err:,debug:,info:,item:,group:,suck:,cmd:,hist:,fatal:,");

	emsg("info: History init\n");
	hist_init();
#define ADDRECS xx
#ifdef ADDRECS
	for (i=1; i<60000; i++) {
		sprintf(bf,"<mm%d>",i);
		sprintf(bf2,"comp.os.vms/%d misc.writing/%d",i,i);
		hist_add(bf,"25 Jan 95 09:54:22 GMT"
			,"25 Feb 95 09:54:22 GMT"
			,bf2);
	}
#endif
	for (i=1; i<60000; i++)  {
		sprintf(bf,"<mm%d>",i);
		if (!hist_exists(bf)) printf("Doesn't exist? {%s} \n",bf);
	}
}
char *chan_getipname() {return "";}
int diskuse_get() {return 0;}
int feed_wanted() {return 0;}
int match_wild(char *a, char *b) {return TRUE;}
char *nntp_hostname() {return "";}
void post_seterr() {}
int suck_update_groups() {return TRUE;}
#endif 

#ifdef TESTITxx
extern char debug_str[];
main()
{
	int num,grp,r,i;
	char bf[200],bf2[200];
	char *s,*rec;
/*test after with odd and even number of records*/
	strcpy(debug_str,"nntp:,err:,debug:,info:,item:,group:,suck:,cmd:,hist:,fatal:,");

	emsg("info: History init\n");
	hist_init();
#define ADDRECS XX
#ifdef ADDRECS
	for (i=1; i<100000; i++) {
		sprintf(bf,"<m%d>",i);
		sprintf(bf2,"comp.os.vms/%d misc.writing/%d",i,i);
		hist_add(bf,"25 Jan 95 09:54:22 GMT"
			,"25 Feb 95 09:54:22 GMT"
			,bf2);
	}
#endif
	
	imsg("info: Flush total_mem %d \n",total_mem);
	hist_flush();
		printf("I hope you deleted history.dat first !!?\n ");
		printf("Enter message id ? "); gets(bf);
		r = hist_find(bf,&grp,&num);
		printf("Found %d Group %d Number %d ",r,grp,num);
		printf("Also test extract functions and date stuff \n");
		rec = hist_exists(bf);
		if (rec==NULL) printf("Could not find record \n");
		if (rec!=NULL) {
		  printf("Dates date_expires %d \n ",hist_date_expires(rec));
		  printf("Dates date_arrived %d \n ",hist_date_arrived(rec));
			s = hist_groupfirst(rec);
			for (i=0;s!=NULL;i++) {
				 printf("Found Group {%s} \n",s);	
				printf("num %d \n",hist_num(rec,i));
				s = hist_groupnext();
			}
		}
	printf("Enter date for after search "); gets(bf);
	i  = atoi(bf);
	rec = hist_after(i);
	for (;rec!=NULL;) {
		printf("date %d  msg %s \n",hist_date_arrived(rec),rec);
		rec = hist_next();
	}
	/*
	int hist_after(int date);
	*/

}
#endif 

static int h_his,nbuffed;
static int file_size;
static int notatend;
static int upto;
char *hist_status(void)
{
	static char bf[200];
	sprintf(bf,"History: disk %dk memory %dk, Total str %dk Total malloc %dk\015\012",file_size/1024,total_mem/1024,str_total()/1024,mem_total()/1024);
	return bf;
}

int hist_upto(void)
{
	return upto;
}
int hist_add(char *mid, char *s_posted, char *s_expires, char *grps)
{
	char bf[11000];
	int expires,posted,arrived;

	arrived = lib_gmtime(); /* Convert to an integer :-) */
	expires = parsedate(s_expires);
	posted = parsedate(s_posted);
	if (strlen(mid)>200) {emsg("hist: Message id too long %d \n",strlen(mid)); return FALSE;}
	if (strlen(grps)>10000) {emsg("hist: Groups too long %d \n",strlen(grps)); return FALSE;}
	sprintf(bf,"%s\t%d~%d~%d\t",mid,arrived,expires,posted);
	ncpy(bf+strlen(bf),grps,10000); /* we won't allow more cross posting thatn that*/
	strcat(bf,"\n");
/* As bug in vaxcrtl prevents seeking and reading and writing we must avoid it */
	hist_cache_add(bf,mid,file_size);
/*
	hist_addidx(mid,file_size);
	hist_append(bf);
*/
	return TRUE;
}
#define CACHE_SIZE 200
static char *c_bf[CACHE_SIZE];
static char *c_mid[CACHE_SIZE];
static int c_fz[CACHE_SIZE];
static int nc;
void hist_cache_add(char *bf, char *mid, int fz)
{
	if (nc>=CACHE_SIZE) emsg("Panic hist cache add\n");
	c_bf[nc] = mystrdup(bf);
	c_mid[nc] = mystrdup(mid);
	if ((c_bf[nc]==NULL) || (c_mid[nc]==NULL)) {emsg("hist: memory allocation failure\n"); exit(1); }
	c_fz[nc] = fz;
 /*wmsg("hist: cache add {%s} offset %d  \n",mid,fz); */ 
	nc++;
	if (nc>=(CACHE_SIZE-2)) hist_cache_flush();
}
void hist_cache_flush(void)
{
	int i;
	if (nc==0) return;
	hist_close(); hist_open_write();
	for (i=0; i<nc; i++) {
		hist_addidx(c_mid[i],file_size); /* The passed one was wrong*/
		hist_append(c_bf[i]);
		free(c_bf[i]); free(c_mid[i]);
	}
	hist_close(); hist_open_read();
	nc = 0;
	hist_flush(); 
}

char *hist_cache_find(char *mid)
{
	static char tmp[1002];
	int i;
	for (i=0; i<nc; i++) {
		if (strcmp(mid,c_mid[i])==0) {
			ncpy(tmp,c_bf[i],1000);
			return tmp;
		}
	}
	return NULL;
}
void hist_flush(void)
{
	hist_cache_flush();
	hist_idx_flush();
	if (nbuffed==0) return; /* Nothing to flush */
	nbuffed = 0;
	hist_close();
	hist_open_read();
}
void hist_close(void)
{
	if (h_his!= -1) close(h_his);
	h_his = -1; 
}
void hist_append(char *s)
{
	int len = strlen(s);
	nbuffed++;
	if (len>10000) wmsg("hist: very long line in history %d \n",len);
	write(h_his,s,len);
	file_size += len;
}
int hist_open_read(void)
{
	return hist_open(O_RDONLY | O_BINARY);
}
int hist_open_write(void)
{
	return hist_open(O_WRONLY | O_APPEND | O_BINARY);
}
int hist_open(int mode)
{
	h_his =  open_log(DNEWS_HISTORY,mode,U_RW);
	if (h_his<0) {
		filemsg("hist: Open failed, trying create \n");
		h_his =  open_log(DNEWS_HISTORY,O_WRONLY | O_BINARY | O_CREAT,U_RW);
		if (h_his>=0) { 
			close(h_his); 
			h_his =  open_log(DNEWS_HISTORY,mode,U_RW);
			return TRUE;
		}
		filemsg("fatal: Could not open/create history file {%s} \n",DNEWS_HISTORY);
		emsg_flush();
		exit(1); /* May mean another dnews is running !!!! panic !!! */
	}
	file_size = file_length(h_his);
	return TRUE;
}

static int seeked;
static int **hidx; 
int hist_size=5;	/* 80000; */
int hist_init(void)
{
	int i;
	char *rec;
	int fz;
	int rlen;
	char *s;
	int nr=0;
	int nd=0;
	char mid[600]; 

	hist_open_read();

	fz = file_length(h_his);
	file_size = fz;
	if (hist_idx_read(fz)) return TRUE;

	emsg("hist: No index found, building one from scratch\n");
	/* with 40 mega bytes, we want about 80,000 in our hash table*/
	/* file_size += len; */

#define FAST_HISTORY TRUE
#ifdef FAST_HISTORY
	hist_size = fz/60;
#else
	hist_size = fz/500;
#endif
	if (hist_size<10000) hist_size = 10000;  /* should be 40000 */
	if (hist_size>350000) hist_size = 350000; /* Should be big enough */	
	hidx = (int **) calloc(hist_size,sizeof(int) );
	if (hidx==NULL) {emsg("fatal: Out of Memory in history"); return FALSE;}

	hist_idx_add(hist_size,0);

	/* If startup is too slow, we could save the index in a file */
	file_size = 0;
	rec = hist_line(0);
	for (;rec!=NULL;) {
		nr++;
		s = strchr(rec,'\t');
		if (s==NULL) {emsg("fatal: history file corrupt, no tab found \n"); return FALSE;}
		rlen = s-rec;
		if (rlen>500) rlen = 500;
		ncpy(mid,rec,rlen);

		seeked = FALSE;
		hist_addidx(mid,file_size);
		file_size += strlen(rec)+1; /* EOL character is not returned */
		rec = hist_next();
	}
	emsg("hist: Num records %d \n",nr);
	hist_idx_flush();
}

/* Rebuild the history file */
static int islogging;
void hist_rebuild(void)
{
	char *rec,*s;
	char mid[600]; 
	int rlen;
	int nd=0,nr=0;
	file_size = 0;

	hist_idx_clear();
	hidx = (int **) calloc(hist_size,sizeof(int) ); /* shouldbe int* */
	if (hidx==NULL) emsg("hist: Memory allocation failure hidx\n");
	rec = hist_line(0);
	for (;rec!=NULL;) {
		nr++;
		s = strchr(rec,'\t');
		if (s==NULL) {
			emsg("fatal: history file corrupt, no tab found \n");
			goto nextrec;
		}
		rlen = s-rec;
		if (rlen>600) rlen = 600;
		ncpy(mid,rec,rlen);


		seeked = FALSE;
		if (hist_exists(mid)!=NULL) {
			nd++;
			if (nd<10) emsg("fatal: Duplicate record found, ignoring (should delete) {%s} \n",mid);
		} else {
			hist_addidx(mid,file_size);
			hnew_write(rec);
		}
		if (seeked) {
			hist_line(file_size);
		}
nextrec:
		file_size += strlen(rec)+1; /* EOL character is not returned */
		rec = hist_next();
	}
	emsg("hist: Num records %d,  deleted %d \n",nr,nd);
	hnew_close();
	hist_close();
	hist_switch();
}
void hist_addidx(char *msgid, int offset) 
{
	int sh;
	sh = str_hash(msgid,hist_size);
 /*wmsg("hist: Hased to %d id {%s} offset %d  \n",sh,msgid,offset);  */
	hist_idx_add(sh,offset);
	hist_addone(&hidx[sh],offset);

}
int hist_addone(int **hptr, int offset)
{
	int i;
	int *h = *hptr;
	int len=0;
	int *tmp;

	if (h!=NULL) {
		for (len=0; h[len]!= -1; len++) ;
	} else len = 0;

	total_mem += (len+2)*sizeof(int);
	tmp = malloc((len+2)*sizeof(int));
	if (tmp==NULL) {emsg("fatal: Out of Memory in history used %d \n",total_mem); return FALSE; }
	memcpy(tmp,h,sizeof(int)*len);
	if (len!=0) free(h); 
	total_mem -= (len+1)*sizeof(int);
	tmp[len] = offset;
	tmp[len+1] = -1;
	*hptr = tmp;
	return TRUE;
}	
int hist_find(char *msgid, int *grp, int *num)
{
	int sh;
	int *h;
	char *s;

	if (strcmp(msgid,"<fake@netwin.com.nz>")==0) {
		*grp = 0;
		*num = group_getfake(0);
		return TRUE;
	}
	*grp = 0; *num = 0;
	sh = str_hash(msgid,hist_size);
/*	dmsg("hist: hash {%s} to %d %d \n",msgid,sh,hist_size);  */
	h = hidx[sh];
	if (h==NULL) return FALSE;
	for (; *h!= -1; h++) {
		s = hist_line(*h);
		/*dmsg("hist: hist_find %d %p \n",*h,s);
		if (s!=NULL) dmsg("hist: find %d {%s} \n",*h,s);*/
		if (s==NULL) {
			emsg("hist: Fatal error hist_find NULL ptr EOF \n");
			return FALSE;
		}
		if (strncmp(s,msgid,strlen(msgid))==0) {
			if (s[strlen(msgid)]=='\t') {
				*grp = hist_group(s,0);
				*num = hist_num(s,0);
				if (*grp<0) {*grp=0; return FALSE;}
				return TRUE;
			}
		}	
	}
	return FALSE;
}
int hist_gotit(char *msgid)
{
	static int lastret;
	static char lastmsgid[400];

	if (strcmp(msgid,lastmsgid)==0) return lastret;
	ncpy(lastmsgid,msgid,399);
	if (hist_exists(msgid)==NULL) 
		lastret = FALSE;
	else 
		lastret = TRUE;
	return lastret;
}
char *hist_exists(char *msgid)
{
	int sh;
	int *h;
	char *s;

	islogging = FALSE;

	
	sh = str_hash(msgid,hist_size);
	if (hidx==NULL) {emsg("hist: hidx is NULL, CRASHING");}
	h = hidx[sh];
	if (islogging) emsg("hist: exists?, [%d:%d] points=%p {%s} \n",sh,hist_size
		,h,msgid);
	if (h!=NULL) {
	  for (; *h!= -1; h++) {
		s = hist_line(*h);
		if (s==NULL) {
			emsg("hist: Fatal error hist_exists NULL ptr  {%s}\n",msgid);
			return NULL;
		}
		if (islogging) emsg("hist: exists, seek %d, got {%s} \n",*h,s);
		if (strncmp(s,msgid,strlen(msgid))==0) {
			if (s[strlen(msgid)]=='\t') {
				return s;
			}
		}	
	  }
	}
	/* Maybe it's in our recent cache on vms? */
	return hist_cache_find(msgid);
}


static char hline[12000];
static int blen;
#define RCHUNK 400 /* should be aboue 300, avg length of 1-2 lines */
char *hist_line(int h) /* Ugly, reads until EOL */
{
	int add=0;
	seeked = TRUE;
	if (lseek(h_his,h,SEEK_SET)<0) filemsg("hist: seek %d \n",h); 
	upto = h;
	notatend = TRUE;
/*
n = read(h_his,bf,RCHUNK);
printf("Seek read at=%d got=%d \n{%s} \n",h,n,bf);
*/
	blen = 0;
	hline[0] = 0; hline[1] = 0;
	return hist_next();
}
/* Read the next line, and return NULL on EOF */
/* Only valid after a call to hist_line */
char *hist_next(void)  /*Read the next line, and return NULL on EOF */
{
	int nread = -1;
	char *s;
	int len;

	len = strlen(hline); /* Search for dummy NULL */

	memmove(hline,hline+len+1,blen-len+1); /* Move the rest over */
	blen = blen - (len+1);
	if (blen<0) blen = 0;
	if (hline[0]==0) blen = 0;
again:
	s = strchr(hline,'\n');
	if (s!=NULL) {
		*s = 0;
		upto += strlen(hline);
		return hline;
	}
	if (nread==0) {
		return NULL;
	} /* End of file found */

	if (blen>8000) {
		emsg("fatal: Error in history file - long too line {%s}\n",hline); 
		blen = 0; hline[0] = 0;
		return NULL;
	}

	nread = read(h_his,hline+blen,RCHUNK);
 	if (nread<0) nread = 0;
ajax:	blen = blen + nread;
	hline[blen] = 0;
	if (2==3) goto ajax;
	goto again;
}
char *hist_line2(int h) /* Ugly, reads until two EOL's so it's got a whole record*/
{
	char *s;
	int add=0;
	int n;

	notatend = TRUE;
	if (lseek(h_his,h,SEEK_SET)<0) filemsg("hist: seek %d \n",h); 
	
again:
	if (add>8000) {emsg("Error in history file - long line \n"); return NULL;}
	n = read(h_his,hline+add,RCHUNK);
	if (n==0) return NULL;
	if (n<0) {emsg("fatal: Reading history file, io error"); return NULL;}
	hline[n+add] = 0;
	s = strchr(hline,'\n');
	if (s==NULL) {add += RCHUNK; goto again;}
	if (strchr(s+1,'\n')==NULL) {add += RCHUNK; goto again;}
	upto += strlen(hline);
	return hline;
}
char *hist_after(int date)
{
	/* Binary search for first message which 'arrived' after 'date' */ 
	int lower=0;
	int upper=file_length(h_his);
	int middle;
	char *s,*start;

	for (;TRUE;) {
		middle = (upper+lower)/2;
		s = hist_line2(middle);
		if (s==NULL) break;  /* Answer must be lower*/ 
		start = strchr(s,'<');
		if (start==NULL) emsg("fatal: impossible history error \n"); 
		middle = middle + start - s;  /*Correct so is start of line*/
		if (middle>=upper) break;
		if (hist_date_arrived(start)>date) upper = middle;
		else if (lower==middle) break;
		else lower = middle;
	}	
	s = hist_line(lower);
	for (;s!=NULL;) {
		if (hist_date_arrived(s)>date) return s;
		s = hist_next();
	}
	return NULL;
}
int hist_date_expires(char *rec)  /*Second date, zero if is '-' */
{
	char *s;
	s = strchr(rec,'\t');
	if (s==NULL) {emsg("fatal: history file corrupt1 {%s}\n",rec); return 0;}	
	s = strchr(s+1,'~');
	if (s==NULL) {emsg("fatal: history file corrupt3 {%s}\n",rec); return 0;}	
	return atoi(s+1);
}
int hist_date_arrived(char *rec)
{
	char *s;
	s = strchr(rec,'\t');
	if (s==NULL) {emsg("fatal: history file corrupt4 {%s}\n",rec); return 0;}	
	return atoi(s+1);
}
static char *gpoint;
char *hist_groupfirst(char *rec)
{
	char *s;

	s = strchr(rec,'\t');
	if (s==NULL) {emsg("fatal: history file corrupt5 {%s}\n",rec); return NULL;}	
	s = strchr(s+1,'\t');
	if (s==NULL) {emsg("fatal: history file corrupt6 {%s}\n",rec); return NULL;}	
	gpoint = s+1;
	return hist_groupnext();
}
static int nextnum;
int hist_groupnextnum(void)
{
	return nextnum;
}
char *hist_groupnext(void)
{
	static char gname[200];
	char *s2;
	int len;

	if (gpoint==NULL) {emsg("Hist_groupnext failed, gpoint null\n"); return NULL;}
	s2 = strchr(gpoint,'/');
	if (s2==NULL) return NULL;
	nextnum = atoi(s2+1);
	len = s2 - gpoint;
	if (len>100) {emsg("Hist groupnext: len too long %d \n",len); return NULL;}
	ncpy(gname,gpoint,len);
	gpoint = strchr(s2,' ');
	if (gpoint==NULL) gpoint = s2 + strlen(s2);
	else gpoint++;

	return gname;
}
char *hist_gname(char *rec, int g) /* Return group and num */
{
	static char gname[200];
	char *s,*s2;
	int i,len;

	s = strchr(rec,'\t');
	if (s==NULL) {emsg("fatal: history file corrupt7 {%s}\n",rec); return NULL;}	
	s = strchr(s+1,'\t');
	if (s==NULL) {emsg("fatal: history file corrupt8 {%s}\n",rec); return NULL;;}	
	s++;
	for (i=0; i<g; i++) { /* Skip over 'g' items */	
		s = strchr(s,' ');
		if (s==NULL) return NULL;
		s++;
	}
	s2 = strchr(s,'/');
	if (s2==NULL) return NULL;
	len = s2 - s;
	if (len>150) {emsg("fatal: history file corrupt, len %d \n",len); return NULL;}
	ncpy(gname,s,len);
	return gname;
}

int hist_group(char *s,int n)
{
	int x;
	char *grp = hist_gname(s,n);
	if (grp==NULL) return -1;
	x = group_find(grp);
	return x;
}
int hist_num(char *rec,int g)
{
	char *s,*s2;
	int i;

	s = strchr(rec,'\t');
	if (s==NULL) {emsg("fatal: history file corrupt9 {%s}\n",rec); return 0;}	
	s = strchr(s+1,'\t');
	if (s==NULL) {emsg("fatal: history file corrupt a {%s}\n",rec); return 0;;}	
	s++;

	for (i=0; i<g; i++) { /* Skip over 'g' items */	
		s = strchr(s,' ');
		if (s==NULL) return 0;
		s++;
	}
	s2 = strchr(s,'/');
	if (s2==NULL) return 0;
	return atoi(s2+1);
}


static int hnew_open = FALSE;
static int n_out,new_size,new_hsize;
void hnew_write(char *s)
{
	int len,h;
	char rec[10000];
	int show=FALSE;
	ncpy(rec,s,9996);
	strcat(rec,"\n");

/*	if (strstr(rec,"3ti8hn")!=NULL) show=TRUE;*/

	if (show) dmsg("hist: addrec %s",rec);
	if (!hnew_open) {
		new_size = 0;
		hnew_open = TRUE;
		remove(DNEWS_HISTORY_NEW_IDX);
		n_out =  open_log(DNEWS_HISTORY_NEW,O_TRUNC | O_WRONLY | O_CREAT | O_BINARY,U_RW);
		if (n_out<0) {emsg("fatal: Could not open/create history file {%s} \n",DNEWS_HISTORY_NEW); return;}
		
		new_hsize = file_size/500;
		if (new_hsize<10000) new_hsize = 10000;  /* should be 40000 */
		if (new_hsize>200000) new_hsize = 200000; /* Should be big enough */	

		hnew_idx_add(new_hsize,0);	
		dmsg("hist: hsize %d",new_hsize);
	}
	s = strchr(rec,'\t');  if (s==NULL) return;
	if (s==rec) return;
	if (*(s-1) != '>') {
		emsg("hist: Message-id contained rubbish, deleting it {%s} \n"
			,lib_safe(s));
		return;
	}

	*s = 0; h = str_hash(rec,new_hsize);
	if (show) dmsg("hist: str_hash {%s} %d %d \n",rec,h,new_hsize);

	 *s = '\t';
	if (show) dmsg("hist: hashed it to %d / %d",h,new_hsize);
	if (h>=new_hsize) emsg("hist: fatal: writing new hash error %d %d %d \n",h,new_hsize,new_size);
	hnew_idx_add(h,new_size);	
	
	len = strlen(rec);
	write(n_out,rec,len);
	new_size += len;
}

void hnew_close(void)
{
	hnew_idx_flush();
	if (n_out!= -1) close(n_out);
	n_out = -1;
	hnew_open = FALSE;
}
void hist_switch(void)
{
	char bf[500];
	int r;
	hnew_close();
	hist_close();
	
	r = remove(DNEWS_HISTORY);  if (r!=0) emsg("hist: 1unable to delete %s \n",DNEWS_HISTORY); 
	r = remove(DNEWS_HISTORY_IDX); 
	if (r!=0) {
		emsg("hist: 2unable to delete %s \n",DNEWS_HISTORY_IDX); 
		filemsg("hist.idx");
	}
	ncpy(bf,DNEWS_HISTORY_NEW,400);
	r = rename(bf,DNEWS_HISTORY);  if (r!=0) emsg("hist: 3unable to delete %s \n",DNEWS_HISTORY_NEW); 
	ncpy(bf,DNEWS_HISTORY_NEW_IDX,400);
	r = rename(bf,DNEWS_HISTORY_IDX);  if (r!=0) emsg("hist: unable 4to delete %s \n",DNEWS_HISTORY_NEW_IDX); 

	hist_open_read();
	hist_idx_read(file_length(h_his));
}
static int ibf[2001];
static int nibf;
void hist_idx_add(int a, int b)
{
	ibf[nibf++] = a;	
	ibf[nibf++] = b;	
	if (nibf>200) hist_idx_flush();
}
void hist_idx_flush(void)
{
	int h_idx,n;
	if (nibf==0) return;

	h_idx = open_log(DNEWS_HISTORY_IDX,O_WRONLY | O_APPEND | O_CREAT | O_BINARY,U_RW);
	if (h_idx<0) {
		emsg("fatal: can't write history index file {%s} \n",DNEWS_HISTORY_IDX); 
		filemsg("History index flush:");
		return; 
	}

	n = write(h_idx,ibf,nibf*sizeof(int));

	if (n<(nibf*sizeof(int))) emsg("fatal: Error writing hist index \n");
	nibf = 0;
	if (h_idx!= -1) close(h_idx);
	h_idx = -1;
}

static int inew[2001];
static int ninew;
void hnew_idx_add(int a, int b)
{
	inew[ninew++] = a;	
	inew[ninew++] = b;	
	if (ninew>200) hnew_idx_flush();
}
void hnew_idx_flush(void)
{
	int h_idx,n;
	if (ninew==0) return;

	h_idx = open_log(DNEWS_HISTORY_NEW_IDX,O_WRONLY | O_APPEND | O_CREAT | O_BINARY,U_RW);
	if (h_idx<0) {fileemsg("fatal: can't write new history index file (flush)\n"); return ;}

	n = write(h_idx,inew,ninew*sizeof(int));
	if (n<ninew*sizeof(int)) emsg("fatal: Error writing hist index \n");
	ninew = 0;
	close(h_idx);	
}
void hist_idx_clear(void)
{
	int i;
	if (hidx!=NULL) {
		for (i=0; i<hist_size; i++) {
			if (hidx[i]!=NULL) free(hidx[i]);
		}
		free(hidx);
		hidx = NULL;
	}
	total_mem = 0;
}

int hist_idx_read(int sz)
{
	int idx,i,n,got,fz;

	hist_idx_clear();	
	idx = open_log(DNEWS_HISTORY_IDX,O_RDONLY | O_BINARY,U_RW);
	if (idx<0) return FALSE;
	
	fz = file_length(idx);
	emsg("hist: idx size %d \n",fz);

	n = read(idx,ibf,sizeof(int)*2);
	if (n<2) {close(idx); return FALSE;}
	hist_size = ibf[0];

	hidx = (int **) calloc(hist_size,sizeof(int) );
	if (hidx==NULL) {emsg("fatal: Out of Memory in history"); close(idx); return FALSE;}

	emsg("hist: hist_size %d \n",hist_size);
	n = read(idx,ibf,sizeof(int)*2000);
	for (;n>0;) {
		n = n/sizeof(int);
		for (i=0; i<n; i+=2) {
			if (ibf[i]<0 || ibf[i]>=hist_size ) {
				emsg("fatal: History index corrupt [%d] #=%d f=%d \n",i,ibf[i],ibf[i+1]);
			} else {
				hist_addone(&hidx[ibf[i]],ibf[i+1]);
			}
		}
		n = read(idx,ibf,sizeof(int)*2000);
	}
	close(idx);

	return TRUE;
}
