/*
todo:
  Add optional date to database via ini setting.
  psize
  xtime, devide by 86400 to get days into a short..
  Use FILE_POINTX if search_date TRUE
  make group_max adjustable
  New fts_search that obeys 'and', dates, group wild card, sort order
	(sort by date or group or score)



  07 08:35:16: point write corrupt data 245 4748 groupmax 239

    Todo List:
	dnewsweb output,  thread by group.
	dnewsweb, allow search group button to use fts database.
	Could not find restart point, seems to hang.
	Check refeed code does not have a loop, add an output call?
	Deal with nused > 32000

	Word file gets corrupt if wrong number of bytes is written to it.
	But it's ok, it will uncorrupt itself.


22 02:20:56: Fatal error, hasdel g=-9976 num=4561 = -5415 
22 02:20:56: Fatal error, hasdel g=-10039 num=358 = -9681 


*/

#include <stdlib.h>
#ifdef WIN32
#include <io.h>
#endif
#ifndef NO_FCNTL
#include <fcntl.h>
#endif
#include <time.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <stdarg.h>
#include <errno.h>
#include "lprintf.h"

#define FTSMAIN 1
#include "ftslib.h"
#include "myfile.h"

void r_free(void *xx);
void *r_malloc(int sz);

char *xstrdup(char *s1);
void xstrfree(char *s1);
char *mystrstr(const char *s1, const char *s2);
char *mystrchr(const char *s,const int c);
#define strstr mystrstr
#define strchr mystrchr


#ifdef WIN32
extern int _fmode;
#endif
char *ncpy(char *dst, char *src, int len);
int lib_flength(char *fname);

#define s_feof feof
#define s_fopen fopen
#define s_fclose fclose
#define s_fseek fseek 
#define s_ftell ftell
#define s_fread fread
#define s_fwrite fwrite

#define FREE_CELL 65534 /* -2 */
#define END_CELL  65535  /* -1 */

void *check_malloc(int size, char *file, int line);
void check_block(int *x, char *name);
void check_free(void *s,char *file,int line);
int fts_doswitch(void);
int fts_del_count(void);
int parsedate(char *rec);
/*
#define mymalloc(p_Size) check_malloc(p_Size,__FILE__,__LINE__)
#define myfree(p_Size) check_free(p_Size,__FILE__,__LINE__)
#define mycalloc(pnum,p_Size) check_calloc(pnum,p_Size,__FILE__,__LINE__)
*/
#define mymalloc(p_Size) r_malloc(p_Size)
#define myfree(p_Size) r_free(p_Size)

int fts_rename_group(void);
char *mystrdup(const char *str);
int dmsg(char * arg_list, ...);
int emsg(char * arg_list, ...);
int imsg(const char * arg_list, ...);
int filemsg(char * arg_list, ...);
char * fts_group_name(int gid);
int fts_del_old(void);
void fts_read_free(VLIST *v);
char *safe_ctime(const time_t *clock, char *bf);
char *safe_asctime(const struct tm *t, char *bf);
struct tm *safe_localtime(const time_t *clock,struct tm *result);
struct tm *safe_gmtime(const time_t *clock,struct tm *result);
#define SPECIAL "microsoft" 
static char root[BFSZ];
#define FILE_WORDS  fts_dir("words.fts")
#define FILE_WORDSOLD  fts_dir("words_o.fts")
#define FILE_WORDSNEW  fts_dir("words_n.fts")
#define FILE_GROUPS  fts_dir("groups.fts")
#define FILE_GROUPSNEW  fts_dir("groups_n.fts")
#define FILE_GROUPSOLD  fts_dir("groups_o.fts")
#define FILE_POINT  fts_dir("point.fts")
#define FILE_POINTX  fts_dir("pointx.fts")
#define FILE_POINTNEW  fts_dir("point_n.fts")
#define FILE_POINTOLD  fts_dir("point_o.fts")
#define FILE_EXPIRE  fts_dir("expire.del")
#define FILE_EXPIRE_DO fts_dir("expire.do")
#define FILE_FLUSH fts_dir("flush.do")

#define FILE_INI  fts_dir("fts.conf")
#define FILE_SWITCH  fts_dir("switch.fts")

#define FILE_HASH1 fts_dir("hash1.fts")
#define FILE_HASH2 fts_dir("hash2.fts")

static int max_groups=60000;
static FILE *fpnew,*fp,*fw;
static int atend;
static int word_upto,point_upto;

static MWORD *words[HASH_WORDS];
static GLIST *groups[HASH_WORDS];
static int group_next,word_next;
static int stat_nwords, stat_nunique;
static int ncached=0;
static int total_items;
static int wtotal;

static int *group_base;
static int *group_base2;
static int *group_newbase;
static char **group_name;

#ifdef WIN32
void sleep(int x);
#endif
void show_this(char *s,MWORD *d);
static int groups_changed;
void fts_append_total(void);
FILE *b_fopen(const char *fname,const char *mode);

/*
#define mymalloc(p_Size) malloc(p_Size)
#define myfree(p_Size) free(p_Size);
#define mycalloc(pnum,p_Size) calloc(pnum,p_Size) 
#define mystrdup(p_Size) strdup(p_Size);
*/

/*
#define lib_gmtime() my_lib_gmtime(__FILE__,__LINE__)
int my_lib_gmtime(char *file, int line);
*/

static int psize;
static char file_point[BFSZ];
static int istime;
static int isread;
void fts_doinit(void);
void fts_init(char *rt, int xisread) 
{
        ncpy(root,rt,BFSZ-1);
        isread = xisread;
        fts_doinit();
} 
extern int memdebug_total;
static int totwords;

static int ftsmem;
MWORD *fts_malloc(void)
{		       
	static MWORD *blocks;
	static int nblocks;
	static int cur;
	
	if (cur>=nblocks) { /* Allocate another 50,000 words */
		blocks = mymalloc(50000 * sizeof(MWORD));
		ftsmem += 50000 * sizeof(MWORD);
		nblocks = 50000;
		cur = 0;
	}
	return &blocks[cur++];	  	
}
int fts_mem(void)
{
	return ftsmem;
}
void fts_doinit(void)
{
        imsg("Initializing FTS system {%s} %d \n",root,isread);
        fts_read_ini();
	
	max_groups = atoi(fts_ini(I_MAX_GROUPS));
	if (max_groups==0) max_groups = 60000;
	if (group_base==NULL) {
		group_base = (int *) mymalloc(sizeof(int)*max_groups);
		group_base2 = (int *) mymalloc(sizeof(int)*max_groups);
		group_newbase = (int *) mymalloc(sizeof(int)*max_groups);
		group_name = (char **) mymalloc(sizeof(char *)*max_groups);
		ftsmem += sizeof(int) * max_groups * 4;

	}

	istime = fts_true(I_INDEX_TIME);
	psize = PSIZE;
	strcpy(file_point,FILE_POINT);
	if (istime) {
		psize = PSIZEX; 
		strcpy(file_point,FILE_POINTX);
	}
	word_upto = 0;
        point_upto = lib_flength(FILE_POINT);
        dmsg("Point upto initialized to %d \n",point_upto);
        /* Read dictionary */
        if (!isread) fts_read_words();
        word_upto = lib_flength(FILE_WORDS);
        dmsg("Word upto initialized to %d \n",word_upto);

        /* Read group index */
        fts_read_groups();
        imsg("FTS system ready.\n");


}

void fts_flush_files(void);
#define EXPIRE_HOUR 11
void fts_background(void)
{
        time_t t;
	FILE *f;
	struct tm atm;
        struct tm *tt;
        static int lasthour;
	static int lasttick;
	static int lastwrite;

	if (lasttick+5>(time(NULL))) return;
	lasttick = time(NULL);

        if (isread) {
                /* Do any tasks required by reader in the background */ 
                fts_doswitch();
                return;
        }

	f = fopen(FILE_FLUSH,"r");
	if (f!=NULL) {
		fclose(f);
		if (remove(FILE_FLUSH)!=0) emsg("Could not remove (%s) %s\n",FILE_FLUSH,strerror(errno));
		if ((time(NULL) - lastwrite)>120) {
			imsg("Found %s so flushing index\n",FILE_FLUSH);
			fts_index_write();
			lastwrite = time(NULL);
		        f = fopen(FILE_SWITCH,"w");
			if (f!=NULL) {
			        fprintf(f,"Time to switch to new versions of fts files due to expire\n");
			        fclose(f); f = NULL;
			}
		} else imsg("Not flushing as did it %d seconds ago\n",(time(NULL) - lastwrite));
	}


        time(&t);
        tt = safe_localtime(&t,&atm);
/*        if ((tt->tm_hour == EXPIRE_HOUR) && (lasthour!=EXPIRE_HOUR)) {*/
		f = fopen(FILE_EXPIRE,"r");
		if (f==NULL) f = fopen(FILE_EXPIRE_DO,"r");
 		if (f!=NULL) {
			fclose(f); f = NULL;
	                emsg("Starting FTS expire job\n");
	                fts_expire();
	                emsg("Expire completed\n");             
			fts_index_write();
			emsg("Written new index file\n");

		}
/*        }*/
        lasthour = tt->tm_hour;
        if (groups_changed) {
                fts_write_group();
        }
        if (ncached>0) fts_cache_write();
        fts_append_total();

        fts_flush_files();
}
void fts_flush_files(void)
{
        if (fw!=NULL) s_fclose(fw); fw = NULL;
        if (fp!=NULL) s_fclose(fp); fp = NULL;
        fw = NULL;
        fp = NULL;
}
void fts_free_words(void);
void fts_free_groups(void);
int lib_flength(char *s)
{
        int r;            
        FILE *f;
        f = s_fopen(s,"rb");
        if (f==NULL) return 0;
        s_fseek(f,0,SEEK_END);
        r = s_ftell(f);
        s_fclose(f);
        return r;
}
void fts_close(void)
{
        if (ncached>0) fts_cache_write();
        if (fp!=NULL) s_fclose(fp);  fp = NULL;
        if (fw!=NULL) s_fclose(fw);  fw = NULL;

        fts_free_words();
        fts_free_groups();
}
void fts_free_words(void)
{
        int i;
	MWORD *w,*wnext;
        for (i=0; i<HASH_WORDS; i++) {
		for (w = words[i]; w!=NULL; w=wnext) {
			wnext = w->next;
		      	myfree(w);
			ftsmem -= (sizeof(*w));
		}
                words[i] = NULL;
        }
}
void fts_free_groups(void)
{
        int i;
	GLIST *w,*wnext;
        for (i=0; i<HASH_WORDS; i++) {
		for (w = groups[i]; w!=NULL; w=wnext) {
		 	if (w->word!=NULL) xstrfree(w->word);
			wnext = w->next;
			ftsmem -= (sizeof(*w));
		      	myfree(w);
		}
                groups[i] = NULL;
        }
}
MWORD *fts_word_add(char *word)
{
	MWORD *next;

        next = fts_malloc(); memset(next,0,sizeof(*next));
        strncpy(next->word,word,MAX_WORD_LEN);
	next->start = -1;
	next->nfree = -1;
	next->nused = -1;
	if (strcmp(word,SPECIAL)==0) imsg("fts_word_add(%s)\n",SPECIAL);
	fts_word_append(fw, next);
	return fts_word_add_hash(next);
}
MWORD *fts_word_add_hash(MWORD *w)
{
        int h = fts_hash(w->word,HASH_WORDS);

	totwords++;
      	w->next = words[h];
	words[h] = w;
	return w;
}
MWORD *fts_word_find(char *word)
{
        int h = fts_hash(word,HASH_WORDS);
        MWORD *d;

        d = words[h];
        for (;d!=NULL; d = d->next) {
                if (strncmp(word,d->word,MAX_WORD_LEN)==0) return d;
        }
        return NULL;
}
MWORD *fts_word_find_add(char *word)
{
        MWORD *d = fts_word_find(word);
        if (d!=NULL) return d;
        return fts_word_add(word);
}
VLIST *fts_read(MWORD *dict)
{
        VLIST *v,*vv;
        int nrefs;
        char bf[70];
	int start = dict->start;
	unsigned short xgid;
	int xnum;
	char xscore;
	int nbits=0;
	int nref = 0;
	short xtime;
	int r;
        vv = v = NULL;
        atend = FALSE;
	nrefs = 0;
	if (start<0) return NULL;
again:			
	if (s_fseek(fp,start,SEEK_SET)!=0) {filemsg("Seek failed in fts_read for {%d} \n",start); return NULL;}
        for (;!feof(fp);) {
	 	r = s_fread(bf,1,psize,fp);
	        if (r!=psize) {filemsg("Read failed in fts_read %d!=%d {%d} %d\n",r,psize,start,nrefs); return NULL;}
	        memcpy(&xgid,bf,2);
	        memcpy(&xnum,bf+2,4);
	        memcpy(&xscore,bf+6,1);
		if (istime) memcpy(&xtime,bf+7,2);
		nrefs++;
		if (nrefs>wtotal) {
		 	imsg("Stopping fts_read as %d words are more than enough\n",wtotal);
			break;
		}
		/*imsg("Read ref %d:%d scored %d\n",xgid,xnum,xscore);*/
		/* if end of list, or jump, but jump is empty */
	
		/* imsg("%d:%d\n",xgid,xnum);*/
		if ((xgid==FREE_CELL) || ((xgid==END_CELL) && (xnum== -1)) ) {
			break;
		} if (xgid==END_CELL)  {   
			nbits++;
		    	start = xnum;
			goto again;
		} else {
               		if (v==NULL) {
				vv = v = mymalloc(sizeof(VLIST));
				ftsmem += sizeof(VLIST);
			} else {
             		 	v->next = mymalloc(sizeof(VLIST)); v = v->next;
				ftsmem += sizeof(VLIST);
                	}      	
                	v->gid = xgid;
                	v->num = xnum;
                	v->score = xscore;
			v->xtime = xtime;
                	v->next = NULL;
			nref++;
		}
	}
/*	imsg("[%d] %s start=%d upto=%d nfree=%d nused=%d bits=%d %d\n",dict->wstart,dict->word,dict->start,dict->upto,dict->nfree,dict->nused,nbits,nref);*/
	return vv;
}
int fts_group_find_add(char *s, int num)
{
        int r;
        r = fts_group_find(s);
        if (r>=0) return r;
        fts_group_add(s,group_next, num);
        fts_append_group(s,group_next,num);

        group_next++;
        return group_next-1;

}
void fts_stats(int *nwords, int *nuni)
{
        *nwords = stat_nwords;
        *nuni = stat_nunique;
        stat_nwords = stat_nunique = 0;
}

char * fts_group_name(int gid)
{
        /* imsg("Name for %d is %p \n",gid,group_name[gid]);*/
        if (gid<0 || gid>=max_groups) return "fts_dud_gid";
        return group_name[gid];
}
int fts_group_add(char *word, int idx, int base)
{
        int h = fts_hash(word,HASH_WORDS);
        GLIST *d,*last,*next;

        next = mymalloc(sizeof(*d));
	ftsmem += (sizeof(*d));
        next->word = xstrdup(word);
        next->idx = idx;
        next->next = NULL;

        if ((idx<max_groups) && (idx>=0)) {
                group_name[idx] = next->word;
        }

        last = NULL;
        d = groups[h];
        for (;d!=NULL; d = d->next) last = d;
        if (last==NULL) groups[h] = next;
        else last->next = next;
        return TRUE;
}

int fts_group_find(char *word)
{
        int h = fts_hash(word,HASH_WORDS);
        GLIST *d;

        d = groups[h];
        for (;d!=NULL; d = d->next) {
                if (strcmp(word,d->word)==0) return d->idx;
        }
        return -1;
}
int fts_rename_both(void);
void fts_expire_point(void)
{
        /* For each word in index */
        /* Read word into memory */
        /* Write out to new word and new point file */
        MWORD *dict;
	int i;
        int nwords;
        VLIST *v,*old,*lastv,*vnext;
        int nok;
        int max = (total_items * atoi(fts_ini(I_WORD_MAX))) / 100;

        
        if (max<100) max = 100;
        fts_del_old(); /* Delete old coppies of indexes, dnews should have closed by now*/

        for (i=0; i<group_next; i++) {
                group_newbase[i] = FALSE;
        }
        if (!fts_open_new()) return;
        for (i=0; i<HASH_WORDS; i++) {
                dict = words[i];
                for (;dict!=NULL;dict=dict->next) {
                        old = fts_read(dict); /* Read in this word */
			if (strcmp(dict->word,SPECIAL)==0) imsg("Expire read in (%s)\n",dict->word);
                        for (nwords=0,v=old; v!=NULL; v=v->next) nwords++;

			if (nwords>total_items) total_items  = nwords;
                        if (nwords>max) imsg("Forgetting word {%s} used %d/%d times %d%% max=%d\n",dict->word,nwords,total_items,(nwords*100)/total_items,max); 
                        if (nwords>max) {
        			dict->start = -1;
				dict->upto = -1;
				if (strcmp(dict->word,SPECIAL)==0) imsg("Expire killit (%s)\n",dict->word);
				goto killit;
			} 
                        nok = 0;
			lastv = NULL;
                        for (v=old; v!=NULL; v=vnext) {
				vnext = v->next;
                                if (fts_is_del(v->gid,v->num)) {
					if (strcmp(dict->word,SPECIAL)==0) imsg("Deleting (%s) nok=%d %d:%d\n",dict->word,nok,v->gid,v->num);
					if (lastv==NULL) old = v->next;
					else lastv->next = v->next;	
					ftsmem -= (sizeof(*v));
					myfree(v);
                                } else {
					lastv = v;
	  				nok++;
          			}
                        }
     			
     
			dict->start = -1;
			dict->upto = -1;
			if (strcmp(dict->word,SPECIAL)==0) imsg("Expire done (%s) nok=%d\n",dict->word,nok);
			if (nok>0) fts_cache_write_one(fpnew,NULL,dict,old);
killit:;
                        for (v=old; v!=NULL; v=vnext) {
				vnext = v->next;
				ftsmem -= (sizeof(*v));
                                myfree(v);
			}

                }
        }
        s_fclose(fpnew); fpnew = NULL;
        imsg("Write words\n");
        if (!fts_write_words()) return ;
        imsg("write groups \n");
        if (!fts_write_group()) return;
        imsg("rename both\n");

	fts_flush_files();
        if (!fts_rename_both()) {
         	emsg("Rename failed, you must rename the file by hand then restart dfts\n");
		abort();
        }
}
#define ftsrename(c,n,o) strcpy(cur,c); strcpy(xnew,n); strcpy(old,o);  
        
int myrename(char *old, char *new);
int fts_rename_both(void)
{
        char old[BFSZ],xnew[BFSZ],cur[BFSZ];
        FILE *f;

        if (fpnew!=NULL) s_fclose(fpnew);  fpnew = NULL;
        if (fp!=NULL) s_fclose(fp);  fp = NULL;
        if (fw!=NULL) s_fclose(fw);  fw = NULL;


        ftsrename(FILE_WORDS,FILE_WORDSNEW,FILE_WORDSOLD);
                if (remove(old)!=0) if (errno!=ENOENT) imsg("Warning only, Unable to remove {%s} %s\n",old,strerror(errno)); 
                if (myrename(cur,old)!=0) if (errno!=ENOENT) {filemsg("Unable to rename {%s} {%s} \n",cur,old); return FALSE;} 
                if (myrename(xnew,cur)!=0) {filemsg("Fatal: Unable to rename {%s} {%s} \n",xnew,cur); return FALSE;} 
        ftsrename(FILE_GROUPS,FILE_GROUPSNEW,FILE_GROUPSOLD);
                if (remove(old)!=0) if (errno!=ENOENT) imsg("Warning only, Unable to remove {%s} %s\n",old,strerror(errno)); 
                if (myrename(cur,old)!=0) if (errno!=ENOENT) {filemsg("Unable to rename {%s} {%s} \n",cur,old); return FALSE;} 
                if (myrename(xnew,cur)!=0) {filemsg("Fatal: Unable to rename {%s} {%s} \n",xnew,cur); return FALSE;} 
        ftsrename(file_point,FILE_POINTNEW,FILE_POINTOLD);
                if (remove(old)!=0) if (errno!=ENOENT) imsg("WarningB only, Unable to remove {%s} %s\n",old,strerror(errno)); 
                if (myrename(cur,old)!=0) if (errno!=ENOENT) {filemsg("Unable to rename {%s} {%s} \n",cur,old); return FALSE;} 
                if (myrename(xnew,cur)!=0) {filemsg("Fatal: Unable to rename {%s} {%s} \n",xnew,cur); return FALSE;}

        f = fopen(FILE_SWITCH,"w");
	if (f!=NULL) {
	        fprintf(f,"Time to switch to new versions of fts files due to expire\n");
	        fclose(f); f = NULL;
	}
	emsg("Switch scheduled\n");
        return TRUE;
}
int myrename(char *old, char *xnew)
{
	int i;
	int r;
 	for (i=0; i<100; i++) {
		if (i>0) emsg("Attempting rename again %s -> %s \n",old,xnew);
		r = rename(old,xnew);
		if (r==0) return r;
		if (errno==ENOENT) return r;
		filemsg("Rename failed\n");
		sleep(1);
	}
	return r;
}
int fts_reload(void)
{
        imsg("Switching to latest fts database\n");
        fts_close();
        fts_doinit();
        if (remove(FILE_SWITCH)!=0) imsg("Warning only, Unable to remove {%s} \n",FILE_SWITCH); 
        return TRUE;

}
int fts_doswitch(void)
{
        FILE *f;

        f = fopen(FILE_SWITCH,"r");
        if (f==NULL) return FALSE;
        fclose(f);
        imsg("Time to switch to the new .fts files\n");
        fts_close();
        fts_doinit();
        if (remove(FILE_SWITCH)!=0) imsg("Warning only, Unable to remove {%s} \n",FILE_SWITCH); 
        return TRUE;
}
int fts_del_old(void)
{
        if (remove(FILE_SWITCH)!=0) imsg("Warning only, Unable to remove {%s} \n",FILE_SWITCH); 
        if (remove(FILE_WORDSOLD)!=0) imsg("Warning only, Unable to remove {%s} \n",FILE_WORDSOLD); 
        if (remove(FILE_GROUPSOLD)!=0) imsg("Warning only, Unable to remove {%s} \n",FILE_GROUPSOLD); 
        if (remove(FILE_POINTOLD)!=0) imsg("WarningA only, Unable to remove {%s} \n",FILE_POINTOLD); 
        return TRUE;
}
int fts_rename_group(void)
{
        char old[BFSZ],new[BFSZ];
        strcpy(old,FILE_GROUPSNEW);
        strcpy(new,FILE_GROUPS);
        if (remove(new)!=0) filemsg("Unable to remove {%s} \n",new);
        if (rename(old,new)!=0) filemsg("Unable to rename {%s} {%s} \n",old,new);
        return TRUE;
}
void show_this(char *s,MWORD *d)
{
   	return;		   /*
        if (strcmp(d->word,"this")==0) {
                imsg("%s This data %p, %d %d \n",s,d,d->wrec.point[0],d->wrec.point[1]);
        }
	*/
}
int fts_write_words(void)
{
        MWORD *dict;
        FILE *fwnew;
        int i;

        word_upto = 0;  
        fwnew = s_fopen(FILE_WORDSNEW,"wb");
        if (fwnew==NULL) {filemsg("Could not open %s\n",FILE_WORDSNEW); return FALSE;}
        
        for (i=0; i<HASH_WORDS; i++) {
                dict = words[i];
                for (;dict!=NULL; dict=dict->next) {
                        dict->wstart = word_upto;
                        show_this("append",dict);
                        if (!fts_word_append(fwnew, dict)) return FALSE;
                }
        }
                
        s_fclose(fwnew); fwnew = NULL;
        return TRUE;
}
int fts_open_new(void)
{
        point_upto = 0;
        atend = FALSE;
        fpnew = s_fopen(FILE_POINTNEW,"wb");
        if (fpnew==NULL) {filemsg("Could not open %s\n",FILE_POINTNEW); return FALSE;}
        return TRUE;
        
}
char *hexbyte(int c);
char *hexbyte(int c)
{
        static char bf[20];
        lprintf(bf,20-1,"%02.2x",(unsigned int) c);
        if (strlen(bf)>2) return bf + strlen(bf) - 2; 
        return bf;
}
int fts_read_words(void)
{
	MWORD *mw;
	WWORD ww;
        int upto;
        FILE *f;
        int n,curlen,nwords=0;
        curlen = 0;

        imsg("Reading words\n");
        f = s_fopen(FILE_WORDS,"rb");
        if (f==NULL) return FALSE;
	upto = 0;
        for (;!s_feof(f);) {
        	n = s_fread(&ww,1,sizeof(WWORD),f);
		if (n<=0) break;
	      	if (n!=sizeof(WWORD)) {filemsg("Couldn't read word in word file\n"); s_fclose(f); return FALSE; }
		mw = fts_malloc();
		mw->start = ww.start;
		mw->upto = ww.upto;
		mw->nfree = ww.nfree;
		mw->nused = ww.nused;
		mw->wstart = upto;

		strncpy(mw->word,ww.word,MAX_WORD_LEN);
	if (strcmp(mw->word,SPECIAL)==0) 
	    imsg("Word {%s} start %d upto %d nused %d wstart %d \n",mw->word,ww.start,ww.upto,ww.nfree,ww.nused,upto);

		
		fts_word_add_hash(mw);
		upto += n;
        }
        s_fclose(f);
        return TRUE;
}
int fts_read_word(int offset, MWORD *mw);
int fts_read_word(int offset, MWORD *mw)
{
	WWORD ww;
        int curlen,nwords=0;
        curlen = 0;

	fseek(fw,offset,SEEK_SET);
	s_fread(&ww,1,sizeof(WWORD),fw);
	mw->start = ww.start;
	mw->upto = ww.upto;
	mw->nfree = ww.nfree;
	mw->nused = ww.nused;
	mw->wstart = offset;
	strncpy(mw->word,ww.word,MAX_WORD_LEN);
        return TRUE;
} 
int fts_word_append(FILE *xfw, MWORD *mword)
{
        int n;
	WWORD dw;
        if (s_fseek(xfw,0,SEEK_END)!=0) {filemsg("Couldn't seek to end of words file \n"); return FALSE;}

	mword->wstart = word_upto;
	dw.upto  = mword->upto;
	dw.start = mword->start;
	dw.nfree = mword->nfree;
	dw.nused = mword->nused;
	strncpy(dw.word,mword->word,MAX_WORD_LEN);
 
        n = s_fwrite(&dw,1,sizeof(WWORD),xfw);
        if (n<sizeof(WWORD)) {filemsg("Word append failed\n"); goto failed;}
        word_upto += n; 
        return TRUE;
failed:;
	emsg("Fts dieing because word append failed \n");
	abort();
	return FALSE;
}
int fts_word_update(MWORD *mword)
{
        int n;
	WWORD dw;
        if (s_fseek(fw,mword->wstart,SEEK_SET)!=0) {filemsg("Couldn't seek to end of words file \n"); return FALSE;}
	dw.upto  = mword->upto;
	dw.start = mword->start;
	dw.nfree = mword->nfree;
	dw.nused = mword->nused;
        n = s_fwrite(&dw,1,sizeof(WWORD)-MAX_WORD_LEN,fw);
        if (n<(sizeof(WWORD)-MAX_WORD_LEN)) {filemsg("Word UPDATE failed\n"); return FALSE;}
	return TRUE;
}
void fts_read_groups(void)
{
        char bf[BFSZ];
        char agroup[BFSZ];
        char *s;
        int aidx;
        FILE *f;

        f = fopen(FILE_GROUPS,"r");
        if (f==NULL) return;
	group_next = 0;
        for (;!feof(f);) {
                if (fgets(bf,BFSZ-10,f)==NULL) break;
                s = strtok(bf,":");
                for (;s!=NULL;) {
                        ncpy(agroup,s,BFSZ-1);  s = strtok(NULL,":"); if (s==NULL) continue;
                        aidx = atoi(s); 
                        s = strtok(NULL,":"); if (s==NULL) continue;
                        if (strcmp(agroup,"total_items")==0) {
                                total_items = atoi(s);
                        } else {
                        	fts_group_add(agroup,aidx,atoi(s));
	                        if (group_next<(aidx+1)) group_next = aidx + 1;
			}
                }
        }
        imsg("Total items in database %d\n",total_items);
        fclose(f);
}
void fts_append_group(char *group, int idx, int num)
{
        FILE *f;

        f = fopen(FILE_GROUPS,"a");
        if (f==NULL) {filemsg("Cannot read groups\n"); return;}
        fprintf(f,"%s:%d:%d\n",group,idx,0);
        fclose(f);
}
void fts_append_total(void)
{
        FILE *f;
        static int last;

        if (last==total_items) return;
        last = total_items;

        f = fopen(FILE_GROUPS,"a");
        if (f==NULL) {filemsg("Cannot read groups\n"); return;}
        fprintf(f,"total_items:%d:%d\n",total_items,total_items);
        fclose(f);
}
int fts_write_group(void)
{
        FILE *f;
        int i;
        GLIST *d;
        
        f = fopen(FILE_GROUPSNEW,"w");
        if (f==NULL) {filemsg("Cannot read groups\n"); return FALSE;}
        for (i=0; i<HASH_WORDS; i++) {
                d = groups[i];
                for (;d!=NULL;d=d->next) {
                        if (d->idx <0) emsg("Panic write_group \n");
                        if (fprintf(f,"%s:%d:%d\n",d->word,d->idx,group_base2[d->idx])<=0) {
                                filemsg("Error writing groups.new file\n");
                                return FALSE;
                        }
                }
        }
        if (fprintf(f,"total_items:%d:%d\n",total_items,total_items)<=0) {
                filemsg("Error writing groups.new file\n");
                return FALSE;
        }
        fclose(f);
        return TRUE;
}


static WLIST *uniq[U_HASH];
#define WSPACE " \n\t\015\012,:\\/;!|$%*()@[]\"'"
static int weight=1;

#define uni_hash(p) abs(((int) p) % U_HASH)
void fts_uni_add(char *word)
{
        WLIST *d,*last,*next;   
        MWORD *wdict = fts_word_find_add(word);
        int h;

        if (wdict==NULL) return; /* Exceeded maximum maybe? */
        h = uni_hash(wdict);
        d = uniq[h];
        last = NULL;
        for (;d!=NULL; d=d->next) {
                if (strcmp(d->dict->word,word)==0) {
                        d->score += weight;
                        if (d->score > 200) d->score = 200;
                        return;
                }
                last = d;
        }

	if (strcmp(word,SPECIAL)==0) imsg("fts_uni_add(%s)\n",SPECIAL);
        next = mymalloc(sizeof(*d));
	ftsmem += (sizeof(*d));
	
        if (last==NULL) uniq[h] = next;
        else last->next = next;
        next->dict = wdict;
        next->score = weight;
        next->next = NULL;
        stat_nunique++;
}
static int last_line_done;
int fts_add_line(char *s)
{
        char bf[300];
        char *ss = strchr(s,'\n');

	last_line_done = time(NULL);
        if (ss!=NULL) *ss = 0;
        ncpy(bf,s,200); s = bf;
        weight = 15;
        s = strtok(s,WSPACE);
        if (s!=NULL) s = strtok(NULL,WSPACE); /* Skip header name */
        for (;s!=NULL;) {
                if (fts_fix_word(s)) {  
                        stat_nwords++;  fts_uni_add(s); 
                }
                s = strtok(NULL,WSPACE);
        }
        if (ss!=NULL) *ss = '\n';
        return TRUE;
}
static int nhead;
static char *head[40];
static int head_len[40];
int fts_find_head(char *hh)
{
        char bf[BFSZ];
        char *s;
        int i;

        if (nhead==0) {
                ncpy(bf,fts_ini(I_HEADERS),BFSZ-1);
                s = strtok(bf,", \t\n");
                for (;s!=NULL;) {
                        head_len[nhead] = strlen(s);
                        head[nhead++] = xstrdup(s);
                        s = strtok(NULL,", \t\n");
                }
        }
        for (i=0; i<nhead; i++) {
                if (strncmp(hh,head[i],head_len[i])==0) return TRUE;
        }
        return FALSE;

}
int lib_gmtime(void);
int fts_add(char *group, int num, char *article)
{
        int item_words;
        char *s;
        FILE *f;
        int nw=0;
	int xdate = 0;
	static int lastdone;
        static int last,lasti;
	
	int old = stat_nwords;

   	
        if (lib_gmtime()-last > 120) {
                fts_append_total();
                if (((total_items-lasti) > lasti) || ((lib_gmtime()-lastdone)>(60*10) )) {
			lastdone = lib_gmtime();
                        fts_cache_write();
                        fts_flush_files();
                        f = fopen(FILE_SWITCH,"w");
			emsg("Switch scheduled\n");
			if (f!=NULL) {
				fprintf(f,"Time to switch to pickup new items %d new of %d \n",total_items-lasti,total_items);
				fclose(f); f = NULL;
			}
			lasti = total_items;
			fts_index_write();
                }
		fts_background(); /* do expire if it's scheduled. */
                last = lib_gmtime(); 
        }

        fts_open();
	fts_word_open();
        
        item_words = atoi(fts_ini(I_ITEM_WORDS));
        s = article - 1; /*strchr(article,'\n');*/
/*imsg("Start of article (%.90s)\n",article);*/
        for (;s!=NULL;) {
                s++;
                if (*s=='\015') s++;
                if (*s=='\n') goto do_body;
/*imsg("Start of line (%.90s)\n",s);*/
                if (fts_find_head(s)) fts_add_line(s);
		if (strncmp(s,"Date: ",6)==0) {
			xdate = parsedate(s+6);
			if (xdate==0) imsg("Date invalid? (%s)\n",s); 
		}
                s = strchr(s,'\n');
        }       
        if (old!=stat_nwords) total_items++;
        imsg("No body in article %s %d \n",group,num);
        return TRUE;

do_body:
	if (xdate==0) {
		xdate = time(NULL);
		imsg("Invalid date, using todays date\n");
	}
        weight = 5;
        s = strtok(s,WSPACE);
        for (;s!=NULL;) {
                if (fts_fix_word(s)) {
                        stat_nwords++;
                        fts_uni_add(s);
                } 

                nw++;
                if (nw > item_words) {dmsg("Got %d %d words\n",nw,item_words); break;}
                s = strtok(NULL,WSPACE);
                if (nw>20) weight=3;
                if (nw>200) weight=1;
        }

        fts_uni_cache(group,num,xdate);
        if (old!=stat_nwords) total_items++;
        return TRUE;
}
static char xtable[500];
void fts_table_init(void);
void fts_table_init(void)
{
        int i;
	int dodigits = fts_true(I_INDEX_NUMBERS);
	int doasian = fts_true(I_INDEX_ASIAN);
	int doall = fts_true(I_INDEX_ALL);
        for (i=0; i<255; i++) {
                xtable[i] = 0;
                if (isalpha(i)) xtable[i] = tolower(i);
                if (dodigits) if (isdigit(i)) xtable[i] = i; 
                if (doasian) {
			xtable[i] = i;
			if (i<127) if (isalpha(i)) xtable[i] = tolower(i);
			if (i==' ') xtable[i] = 0;
			if (i=='[') xtable[i] = 0;
			if (i==']') xtable[i] = 0;
			if (i==',') xtable[i] = 0;
			if (i=='.') xtable[i] = 0;
			if (i==':') xtable[i] = 0;
			if (i=='?') xtable[i] = 0;
			if (i=='(') xtable[i] = 0;
			if (i==')') xtable[i] = 0;
			if (i=='/') xtable[i] = 0;
			if (i==';') xtable[i] = 0;
			if (i=='\'') xtable[i] = 0;
			if (i=='"') xtable[i] = 0;
		}
                if (i=='.') xtable[i] = '_';
                if (i=='_') xtable[i] = '_';
                if (i=='-') xtable[i] = '_';
        }
}
int fts_fix_word( char *xx)
{
        unsigned char *s= (unsigned char *) xx;
        unsigned char *ss = (unsigned char *) xx;
        int n=0;
        static int done_init;
        if (!done_init) {fts_table_init(); done_init=TRUE;}

        for (;*s!=0;s++) {
                *s = xtable[*s];
                n++;
                if (*s==0) break;
        }
	if (n==1) return FALSE;
	if (n>2) if (ss[n-1] == '_') ss[n-1] = 0; /* Strip trailing full stop*/
        if (n>20) return FALSE;
        if (n>MAX_WORD_LEN) ss[MAX_WORD_LEN] = 0;
        if (n==0) return FALSE;
        return TRUE;
}
static WCACHE *clist[HASH_CACHE];
#define cache_hash(p) abs(((int) p) % HASH_CACHE)

void fts_uni_cache(char *group, int num, int xdate) /* Copy unique words into cache */
{       
        WLIST *u,*unext;
        int gid = fts_group_find_add(group,num);
        int i;
        for (i=0; i<U_HASH; i++) {
                u = uniq[i];
                for (;u!=NULL;u=unext) { /* For each unique word in article */ 
			unext = u->next;
                        fts_cache_add(u->dict,u->score,gid,num,xdate);
			ftsmem -= (sizeof(*u));
                        myfree(u);
                }
                uniq[i] = NULL; /* And free for next article */
        }
}
void fts_cache_add(MWORD *dict, int score, int gid, int num, int xdate)
{
        WCACHE *c,*last,*next;
        
        int h = cache_hash(dict);
        ncached++;
        if (ncached>10000) fts_cache_write();
        last = NULL;
        c = clist[h];
        for (;c!=NULL;c=c->next) {
                if (c->dict==dict) {
                        fts_cache_addv(c,dict,score,gid,num,xdate);
                        return;
                }
                last = c;
        }
	ftsmem += (sizeof(*c));
        next = mymalloc(sizeof(*c));
        next->dict = dict;
        next->list = NULL;
        next->next = NULL;
        if (last==NULL) clist[h] = next;
        else last->next = next;
        fts_cache_addv(next,dict,score,gid,num,xdate);
}
void fts_cache_addv(WCACHE *list,MWORD *dict, int score, int gid, int num, int xtime)
{
        VLIST *v,*next,*last;
	ftsmem += (sizeof(*v));
        next = mymalloc(sizeof(*v));
        next->next = NULL;
        next->gid = gid;
        next->num = num;
        next->score = score;
	next->xtime = (unsigned short) (xtime / 86400);
        last = NULL;
        v = list->list; 
        for (;v!=NULL;v=v->next) last=v;
        if (last==NULL) list->list = next;
        else last->next = next;
}
void fts_cache_free(VLIST *v);
void fts_cache_write(void)
{
        WCACHE *c,*next;
        int i;
        emsg("Writing cache...%d \n",ncached);
        if (ncached==0) return;
	for (i=0; i<HASH_CACHE; i++) {
		c = clist[i];
		dmsg("ftslib: clist [%d]\n",i);
		for (;c!=NULL;c=next) {
			next=c->next;
			dmsg("ftslib: write_one %p\n",c);
			fts_cache_write_one(fp,fw,c->dict,c->list);
			dmsg("ftslib: free the list %p\n",c->list);
			fts_cache_free(c->list);
			ftsmem -= (sizeof(*c));
			myfree(c);
		}
		dmsg("ftslib: clist next\n");
		/*if (clist[i]!=NULL) {
			myfree(clist[i]);
			ftsmem -= (sizeof(*clist[i]));
		}*/
		clist[i] = NULL;                
	}
        emsg("done\n");
        ncached = 0;

} 
void fts_cache_free(VLIST *v)
{
	VLIST *vnext;
        for (; v!=NULL; v=vnext) {
		vnext = v->next;
                myfree(v);
		ftsmem -= sizeof(*v);
        }
}
void fts_cache_write_one(FILE *fp, FILE *fw, MWORD *w, VLIST *list)
{
        VLIST *v;
        int i;
	int padding;
	int nv;
	int atend = FALSE;
	int nwritten = 0;

        if (fp==NULL) return;
        for (v=list, nv=0; v!=NULL; v=v->next) nv++;

        for (v=list; v!=NULL; v=v->next) {
		if (w->upto>=0) fts_point_seek(fp,w->upto);
		if ((w->upto<0) || (w->nfree<=0)) {
		 	if (w->upto>=0) fts_point_write(fp,END_CELL,point_upto,0,0);
			w->upto = point_upto;
			if (w->start<0) w->start = w->upto;
	  		fts_point_seek(fp,w->upto);
			w->nfree = 32000;
			atend = TRUE;
		}             
		w->upto += fts_point_write(fp,v->gid,v->num,v->score,v->xtime);
		w->nfree--;
		nwritten++;
		if (w->nused < 30000) w->nused++;
		if (nwritten>29000) break;
        }
	if (atend) {
		padding = w->nused + nv;
		padding = padding / 5;  /* Add a gap of about 20% */
		if (padding < 2) padding = 2;
		for (i=0; i<padding-1; i++) {
		 	fts_point_write(fp,FREE_CELL,0,0,0); /* Free Cells*/
		}
	 	fts_point_write(fp,END_CELL,-1,0,0); /* Jump point for future use */
		w->nfree = padding-1;
	}
	if (fw!=NULL) fts_word_update(w);
} 
static int point_atend;
int fts_open(void)
{       
 	wtotal = atoi(fts_ini(I_WORD_TOTAL));
	if (wtotal==0) wtotal = 30000;

        atend = FALSE;
        if (fp!=NULL) s_fclose(fp); fp = NULL;
        if (isread) {
                fp = s_fopen(file_point,"rb");
        } else {
                fp = s_fopen(file_point,"r+b"); 
		if (fp==NULL) fp = s_fopen(file_point,"w+b"); 
        }
        if (fp==NULL) {filemsg("Could not open %s\n",file_point); return FALSE;}
        s_fseek(fp,0,SEEK_END);
        point_upto = s_ftell(fp);
	point_atend = TRUE;
	return TRUE;
}
void fts_word_open(void)
{       
        if (fw!=NULL) { s_fclose(fw); fw = NULL;}
        if (isread)     fw = s_fopen(FILE_WORDS,"rb");
        else            fw = s_fopen(FILE_WORDS,"r+b");
        if (!isread) if (fw==NULL) {fw = s_fopen(FILE_WORDS,"w+b"); emsg("Creating %s\n",FILE_WORDS);}
        if (fw==NULL) filemsg("Could not open %s\n",FILE_WORDS);
}
int fts_point_seek(FILE *fp, int go)
{
	if (go==point_upto) point_atend = TRUE;
	else point_atend = FALSE;
 	if (s_fseek(fp,go,SEEK_SET)!=0) return FALSE;
 	return TRUE;
}
int fts_point_write(FILE *fp, int gid, int num, int score, int xtime)
{
	char bf[BFSZ];
	unsigned short xxtime = (unsigned short) xtime;
	unsigned short xgid = gid;
	char xscore = score;
	int rlen = psize;
	int r;

	/*if ( ((gid>32000) && (gid<65500)) || ((gid>group_next) && (gid<65500)) || (gid<0) || (num<-2))	emsg("point write corrupt data %d %d groupmax %d\n",gid,num,group_next);*/
	if ( ((gid>group_next) && (gid<65500)) || (gid<0) || (num<-2))	emsg("point write corrupt data %d %d groupmax %d\n",gid,num,group_next);
	memcpy(bf,&xgid,2);
	memcpy(bf+2,&num,4);
	memcpy(bf+2+4,&xscore,1);
	if (istime) memcpy(bf+2+4+1,&xxtime,2);
	r = s_fwrite(bf,1,rlen,fp);
	if (r<rlen) {
	 	filemsg("Writing to point file failed, fts will now die\n");
		emsg("fts exiting\n");
		abort();
	}
	if (point_atend) point_upto += r;
	return r;
}
int sort_point(const void *aa, const void *bb);
int sort_point(const void *aa, const void *bb)
{
        VLIST *a,*b; 
        int r;  
        a = *((VLIST **) aa);
        b = *((VLIST **) bb);
        r = a->gid - b->gid;
        if (r==0) r = a->num - b->num;
        return r;
}
int sort_score(const void *aa, const void *bb);
int sort_score(const void *aa, const void *bb)
{
        VLIST *a,*b;    
        a = *((VLIST **) aa);
        b = *((VLIST **) bb);
        return  b->score - a->score;
}
int sort_date_score(const void *aa, const void *bb)
{
        VLIST *a,*b;    
	int r;
        a = *((VLIST **) aa);
        b = *((VLIST **) bb);
	r = (int) b->xtime - (int) a->xtime;
	if (r!=0) return r;
        return  b->score - a->score;
}
int sort_gid_score(const void *aa, const void *bb)
{
        VLIST *a,*b;    
	int r;
        a = *((VLIST **) aa);
        b = *((VLIST **) bb);
	r = (int) a->gid - (int) b->gid;
	if (r!=0) return r;
        return  b->score - a->score;
}
int fts_vlist_times(VLIST *v, int times)
{
        int n=0;
        for (;v!=NULL;v=v->next) {
                v->score *= times;
                n++;
        }
        return n;
}
#define MAX_SEARCH 10000
/*
New version

New fts_search
	allow 'and' 
	Group wild card list
	Sort order (score,date,group)


12:06:33 1:info: make_mask(*.dnews) 
16 12:06:33 1:info: make_mask(netwin.dmail) (*.dnews) 
16 12:06:33 1:info: Mask[0] = true
16 12:06:33 1:info: make_mask(netwin.dnews) (*.dnews) 
16 12:06:33 1:info: Mask[1] = true
16 12:06:33 1:info: ismask=1, isand=0, wild=(*.dnews), xsince=0

*/
int match_wild(char *p,char *text);
int domatch(char *text, char *p);
int fts_make_mask(char *mask, char *wild)
{
	int h;
        GLIST *d;
	char bf[BFSZ];
	char *s;
	if (strlen(wild)==0) return FALSE;
	if (strcmp(wild,"*")==0) return FALSE;
	ncpy(bf,wild,BFSZ-10);
	s = strtok(bf,", ");
	for (;s!=NULL;) {
	 for (h=0; h<HASH_WORDS; h++) {
	   for (d=groups[h]; d!=NULL; d=d->next) {
		if (match_wild(s,d->word)) {
			if (d->idx<0) continue;
			if (d->idx>=max_groups) continue;
			mask[d->idx] = TRUE;
		}
	   }
	 }
	 s = strtok(NULL,", ");
	}
	return TRUE;
}
int fts_validgid(int x)
{
	if (x<0) return FALSE;
	if (x>=max_groups) return FALSE;
	return TRUE;
}
int fts_search(char *terms[], int nterms, int isand, char *gwild, int xsince, int xbefore, int stype
	       , int r_terms[], int from, int to, char * r_name[] 
	       , int r_num[], int r_score[], int r_xtime[])
{
        int i;
        char bf[BFSZ];
        MWORD *dict;
        VLIST *v;
        int total = 0;
        VLIST *listv[20];
        int t;
	int ndup;
	int xtot=0;
        static VLIST *p[MAX_SEARCH];
        static VLIST *p2[MAX_SEARCH];
        int np=0,nu=0;
	char *mask;
	int ismask;
	int z_nbefore,z_nmask,z_nsince;
	z_nbefore=z_nmask=z_nsince=0;


	if (!fts_open()) {
		for (i=0; i<=nterms; i++) {
			r_terms[i] = 0;
			r_name[i]=NULL; 
		}
		return 0;	
	}
	fts_word_open();

	ftsmem += (max_groups);
	mask = mymalloc((max_groups));
	for (i=0; i<max_groups; i++) mask[i] = FALSE;
	ismask = fts_make_mask(mask,gwild);
	imsg("ismask=%d, isand=%d, wild=(%s), xsince=%d, xbefore=%d stype=%d\n",
		ismask,isand,gwild,xsince,xbefore,stype);
        
	if (nterms>20) nterms = 20;
        for (t=0; t<nterms; t++) {
                ncpy(bf,terms[t],BFSZ-1);
                fts_fix_word(bf);
                dict = fts_index_find(bf);
		if (dict==NULL) imsg("Word %.80s not in index\n",bf);
                if (dict==NULL) listv[t] = NULL;                        
                else listv[t] = fts_read(dict);
                r_terms[t] = fts_vlist_times(listv[t],nterms-t);
                total += r_terms[t];
        }
        r_terms[nterms] = total;

        /* Construct an array of pointers to all 5 lists */
        np = 0;
        for (t=0; t<nterms; t++) {
                v = listv[t];
                for (;v!=NULL;v = v->next) {
			xtot++;
			if (ismask) if (fts_validgid(v->gid)) if (!mask[v->gid]) {z_nmask++; r_terms[t]--; continue;}

			if (istime) {
			 if (xbefore!=0) {
				if (v->xtime>(xbefore/86400)) {z_nbefore++;r_terms[t]--; continue;}
			 }
			 if (xsince!=0) {
				if (v->xtime<(xsince/86400)) {z_nsince++; r_terms[t]--; continue;}
			 }
			}
                        p[np++] = v;
                        if (np>MAX_SEARCH) {
                                imsg("Too many matches to sort all %d\n",np); 
                                goto gotarray;
                        }

                }
        }
gotarray:
	imsg("nbefore %d nsince %d nmask %d\n",z_nbefore,z_nsince,z_nmask);
	imsg("Search, found total of %d/%d %d %d words\n",np,xtot,nterms,total);
        qsort(p,np,sizeof(p[0]),sort_point);


        /* Remove duplicates and add up score; */
        p2[0] = p[0];
	ndup = 1;
        for (i=1,nu=0; i<(np); i++) {
                if ((p2[nu]->gid == p[i]->gid) && (p2[nu]->num == p[i]->num)) {
                        p2[nu]->score += p[i]->score;
			ndup++;
                } else {
			if (isand) if (ndup<nterms) {
				nu--;
			} 
			ndup = 1;
	                nu++;
	                p2[nu] = p[i];	
                }
        }
	imsg("nu=%d, np=%d, ndup=%d, isand=%d\n",nu,np,ndup,isand);
	if (nu>=0) if (isand) if (ndup<nterms) nu--;
        nu++;
        if (np==0) nu = 0;
        /* Now sort in score order */ 
	/* stype 
		1 = score order
		2 = gid/score
		3 = date/score
	*/


	if (stype==2) qsort(p2,nu,sizeof(p[0]),sort_date_score);
	else if (stype==3) qsort(p2,nu,sizeof(p[0]),sort_gid_score);
	else		qsort(p2,nu,sizeof(p[0]),sort_score);


	/* Addition, return actual number of matches.*/
        r_terms[nterms] = nu;
        
	/* Now return answers to user arrays */ 
        
        for (i=from; i<=to; i++) {
                if (i>=nu) r_name[i-from]=NULL; 
                else {
                        r_name[i-from] = fts_group_name(p2[i]->gid);
                        r_num[i-from] = p2[i]->num;
                        r_score[i-from] = p2[i]->score;
                        r_xtime[i-from] = p2[i]->xtime;
                }
        }

	for (t=0; t<nterms; t++) {
		fts_read_free(listv[t]);
	}

	fts_flush_files();
	myfree(mask);
	ftsmem -= max_groups;
        return total;
}
int fts_search_old(char *terms[], int nterms, int r_terms[], int from, int to, char * r_name[], int r_num[], int r_score[], int r_xtime[])
{
        int i;
        char bf[BFSZ];
        MWORD *dict;
        VLIST *v;
        int total = 0;
        VLIST *listv[20];
        int t;
        static VLIST *p[MAX_SEARCH];
        static VLIST *p2[MAX_SEARCH];
        int np=0,nu=0;

	fts_open();
	fts_word_open();

        if (nterms>20) nterms = 20;
        for (t=0; t<nterms; t++) {
                ncpy(bf,terms[t],BFSZ-1);
                fts_fix_word(bf);
                dict = fts_index_find(bf);
                if (dict==NULL) listv[t] = NULL;                        
                else listv[t] = fts_read(dict);
                r_terms[t] = fts_vlist_times(listv[t],nterms-t);
                total += r_terms[t];
        }
        r_terms[nterms] = total;
        /* Construct an array of pointers to all 5 lists */
        np = 0;
        for (t=0; t<nterms; t++) {
                v = listv[t];
                for (;v!=NULL;v = v->next) {
                        p[np++] = v;
                        if (np>MAX_SEARCH) {
                                imsg("Too many matches to sort all %d\n",np); 
                                goto gotarray;
                        }

                }
        }
gotarray:
        qsort(p,np,sizeof(p[0]),sort_point);


        /* Remove duplicates and add up score; */
        p2[0] = p[0];
        for (i=1,nu=0; i<(np); i++) {
                if ((p2[nu]->gid == p[i]->gid) && (p2[nu]->num == p[i]->num)) {
                        p2[nu]->score += p[i]->score;
                } else {
                        nu++;
                        p2[nu] = p[i];
                }
        }
        nu++;
        if (np==0) nu = 0;
        /* Now sort in score order */ 
        qsort(p2,nu,sizeof(p[0]),sort_score);

        /* Now return answers to user arrays */ 
        
        for (i=from; i<=to; i++) {
                if (i>=nu) r_name[i-from]=NULL; 
                else {
                        r_name[i-from] = fts_group_name(p2[i]->gid);
                        r_num[i-from] = p2[i]->num;
                        r_score[i-from] = p2[i]->score;
                        r_xtime[i-from] = p2[i]->xtime;
                }
        }

	for (t=0; t<nterms; t++) {
		fts_read_free(listv[t]);
	}

	fts_flush_files();
        return total;
}

void fts_searchtest(char *wordlist)
{       
        char bf[BFSZ];
        MWORD *dict;
        VLIST *v,*vnext;

        ncpy(bf,wordlist,BFSZ-1);
        fts_fix_word(bf);
        dict = fts_word_find(bf);
        if (dict==NULL) {emsg("No words found {%s}\n",bf); return;}
        v = fts_read(dict);
        if (v==NULL) {emsg("Word doesn't point to anything {%s}\n",bf); return;}
        for (; v!=NULL; v=vnext) {
                emsg("%s group=%d, num=%d, score=%d\n",bf,v->gid,v->num,v->score);
		vnext = v->next;
                myfree(v);
		ftsmem -= sizeof(*v);
        }
}
void fts_test_word(char *wordlist)
{       
        char bf[BFSZ];
        MWORD *dict;
        VLIST *v,*vnext;

        fts_open();
	fts_word_open();

        ncpy(bf,wordlist,BFSZ-1);
        fts_fix_word(bf);
        dict = fts_word_find(bf);
        if (dict==NULL) {printf("No words found {%s}\n",bf); return;}
	printf("dict info nused=%d start=%d upto=%d\n",dict->nused,dict->start,dict->upto);
        v = fts_read(dict);
        if (v==NULL) {printf("Word doesn't point to anything {%s}\n",bf); return;}
        for (; v!=NULL; v=vnext) {
                printf("%s group=%d, num=%d, score=%d\n",bf,v->gid,v->num,v->score);
		vnext = v->next;
                myfree(v);
		ftsmem -= sizeof(*v);
        }
}

void fts_read_free(VLIST *v)
{
	VLIST *vnext;
        for (; v!=NULL; v=vnext) {
		vnext = v->next;
                myfree(v);
		ftsmem -= sizeof(*v);
        }
}
int fts_hash(char *s, int max)
{
        int total=0;
        int i;
        for (i=0;(*s!=0)  && (i<MAX_WORD_LEN);s++,i++) {
                total += ((*s))*i;    
        }
	total = abs(total);
        return total % max;
}
char *fts_dir(char *s)
{
        static char bf[500];
#ifdef __MWERKS__
        lprintf(bf,500-1,"%s:%s",root,s);
#else
        lprintf(bf,500-1,"%s/%s",root,s);
#endif
        return bf;
}
void fts_trigger(void)
{
	FILE *f;
	/* Triger an fts expire to write indexes  */
	if ((time(NULL) - last_line_done)<60) {
		return;
	}
	f = fopen(FILE_FLUSH,"w");
	if (f==NULL) {emsg("Could not created %s\n",FILE_FLUSH,strerror(errno)); return;}
	fclose(f);
}
void fts_expire(void)
{
        char group[BFSZ];
        char fdel[BFSZ];
        char fdo[BFSZ];
        FILE *f;
        char bf[BFSZ];
        char *s;
        int g;

emsg("Expire starts\n");

        fts_cache_write();
        fts_flush_files();


        fts_open();
	fts_word_open();

        ncpy(fdel,FILE_EXPIRE,BFSZ-1);
        ncpy(fdo,FILE_EXPIRE_DO,BFSZ-1);
        if (rename(fdel,fdo)!=0) filemsg("Rename failed %s %s\n",fdel,fdo);
        f = fopen(fdo,"r");     
        if (f==NULL) {
emsg("Expire nothing to process\n");
                filemsg("No file to expire at the moment {%s}\n",fdo); 
                return;
        }
        for (;!feof(f);) {
                fgets(bf,BFSZ-10,f);
                s = strtok(bf," \n\t"); if (s==NULL) continue;
                ncpy(group,s,BFSZ-1);
                s = strtok(NULL," \n\t"); if (s==NULL) continue;
                g = fts_group_find(group);
                /*-----------*/
                if (g>=0) fts_mark_del(g,atoi(s));
        }
        fclose(f);
emsg("Expire point file\n");
        fts_expire_point();
emsg("Expire free del\n");
	total_items -= fts_del_count();
        if (total_items<0) total_items = 0;

        fts_free_del();
	fts_append_total(); /* Store new total*/

emsg("Expire remove \n");
        if (remove(fdo)!=0) filemsg("Couldn't remove file {%s} \n",fdo);
}

#define HASH_DEL 10000
/* typedef struct Dlist {int gid; int num; struct Dlist *next;} DLIST;*/
static DLIST *xdel[HASH_DEL];
int hashdel(int g, int n);
int hashdel(int g, int n)
{
 	int r = ((g+n) % HASH_DEL);
	if (r<0) {
		emsg("Fatal error, hasdel g=%d num=%d = %d \n",g,n,r);
		r = 0;
	}
	return r;
}
void fts_mark_del(int g, int num)
{
        DLIST *d,*next;

	if ((g<0) || (num<0)) emsg("fts_mark_del: %d %d\n",g,num);

        d = xdel[hashdel(g,num)];
	ftsmem += (sizeof(DLIST));
        next = (DLIST *) mymalloc(sizeof(DLIST));
        next->gid = g;
        next->num = num; 
	next->used = FALSE;
        next->next = d;
        xdel[hashdel(g,num)] = next;    
}
int fts_is_del(int gid, int num)
{
        DLIST *d;
	int i;				 
	if (((gid>32000) && (gid<65500)) || ((gid>group_next) && (gid<65500)) || (gid<0) || (num<-2)) {emsg("fts_is_del: %d %d\n",gid,num); return  TRUE;}
	i = hashdel(gid,num);
        d = xdel[i];
        for ( ;d!=NULL; d=d->next) {
                if ((d->gid==gid) && (d->num==num)) {
			d->used = TRUE;
                        return TRUE;
                }
        }
        return FALSE;
}
int fts_del_count(void)
{
        DLIST *d,*next;
        int i,ndel=0;
        for (i=0; i<HASH_DEL; i++) {
                d = xdel[i];
                for (;d!=NULL; d=next) {
                        next = d->next;
			if (d->used) ndel++;
                }
        }
	return ndel;
}

void fts_free_del(void)
{
        DLIST *d,*next;
        int i;
        for (i=0; i<HASH_DEL; i++) {
                d = xdel[i];
                for (;d!=NULL; d=next) {
                        next = d->next;
                        myfree(d);
			ftsmem -= sizeof(*d);
                }
                xdel[i] = NULL;
        }
}


/*--- INI FILES ----------------------------------------*/
static char *ini_values[200];
static int ini_true[200];
int fts_ini_find(char *s)
{
        int i;
        for (i=0; fini_names[i]!=NULL; i++) {
                if (strcmp(fini_names[i],s)==0) return i;
        }
        emsg("Unknown fts.conf key word {%s}\n",s);
        return -1;
}
char *fts_ini(int x)
{
        if (ini_values[x]!=NULL) return ini_values[x];
        return "";
}
int fts_true(int x)
{
        return ini_true[x];
}
int fts_read_ini(void)
{
        char bf[BFSZ];
        FILE *f;
        int i;
        char *s;

        f = fopen(FILE_INI,"r");
        if (f==NULL) {
                imsg("Unable to find ini file {%s} ok if not running FTS \n",FILE_INI);
                return FALSE;
        }       
        for (;!feof(f);) {
        if (fgets(bf,BFSZ-10,f)!=NULL) {
                        s = strtok(bf," \t\n"); if (s==NULL) continue;
                        if (*s==0 || *s=='#') continue;
                        strlwr(s);
                        i = fts_ini_find(s);
                        if (i<0) { emsg("Unknown ini file setting {%s} \n",s); continue; }
                        s = strtok(NULL,"\n"); if (s==NULL) continue;
                        ini_values[i] = NULL;
                        ini_values[i] = xstrdup(s);
                        strlwr(s);
                        ini_true[i] = FALSE;
                        if (strstr(s,"true")!=NULL) ini_true[i] = TRUE;
                }
        } 
        fclose(f);      
        return TRUE;
}
/*--- END INI FILES ----------------------------------------*/
FILE *b_fopen(const char *fname,const char *mode)
{
        FILE *f;
#ifdef WIN32
        _fmode = _O_BINARY;
#endif
        f = fopen(fname,mode);
#ifdef WIN32
        _fmode = _O_TEXT;
#endif
        return f;
}


/*
Write a hash index for the word file:

index1.fts
1
43
566
333
index2.fts
(1)nnn:nnn:nnn:nnn (offsets to words)
(43)nnn:nnn:nnn:nnn (offsets to words hash 2)

*/
void fts_index_write(void)
{
	MWORD *w;
	FILE *h1,*h2;	
	int i;
	int n,tot;
	h1 = fopen(FILE_HASH1,"wb");
	h2 = fopen(FILE_HASH2,"wb");
	if (h1==NULL) {filemsg("Couldn't write {%s}\n",FILE_HASH1); return;}		
	if (h2==NULL) {fclose(h1); filemsg("Couldn't write {%s}\n",FILE_HASH2); return;}		
	tot = 0;
        for (i=0; i<HASH_WORDS; i++) {
		fwrite(&tot,1,sizeof(int),h1); /* Pointer to this */
                w = words[i];
                for (;w!=NULL; w=w->next) {
                        n = fwrite(&w->wstart,1,sizeof(w->wstart),h2);
			if (n<sizeof(w->wstart)) {filemsg("Error writing index\n"); goto die;}
			tot += n;
		} 
		n = -1;	 /* End of list for this hash value */
                n = fwrite(&n,1,sizeof(w->wstart),h2);
		tot += n;
	}
	fclose(h1);
	fclose(h2);


	return;
die:
	emsg("exiting due to disk problem\n");
	abort();
}
MWORD *fts_index_find(char *word)
{
        int h = fts_hash(word,HASH_WORDS);
        static MWORD w;
	FILE *h1,*h2;	
	int offset;
	h1 = fopen(FILE_HASH1,"rb");
	h2 = fopen(FILE_HASH2,"rb");
	if (h1==NULL) {filemsg("Couldn't open {%s} wait 2 minutes \n",FILE_HASH1); return NULL;}		
	if (h2==NULL) {fclose(h1); filemsg("Couldn't open {%s} wait 2 minutes \n",FILE_HASH2); return NULL;}		

	if (fseek(h1,h*sizeof(int),SEEK_SET)!=0) {filemsg("Couldn't seek in hash1 file\n"); goto failed;}
	if (fread(&offset,1,sizeof(int),h1)<sizeof(int)) {filemsg("Couldn't read hash1 file\n"); goto failed;}
	fclose(h1); h1 = NULL;
	if (fseek(h2,offset,SEEK_SET)!=0) {filemsg("Couldn't seek in hash2 file\n"); goto failed;}
	for (;!feof(h2);) {
		if (fread(&offset,1,sizeof(int),h2)!=sizeof(int)) break;
		if (offset<0) break;
		fts_read_word(offset,&w);
		if (strncmp(word,w.word,MAX_WORD_LEN)==0) {
			fclose(h2); return &w;
		}
	}
	fclose(h2); h2 = NULL;
        return NULL;
failed:
	if (h1!=NULL) fclose(h1);
	if (h2!=NULL) fclose(h2);
	return NULL;
}
