/* Copyright (C) 1995 NetWin World, New Zealand, All rights reserved. */
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include "emsg.h"
#include "str.h"
#include "group.h"
#include "lib.h"
#include "item.h"
#include "dnews.h"
#include "sary.h"
#include "hist.h"
#include "nntp.h"
#include "suck.h"
#include "db.h"
#include "expire.h"
#include "feed.h"
#include "cmd.h"

#undef mymalloc
#undef mycalloc
#undef myfree
#define mymalloc(x) mymalloc2(x,M_EXPIRE)
#define mycalloc(x,y) mycalloc2(x,y,M_EXPIRE)
#define myfree(x) myfree2(x,M_EXPIRE)

/*
	pile 1 groups * use 10%
	pile 2 groups rec.*,comp.*,news.*,soc.* use 50%
	pile 3 groups alt.* use 20%
	pile 4 groups *binaries*,*pictures*,*warez* use 30%
	pile 5 groups local.*,netwin.* use 10%
	pile 6 groups *jobs* use 4% 
	pile 7 groups control* use 0%
	reserve 20%
---------------------------------------------------------------    
Do in blocks of 100 items so background is not badly affected.
Do history file until have 100,000 items to delete.
Then sort in group order, and for each group file.
Read file, delete items, write file.
Do 1 files at a time.
Change main loop so if 'background' then it calls background after checking
there is no 'real' work to do.

Also need to re-create index for history file 


Only needs exclusive access for long enough to read any new items and
append them to the new history file.
*/
void copy_mid(char *dst, char *src);
void expire_log_close(void);
void expire_log(int grp, int num);
void *getmem_malloc(void **xtop, int size);
void getmem_free(void **xtop);
static int *min_vals;
static int nmin;

static int pused,remember,inpurge,inlinks,inxover,inupgrade,inresort,inrefeed,inreindex,infindlost;
static int nkill;
static int s_nhist,s_nold,s_nhistdel;
static int s_nart,s_nartdel,s_nrealdel,s_time;
static int donetoday = TRUE;
static double s_speed;
static int t_pile_hist;
static int t_xover;
static int t_resort;
typedef struct {int min,max,keep,xnormal,purge,unread;} Rules;
static Rules **rcache;
static int nrcache,estate;
static int purge_upto;
static char expire_started[50],expire_end[50];
static int refeed_num;
void expire_cache_clear(void);
void min_clear(void);
static int used_start,used_end,pused_old;
void ex_clear(void);
void expire_pile(void);
int expire_history_pile(void) ;
void expire_pile_line(char *h);

/* Assume the following variables are defined, somehow :-)  IN HOURS */
static int maxitems,minitems,unread;
static int keep,xnormal,purge;  /* In hours NOT days as in the file */
static int total_disk;

int expire_gone_free(void);
int expire_pile_gone(int grp, int num);
void expire_purgelinks(void);
int db_reindex(void);
int expire_small(void);
void expire_refeed(void);
void expire_doxover(void);
static int end_time=0;
static int isdel;
static char del_list[BFSZ];
void expire_groups(char *s)
{
	strcpy(del_list,s);
	if (strlen(s)>0) isdel = TRUE; else isdel = FALSE;
}
void wmsg_lines(String *str)
{
	char bf[2000];
	char *s;
	ncpy(bf,str_s(str),1000);	
	s = strtok(bf,"\n\r");
	for (;s!=NULL;) {
		wmsg("%s\n",s);
		s = strtok(NULL,"\r\n");
	}
}
void expire_clear(void)
{
	inpurge = inlinks = inxover = inupgrade = inresort = inreindex = infindlost = inrefeed = FALSE; 
}
static int expire_giveup;
static char expire_reason[BFSZ];
char *expire_err(void)
{
	return expire_reason;
}
int expire_isgiveup(void)
{
	return expire_giveup;
}
void expire_set_start(void)
{
    used_start = lib_pused();
}
static char expire_subject[BFSZ];
char *expire_subj(void)
{
	lprintf(expire_subject,BFSZ-1,"Daily expire report, start %d%% end %d%%",used_start,lib_pused()); 
	return expire_subject;	
}
void expire_failed(char *s)
{
   time_t ltime;
   time( &ltime );
   expire_giveup = TRUE;
   lprintf(expire_reason,BFSZ-1,"%s Last expire failed (%s)\r\n",ctime(&ltime),s);
}
static int npass;
static int ispile;
void expire_do(void)
{
        static int i;
	static h_type uptoi;
        static char upto[BFSZ]; /* Message id we are about to process */
        char *h;
	int tstart;
	int nsmall = expire_small();
	int oldstate = estate;
	static int last_check;
	static int last_estate = -3;
	static int lastbit,nreturn;
	int xdelay = atoi(config_get(C_EXPIRE_DELAY));
	int tdelay = 2;
		

	tstart = lib_gmtime();	

	if (tstart-last_check>300) {
		last_check = tstart;
		hist_check("hist_check, 5 minutes");
        }
        if (!(is_master()||get_expire_mode())) { /* brian */
        if (!(config_true(C_EXPIRE_FAST))) {
                tdelay = 0;
		if ( ((lib_gmtime()-lastbit >xdelay) && (nreturn > 1))  || (nreturn > 10)) {
			dmsg("expire: Doing a bit of expiring anyway %d \n",nreturn);
		} else {
			nreturn++;
			if (nntp_iswaiting()) return; /* If any IO's waiting, then return */
		}
	} 
	}
	nreturn = 0;
	lastbit = lib_gmtime();
/*imsg("Expire do %d %d %d \n",estate,purge_upto,uptoi);*/
	vtrace();
 	dmsg("expire: Expire do %d %d %d \n",estate,purge_upto,uptoi);
        if (inpurge) { expire_purgelost(); goto ret;}
        else if (inlinks) { expire_purgelinks(); goto ret;}
        else if (inxover) { expire_doxover(); goto ret;}
        else if (inupgrade) { expire_upgrade(); goto ret;}
        else if (inresort) { expire_resort(); goto ret;}
        else if (inreindex) { expire_reindex(); goto ret;}
        else if (infindlost) { expire_findlost(); goto ret;}
        else if (inrefeed) { expire_refeed(); goto ret;}
        pused = lib_pused();
	if (ispile) {expire_pile(); goto ret;}
	if (estate!=last_estate) {
		imsg("expire: Expire is upto state %d\n",estate);
		last_estate = estate;
	}

	vtrace();
        if (estate==0) {
		vtrace();
		t_pile_hist = 0;
		t_resort = 0;
		t_xover = 0;
		npass = 1;
		wmsg("Expire starting, expire_quanta %d \n",expire_quanta());
		hist_check("starting");
		strcpy(expire_reason,"");
		ex_clear();
/* between here */
		hist_flush();
		expire_giveup = FALSE;
		donetoday = FALSE;
                pused_old = pused; 
                expire_load(); /* Get new expire rules */
                used_start = pused;
                uptoi = 0;
                s_nhist=s_nold=s_nhistdel=s_nart=s_nartdel=s_nrealdel=0;
                s_time = lib_gmtime();
		end_time = 0;
                nntp_setsleep(0); /* Keep calling unless you get packets */
                strcpy(expire_started,lib_now());
                h = hist_line(0);
                if (h==NULL) {emsg("Nothing to expire\n"); goto fail;}
                s_nhist++;
                expire_line(h);
		if (expire_giveup) goto fail;
                uptoi = hist_upto();
                h = hist_next(); 
                if (h==NULL) {
                        estate = 3; goto ret;
                }
                copy_mid(upto,h);
                estate=1;
                /* SHOULD CLEAR RULE CACHE ETC.  WHEN THAT EXISTS*/
                expire_cache_clear();
                imsg("expire: Percent disk used %d, chunk %d/%d \n",pused,kill_max(),nsmall);
/* and here */
                goto ret;
        }


        if (estate==1) {
		vtrace();
                h = hist_line(uptoi);
                dmsg("expire: restart search from {%s} \n",h);
                if (uptoi!=0) for (i=0; i<200; i++) {
                        if (h==NULL) break;
                        if (strstr(h,upto)!=NULL) goto found_restart;
                        h = hist_next();
                }
                emsg("expire: Failed to find restart position, %d {%s} \n",uptoi,upto);
                hist_line(uptoi); h = hist_next();
                goto notfound;
found_restart:  dmsg("expire: Found restart position {%s} \n",h);
notfound:       if (h!=NULL) 
		dmsg("expire: Restarting at {%s} {%s} \n",upto,h);
                if (h==NULL) {
                        emsg("expire: Failed to find restart position {%s} \n",upto);
                         goto fail;
                }
		ex_start("History");
                for (i=0;h!=NULL;i++) {
                        s_nhist++;
 			dmsg("Expire line (%s)\n",h);
                        expire_line(h); uptoi = hist_upto();
			if (expire_giveup) goto fail;
                        h = hist_next();
		
                        if (h==NULL) {
                        	dmsg("expire: hist_next() finished, END OF FILE\n");
				hist_check("hist_switch");
		                hist_switch();  
				break;
			}
                        if (pused>89) if (nkill>nsmall) {
                                emsg("expire: Use is high so killing %d items at a time, this may slow expire down\n",nsmall);
 				hist_check("hist_startpurge");
                               goto do_kill; 
                        }
                        if ((nkill+1000)>kill_max()) {
do_kill:                        copy_mid(upto,h);
                                estate = 2;
				imsg("expire: purging files now\n");
      				dmsg("expire: Expire exit %d %d \n",estate,purge_upto);
 				hist_check("hist_startpurge2");
				ex_end("History");
                                goto ret;
                        }
                        if ((lib_gmtime()-tstart)>expire_quanta()) {
                                copy_mid(upto,h);
      				dmsg("expire: Expire exit2 %d %d \n",estate,purge_upto);
				ex_end("History");
                                goto ret; 
                        }
                }
		ex_end("History");
                estate=3;
        }       
     	dmsg("expire: Expire3 exit %d %d \n",estate,purge_upto);

        if (estate==2 || estate==3) { /* Doing a block of items's */
		ex_start("Delete Files");
		vtrace();
                purge_upto = db_doexpire();
		ex_end("Delete Files");
		vtrace();
                if (purge_upto>=0) {
                        goto ret;
                }
                nkill = 0;
                if (estate==2) {estate=1;  npass++; goto ret; } /* Do rest*/
    		dmsg("expire: Expire566 exit %d %d \n",estate,purge_upto);
                estate = 4; goto ret;
        }
   	dmsg("expire: Expire4 exit %d %d \n",estate,purge_upto);

        if (estate==4) {
		vtrace();
		hist_check("hist_state4");
fail:           suck_expire_end(); /* Set suck loop to idle */
                nntp_setsleep(3); /* Switch back to sleeping mode */
		expire_log_close();
                strcpy(expire_end,lib_now());
                wmsg("Expire history completed\n");
                wmsg_lines(expire_status());
                used_end = pused;
                lprintf(expire_subject,BFSZ-1,"Daily expire report, start %d%% end %d%%",used_start,used_end); 

		/*
                lprintf(bf,2000-1,"Daily expire report,start %d%% end %d%%",used_start,used_end); 
                if (config_true(C_REPORT_OLD)) lprintf(bf,2000-1,"Daily expire report",used_start,used_end); 
                if (!get_expire_mode()) dnews_manager_report(bf);
                stats_clear();
		feed_statsclear();
		*/
                estate = 0;
		dmsg("expire: clear item header cache\n");
	        item_clearcache();
		gage_write();
		if (!expire_giveup) if (!donetoday) suck_resort(); /* Schedule an index sort */
                dmsg("expire: Expire75 exit %d %d \n",estate,purge_upto);
                goto ret;
        }
 	dmsg("expire: Expire5 exit %d %d \n",estate,purge_upto);
ret:
	lastbit = lib_gmtime();
	if ((lib_gmtime()-tstart)>5) wmsg("expire: Expire took %d seconds. state=%d was=%d upto=%d\n",lib_gmtime()-tstart,estate,oldstate,purge_upto);
}
/*enum {MSTATE_FAIL = -1, MSTATE_NORMAL, MSTATE_DONE};*/
void expire_pile(void)
{
	int r;
	int t;
	/*
		For each pile
			If using more than it should
				Delete oldest one ( if more than one left)
					
		Scan index, make list of items that are deleted
		Scan history, removing all deleted items.

  
	*/
	pused = lib_pused();
	dmsg("Expire pile %d\n",estate);
	switch (estate) {
		case 0: 
			vtrace();
			t_pile_hist = 0;
			t_resort = 0;
			t_xover = 0;
			expire_gone_free();
			strcpy(expire_reason,"");
			ex_clear();
			hist_flush();
			expire_giveup = FALSE;
			donetoday = FALSE;
	                expire_load(); /* Get new expire rules */
	                used_start = pused;
			imsg("Expire starting at %d\n",pused);
	                s_nhist=s_nold=s_nhistdel=s_nart=s_nartdel=s_nrealdel=0;
	                s_time = lib_gmtime();
			end_time = 0;
			nntp_setsleep(0); /* Keep calling unless you get packets */
	                strcpy(expire_started,lib_now());

			r = db_expire_pile();
			if (r==MSTATE_FAIL) estate = 3;
			if (r==MSTATE_NORMAL) break;
			if (r==MSTATE_DONE) {
				imsg("expire: pile, piles deleted, now patch the index\n");
				estate++;
                                if (get_expire_mode()) {
					tell_master("index_pause");
					db_flush2();
					db_clear_cache("index pause");
				}
			}
			break;
		case 1: 
			vtrace();
			t_pile_hist = 0;
			r = db_expire_index();
			if (r==MSTATE_FAIL) estate = 3;
			if (r==MSTATE_NORMAL) break;
			if (r==MSTATE_DONE) {
				estate++;
                                if (get_expire_mode()) {
					db_flush2();
					db_clear_cache("expire2");
					tell_master("index_unpause");
				}
				imsg("expire: pile, now process history file\n");
			}
			break;
		case 2:
			vtrace();
			t = lib_gmtime();
			r = expire_history_pile();
			t_pile_hist += lib_gmtime() - t;
			if (r==MSTATE_FAIL) estate = 3;
			if (r==MSTATE_NORMAL) break;
			if (r==MSTATE_DONE) {
				estate++;
				imsg("expire: pile, finished, now switch to new index \n");
			}
			break;
		case 3: /* Expire finished */ 
			vtrace();
			suck_expire_end(); /* Set suck loop to idle */
	                nntp_setsleep(3); /* Switch back to sleeping mode */
			expire_log_close();
			strcpy(expire_end,lib_now());
			wmsg("Expire completed\n");
			wmsg_lines(expire_status());
			used_end = pused;
                        estate = 0;
                        /*
	                lprintf(bf,2000-1,"Daily expire report,start %d%% end %d%%",used_start,used_end); 
			if (!get_expire_mode()) dnews_manager_report(bf); "Daily expire report"); 
                        stats_clear();
			feed_statsclear();
			*/
			item_clearcache();
			gage_write();
			if (!expire_giveup) if (!donetoday) suck_resort(); /* Schedule an index sort */
			expire_gone_free();
			break;
	}

	vtrace();
	dmsg("Expire pile done %d\n",estate);
}
/* 1000000 20mb 8 bytes each 



40K groups,  1000 items.  

*/
static void *phead;

typedef struct PAIR {int grp; int num; struct PAIR *next;} Pair;
#define PILE_HASH 10000
static Pair *pile_h[PILE_HASH];
int expire_pile_hash(int grp, int num)
{
	return abs((grp*13 + num) % PILE_HASH);
}
int expire_gone_free(void)
{
	int h;
	Pair *p,*next;
	for (h=0; h<PILE_HASH; h++) {
		for (p=pile_h[h]; p!=NULL; p = next) {
			next = p->next;
/*			myfree(p);*/
		}
		pile_h[h] = NULL;
	}

	getmem_free(&phead);
	phead = NULL;

	return TRUE;
	
}
int expire_pile_gone(int grp, int num)
{
	Pair *p;
	int h = expire_pile_hash(grp,num);
	for (p=pile_h[h];p!=NULL;p=p->next) {
		if (p->grp==grp) if (p->num==num) return TRUE;
	}
	if (num<group_getfrom(grp)) {
		imsg("Removing old history entry %s:%d\n",group_name(grp),num);
		return TRUE;
	}
	return FALSE;
}
void expire_pile_del(int gid, int num)
{
	int h;
	Pair *m;
	int grp = gid_to_group(gid);
	if (gid<0) return;
	h = expire_pile_hash(grp,num);

	m = getmem_malloc(&phead,sizeof(Pair));
/*	m = mymalloc(sizeof(Pair));*/
	m->grp = grp;
	m->num = num;
	m->next = pile_h[h];
	pile_h[h] = m;
}


int expire_history_pile(void) /*enum {MSTATE_FAIL = -1, MSTATE_NORMAL, MSTATE_DONE};*/
{
	/* Scan history index  */
	/* Remove lines for messages we have deleted */

        static int i;
        static h_type uptoi;
	static char upto[BFSZ]; /* Message id we are about to process */
        char *h;
	int tstart;
	int nsmall = expire_small();
	static int last_check;
	static int last_estate = -3;
	static int lastbit,nreturn;
	int xdelay = atoi(config_get(C_EXPIRE_DELAY));
	int tdelay = 2;
	static int estate;
		
	ex_start("History");

	tstart = lib_gmtime();	
	nreturn = 0;
	lastbit = lib_gmtime();
	vtrace();
        if (estate==0) {
		min_clear();
		hist_flush();
                h = hist_line(0);
                if (h==NULL) {emsg("Nothing to expire\n"); goto fail;}
                s_nhist++;
                expire_pile_line(h);
		if (expire_giveup) goto fail;
                uptoi = hist_upto();
                h = hist_next(); 
                if (h==NULL) {
                        estate = 3; goto done;
                }
                copy_mid(upto,h);
                estate=1;
        }


        if (estate==1) {
                h = hist_line(uptoi);
                if (uptoi!=0) for (i=0; i<200; i++) {
                        if (h==NULL) break;
                        if (strstr(h,upto)!=NULL) goto found_restart;
                        h = hist_next();
                }
                emsg("expire: Failed to find restart position, %d {%s} \n",uptoi,upto);
                hist_line(uptoi); h = hist_next();
                goto notfound;
found_restart:  dmsg("expire: pile Found restart position {%s} \n",h);
notfound:       if (h!=NULL) 
		dmsg("expire: Restarting at {%s} {%s} \n",upto,h);
                if (h==NULL) {
                        emsg("expire: Failed to find restart position {%s} \n",upto);
                         goto fail;
                }
                for (i=0;h!=NULL;i++) {
                        s_nhist++;
 			dmsg("Expire line (%s)\n",h);
                        expire_pile_line(h); uptoi = hist_upto();
			if (expire_giveup) goto fail;
                        h = hist_next();
		
                        if (h==NULL) {
                                if (get_expire_mode())
                                {
                                    tell_master("history_flush_pause");
                                    /* Tell master to flush history THEN */
                                    h = hist_line(uptoi);
                                    h = hist_next();
                                    for (;h!=NULL;)
                                    {
                                        expire_pile_line(h);
                                        uptoi = hist_upto();
                                        h = hist_next();
                                    }
                                }
                                dmsg("expire: hist_next() finished, END OF FILE\n");
				hist_check("hist_switch");
                                hist_switch();  
                                tell_master("history_flush_unpause");
  /*removed and placed in flush_unpause                              tell_master("reread_history");*/

                                break;
			}
                        if (i>1000) if ((lib_gmtime()-tstart)>expire_quanta()) {
                                copy_mid(upto,h);
      				dmsg("expire: Expire exit2 %d %d \n",estate,purge_upto);
				ex_end("History");
                                goto ret; 
                        }
                }
		ex_end("History");
                goto done;
        }       
ret:
	ex_end("History");
	lastbit = lib_gmtime();
	if ((lib_gmtime()-tstart)>5) wmsg("expire: pile Expire took %d seconds. state=%d upto=%d\n",lib_gmtime()-tstart,estate,purge_upto);
	return MSTATE_NORMAL;
done:
	estate = 0;
	ex_end("History");
	return MSTATE_DONE;
fail:
	estate = 0;
	ex_end("History");
	return MSTATE_FAIL;
}
void pile_expire_pile(void)
{
	estate = 1;
}
void expire_cache_clear(void)
{
        int i;
        for (i=0; i<nrcache; i++) {
                if (rcache[i]!=NULL) {
                        myfree(rcache[i]);
                        rcache[i] = NULL;
                }
        }
}
#define FILE_ESTAT file_work("estatus.dat")
void expire_write_current_status(char *msg) {
        static String *tmp2;
                
        if (tmp2==NULL) tmp2 = str_new();
        str_clear(tmp2);
        str_add(tmp2,expire_status());
        if (strlen(msg)>0) str_addsn(tmp2,msg);
        if (!str_write(tmp2,FILE_ESTAT)) {
            emsg("unable to write spawned expire status file\n");
        }
}

String *expire_status(void)
{
        static char bf[BFSZ];
        static String *tmp;
        static int t;
        char *state;
	char *spwn;
        int i;
        
        if ((is_spawn_expire()) && !get_expire_mode()) { 
            if (tmp==NULL) tmp = str_new();
            str_clear(tmp);
            if (!str_read(tmp,FILE_ESTAT)) {
                imsg("unable to read spawned expire status file\n");
                str_clear(tmp);
                str_addsn(tmp,"Spawned expire status information unavailable\n\n");
            }
            return tmp;
        }

	state = "Done";
	if (end_time == 0) {
		state = "Working";
		if (s_time==0) state = "None";
	}


        if (tmp==NULL) tmp = str_new();
        str_clear(tmp);

	if (end_time>0) t = end_time;
	else t = lib_gmtime();


        if (estate!=0) t = lib_gmtime() - s_time;
        if (t<1) t=1;
        if (estate!=0) s_speed = (double) s_nart/((double) t);

	if (is_debug()) {
 	 for (i=0; i<ex_stat_n(); i++) {
	 	lprintf(bf,BFSZ-1,"  Expire Mode %s %d seconds\r\n",ex_stat_name(i),ex_stat_total(i));
		str_addsn(tmp,bf);
	 }
	} 
	if (config_true(C_SPAWN_EXPIRE)) spwn = "Spawned "; else spwn = "";
	lprintf(bf,BFSZ-1,"%sExpire started at %s %s, start %d%% end %d%% \015\012",spwn,expire_started,state,used_start,used_end); str_addsn(tmp,bf);
        lprintf(bf,BFSZ-1,"  Processed %d removed %d deleting %d/%d/%d speed %g pass %d\015\012",s_nhist,s_nhistdel,s_nartdel,purge_upto,s_nrealdel,s_speed,npass); str_addsn(tmp,bf);
	if (expire_ispile()) { 
		lprintf(bf,BFSZ-1,"  Pile hist_mins %d, xover_mins %d, resort_mins %d, %s\015\012",t_pile_hist/60,t_xover/60,t_resort/60,
		 xov_expire_status()); str_addsn(tmp,bf);
	} else {
         lprintf(bf,BFSZ-1,"  Xover_mins %d, resort_mins %d, %s\015\012",t_xover/60,t_resort/60,xov_expire_status()); str_addsn(tmp,bf);
	}
	if (end_time>0) {
		t = end_time-s_time;
		lprintf(bf,BFSZ-1,"Expire finished at: %s, Elapsed %d hours, %d minutes\n",expire_end,t/(60*60), (t % (60*60) )/60);      str_addsn(tmp,bf);
	}

        str_addsn(tmp,"\015\012");
        return tmp;
}
void copy_mid(char *dst, char *src)
{
        char *s;
        int len;
        dst[0] = 0;
        s = strchr(src,'\t');
        if (s==NULL) {emsg("fatal: dud message id in expire\n"); return;}
        len = s - src;
        if ((len<0) || (len>200)) {
                emsg("fatal: dud message id in expire, negative value %d\n",len);
                strcpy(dst,"<dud>");
                return;
        }
        ncpy(dst,src,s-src);
        return;
}
/* Message-ID<tab>arrived~expires~posted<tab>group/num group/num ... */
static char curgrp[BFSZ];

int expire_small(void)
{
 	int d;
 	
 	d = atoi(config_get(C_EXPIRE_SMALL));
	if (d>0) return d;
	d = db_total_space();
	d = d * 30;
	if (d<1000) d = 1000;
	if (d>1000000) d = 1000000;
        return d;
}
void min_clear(void)
{
	int i;
	for (i=0; i<nmin; i++) {
		min_vals[i] = -1;
	}
}
void min_set(int g, int n)
{
	if (g<0) return;
	if (g>=nmin) return;
	if (n<min_vals[g]) min_vals[g] = n;
	if (min_vals[g]<0) min_vals[g] = n;
}
int min_get(int g)
{
	if (g<0) return 0;
	if (g>=nmin) return 0;
	return min_vals[g];
}
static int trace_one,trace_expire;
static char trace_mid[BFSZ];
void expire_tracemid(char *s)
{
	strcpy(trace_mid,s);
	trace_expire = TRUE;
}
void expire_pile_line(char *h)
{
        char *g;
        char fixed[10000];
        char *s,*out;
        int arrived,expires,first;
	int islink;
	int now = lib_gmtime();
        int kill_grp[900],kill_num[900],kill[900];
        int ng,gi,i,len;
	int dodel = FALSE;

        arrived = hist_date_arrived(h);
        expires = hist_date_expires(h);
	trace_one = FALSE;
	if (trace_expire) {
		if (strstr(h,trace_mid)!=NULL) trace_one = TRUE;
	}

        if (h!=NULL) dmsg("expire: Expire line %s\n",h);
        g = hist_groupfirst(h);
	strlwr(del_list);
	dodel = FALSE;
        for (ng=0;g!=NULL;) {
                s_nart++;
                ncpy(curgrp,g,100);
                gi = group_find(g);
                if (gi<0) {
                        imsg("Couldn't find group {%s} \n",g);
                        kill[ng] = TRUE;
                        goto next2;
                }
                kill_grp[ng] = gi;
                kill_num[ng] = hist_groupnextnum();
                kill[ng] = FALSE;

		min_set(gi,kill_num[ng]);
		if (isdel) {
			strlwr(curgrp);
			if (group_match(del_list,curgrp)) {
				dodel = TRUE;
				goto xxx;
			}
		}
		if (expire_pile_gone(gi,kill_num[ng])) {
xxx:  			xcache_forget(kill_grp[ng],kill_num[ng]);
                        kill[ng] = TRUE;
                        s_nartdel++;
                }
                ng++;
                if (ng>800) {emsg("expire: ng excceeeded %d \n",ng); return;}
next2:          g = hist_groupnext();
        }
        if (ng==0) {
                s_nold++; /* SHOULD DELETE IF OLDER THAN REMEMBER VALUE*/
                if (isdel || (lib_gmtime()-(remember*60*60*24) >arrived)) {
                        s_nhistdel++;
                        return;
                } 
        }
        for (i=0; i<ng; i++) if (kill[i]) {
		for (i=0; i<ng; i++) {
        		kill[i] = TRUE;
		}
		goto cont;
	}
cont:;

	if (dodel) {
		for (i=0; i<ng; i++) {
	       		kill[i] = TRUE;
		}
	}

        /* Don't kill first one if any of rest are still left */
        for (i=1; i<ng; i++) if (!kill[i]) {
        	kill[0] = FALSE;
	}

        if (ng>0) if (kill[0]) {
        	s_nrealdel++;
		expire_log(kill_grp[0],kill_num[0]);
	}
        for (i=0; i<ng; i++) {
                if (kill[i]) {
                        goto rewrite;
                }
        } 
        if (!hnew_write(h)) expire_failed("Writing new history file");
    return;
rewrite:
        s = strchr(h,'\t');
        if (s!=NULL) s = strchr(s+1,'\t');
        if (s==NULL) {emsg("fatal: history file corrupt, ignoring line\n"); return;}
        len = s-h+1;
        if (len<0 || len>9000) {emsg("fatal: History corrupt, ignoring line\n"); return;}
        ncpy(fixed,h,s-h+1);
        len = strlen(fixed);
        out = fixed + len;
        first = TRUE;
        for (i=0; (i<ng) && (len<9000); i++) {
	  if (i!=0) islink = TRUE; else islink = FALSE;
          if (!kill[i]) {
                if (!first) { strcat(out," "); out++; len++;}
                first = FALSE;
                lprintf(out,BFSZ-1,"%s/%d",group_name(kill_grp[i]),kill_num[i]);
                len += strlen(out);     
                out += strlen(out);
          } else {
		 /* db_kill(kill_grp[i],kill_num[i],islink); */
		 nkill++;
	  }
        } 

        if (!hnew_write(fixed)) {
		expire_failed("Writing new history file");
	}
        return;
}

void expire_line(char *h)
{
        char *g;
        char fixed[10000];
        char *s,*out;
        int arrived,expires,first;
	int islink;
	int now = lib_gmtime();
        int kill_grp[900],kill_num[900],kill[900];
        int ng,gi,i,len;
	int dodel = FALSE;

        arrived = hist_date_arrived(h);
        expires = hist_date_expires(h);
	trace_one = FALSE;
	if (trace_expire) {
		if (strstr(h,trace_mid)!=NULL) trace_one = TRUE;
	}

        if (h!=NULL) dmsg("expire: Expire line %s\n",h);
        g = hist_groupfirst(h);
	strlwr(del_list);
	dodel = FALSE;
        for (ng=0;g!=NULL;) {
                s_nart++;
                ncpy(curgrp,g,100);
                gi = group_find(g);
                if (gi<0) {
                        imsg("Couldn't find group {%s} \n",g);
                        kill[ng] = TRUE;
                        goto next2;
                }
                expire_rules(gi);
                kill_grp[ng] = gi;
                kill_num[ng] = hist_groupnextnum();
                kill[ng] = FALSE;
		if (isdel) {
			strlwr(curgrp);
			if (group_match(del_list,curgrp)) {
				dodel = TRUE;
				goto xxx;
			}
		}
		if (expire_item(gi,kill_num[ng],arrived,expires)) {
			if (expires<=0) {
				if (gi>=0) gage_add(gi,now-arrived);
			}
	                dmsg("expire_item: Rules max %d min %d  keep %d xnormal %d purge %d, arrived %d, expires %d, unread %d \n",
				maxitems,minitems,keep,xnormal,purge,arrived,expires,unread);
/*
*/
xxx:  			xcache_forget(kill_grp[ng],kill_num[ng]);
                        kill[ng] = TRUE;
                        s_nartdel++;
                        dmsg("expire: Expring item %s %d \n",g,kill_num[ng]);
                }
                ng++;
                if (ng>800) {emsg("expire: ng excceeeded %d \n",ng); return;}
next2:          g = hist_groupnext();
        }
        if (ng==0) {
                s_nold++; /* SHOULD DELETE IF OLDER THAN REMEMBER VALUE*/
                if (isdel || (lib_gmtime()-(remember*60*60*24) >arrived)) {
                        s_nhistdel++;
                        return;
                } 
        }
	if (config_true(C_EXPIRE_ANY) || isdel) {
	        for (i=0; i<ng; i++) if (kill[i]) {
			for (i=0; i<ng; i++) {
	        		kill[i] = TRUE;
			}
			goto cont;
		}
	} 
cont:;

	if (dodel) {
		for (i=0; i<ng; i++) {
	       		kill[i] = TRUE;
		}
	}

        /* Don't kill first one if any of rest are still left */
        for (i=1; i<ng; i++) if (!kill[i]) {
        	kill[0] = FALSE;
	}

        if (ng>0) if (kill[0]) {
        	s_nrealdel++;
		expire_log(kill_grp[0],kill_num[0]);
	}
        for (i=0; i<ng; i++) {
                if (kill[i]) {
                        goto rewrite;
                }
        } 
        if (!hnew_write(h)) expire_failed("Writing new history file");
    return;
rewrite:
        s = strchr(h,'\t');
        if (s!=NULL) s = strchr(s+1,'\t');
        if (s==NULL) {emsg("fatal: history file corrupt, ignoring line\n"); return;}
        len = s-h+1;
        if (len<0 || len>9000) {emsg("fatal: History corrupt, ignoring line\n"); return;}
        ncpy(fixed,h,s-h+1);
        len = strlen(fixed);
        out = fixed + len;
        first = TRUE;
        for (i=0; (i<ng) && (len<9000); i++) {
	  if (i!=0) islink = TRUE; else islink = FALSE;
          if (!kill[i]) {
                if (!first) { strcat(out," "); out++; len++;}
                first = FALSE;
                lprintf(out,BFSZ-1,"%s/%d",group_name(kill_grp[i]),kill_num[i]);
                len += strlen(out);     
                out += strlen(out);
          } else {db_kill(kill_grp[i],kill_num[i],islink); nkill++;}
        } 

        if (!hnew_write(fixed)) {
		expire_failed("Writing new history file");
	}
        return;
}
#define EXPIRE_DEL  file_fts("expire.del")
#define EXPIRE_NEW  file_fts("expire.new")
static FILE *fe;
void expire_log(int grp, int num)
{
 	if (!config_true(C_FTS_EXPIRE)) return;
	if (fe==NULL) {
		fe = myfopen(EXPIRE_NEW,"a");
		if (fe==NULL) {filemsg("Can't append to {%s} \n",EXPIRE_NEW); return;}
	} 
	fprintf(fe,"%s %d\n",group_name(grp),num);
}
void expire_log_close(void)
{
	char fold[BFSZ],fnew[BFSZ];
 	if (!config_true(C_FTS_EXPIRE)) return;

	if (fe!=NULL) fclose(fe);
	fe = NULL;
	ncpy(fold,EXPIRE_DEL,BFSZ-1);
	ncpy(fnew,EXPIRE_NEW,BFSZ-1);
	if (lib_exists(fold)) {
		imsg("%s has not been processed by xmit yet for ftslib\n",fold);
		return;
	}
	if (rename(fnew,fold)!=0) filemsg("%s has not been processed by xmit yet\n",fold);
}
 
int expire_item(int g, int num, int arrived, int expires)
{
        int end;
        int age,now;
	int iscached;
	char *reason="none";
        if (g<0) return TRUE; /* Group doest exist, so better kill item :-)*/
	iscached = group_iscachedreal(g);
        end = group_getend(g);
        now = lib_gmtime();
        age = (now-arrived)/(60*60);  /* Age in hours */

/*      dmsg("expire: {%s} %d, age=%d, now=%d, end=%d, expire=%d purge=%d, keep=%d, xnormal=%d, max=%d, min=%d \n",
                        group_name(g),num,age,now,end,expires,purge,keep
                        ,xnormal,maxitems,minitems);                     
*/
#define gotit(xreason) {reason=xreason; goto done;}
	if (trace_one) {
		if (age>unread) if (!iscached) gotit("del:unread");
		if ((end-num) < minitems) gotit("leave:minitems");
		if (((end-num) >= maxitems)  && (expires==0)) gotit("del:maxitems");
		if (purge!=0) if (age>purge) gotit("del:purge");
		if (age<keep) gotit("leave:keep");
		if (expires!=0) if (now<expires) gotit("leave:expire");
		if (expires!=0) if (now>=expires) gotit("del:expire");
		if (age>xnormal) gotit("del:default");
		gotit("leave:no_rule");   /* Keep the item */
done:		
		imsg("Expire: end=%d, age=%d, exp=%d, now=%d, K:%d D:%d P:%d Min:%d Max:%d %s:%d u:%d %s\n"
	   ,end,age/24,expires,now
	   ,keep/24,xnormal/24,purge/24
	   ,minitems,maxitems,group_name(g),num,unread/24,reason);
	}
	if (age>unread) if (!iscached) return TRUE;
        if ((end-num) < minitems) return FALSE; 
        if (((end-num) >= maxitems)  && (expires==0)) return TRUE; 
        if (purge!=0) if (age>purge) return TRUE;
        if (age<keep) return FALSE;
        if (expires!=0) if (now<expires) return FALSE;
        if (expires!=0) if (now>=expires) return TRUE;
        if (age>xnormal) return TRUE;    
        return FALSE;   /* Keep the item */
}

#define DNEWS_EXPIRE "expire.conf"
static Sary *expire;
/*
        total 200
        remember 14
        upto 90% groups * keep 1 normal 2 purge 33 maxitems 100 minitems  0 

        Figure out the rules for this group 
        Add caching if this is a bottle neck, it probably is

*/
static char *issplit;
int expire_malloc_groups(void)
{
	int i;
	issplit  = (char *) mymalloc(max_groups);
	min_vals  = (int *) mymalloc(sizeof(int)*max_groups);
	nmin = max_groups;
	for (i=0; i<nmin; i++) {
		min_vals[i] = -1;
	}

	return TRUE;
}
void expire_rules(int g)
{
        int i;
        char *s;
        static int last_time;
        Rules *r;
	if ((g<0) || (g>=group_n())) return; 
        if (expire==NULL) expire_load();
	if (rcache==NULL) {
		rcache = (Rules **) mymalloc(max_groups*sizeof(Rules *));
		memset(rcache,0,max_groups*sizeof(Rules *));
	}

        if ( (lib_gmtime()-last_time) > 560) {
                expire_cache_clear();
                pused_old = pused;
                last_time = lib_gmtime();
        }
        if ( (pused_old-pused)>1 ) {
                imsg("expire: Disk usage changed, using new rules %d%%->%d%%\n",pused_old,pused);
                expire_cache_clear();
                pused_old = pused;
                last_time = lib_gmtime();
        }

        if (rcache[g]!=NULL) {
                r = rcache[g];
                maxitems = r->max; minitems = r->min;
                keep = r->keep; xnormal = r->xnormal; purge = r->purge;
		unread = r->unread;
                return;
        }
        for (i=0; i<s_len(expire); i++) {
                s = get_word(s_line(expire,i),0);
                if (*s=='#' || *s==0) continue;
                if (strcmpnc(s,"upto")==0) expire_upto(s_line(expire,i));
                else if (strcmpnc(s,"remember")==0) expire_remember(s_line(expire,i));
                else if (strcmpnc(s,"reserve")==0) ;
                else if (strcmpnc(s,"pile")==0) ;
                else if (strcmpnc(s,"total")==0) expire_total(s_line(expire,i));
                else emsg("expire: syntax error in expire.conf line %d (%s)\n",i,s);
        }
        r = (Rules *) mycalloc(1,sizeof(Rules));
        r->max = maxitems ; r->min = minitems ;
        r->keep = keep; r->xnormal = xnormal; r->purge = purge;
	r->unread = unread;
        rcache[g] = r;
        if (g>=nrcache) nrcache = g+1;
}
/*
	pile 1 groups * use 10%
	pile 2 groups rec.*,comp.*,news.*,soc.* use 50%
	pile 3 groups alt.* use 20%
	pile 4 groups *binaries*,*pictures*,*warez* use 30%
	pile 5 groups local.*,netwin.* use 10%
	pile 6 groups *jobs* use 4% 
	pile 7 groups control* use 0%
	reserve 20%
*/
void expire_pile_add(char *rec2);

static int pile_reserve;
static char *pile_match[300];
static int pile_n[300];
static double pile_max[300];
static int pile_sort[300];
static int pile_unread[300];
static double pile_days[300];
static double pile_xdays[300];
static double pile_xmax[300];
static int pile_xsort[300];
static int pile_xunread[300];
static double pile_tot;
static int npile;
static int top_pile;

char *expire_pile_groups(int p)
{
	int i;
	for (i=0; i<npile; i++) {
		if (pile_n[i]==p) return pile_match[i];
	}
	return "none-old-format";
}
static int unread_pile;
int expire_unread_pile(void)
{
	return unread_pile;
}
int expire_npiles(void)
{
	return top_pile;
}
int expire_pile_read(void)
{
	int i;
	char cmd[BFSZ],p1[BFSZ];
	char *line;
	int gotup = FALSE;
	if (expire==NULL) expire_load();
        for (i=0; i<s_len(expire); i++) {
		line = s_line(expire,i);
                ncpy(cmd,get_word(line,0),BFSZ-1);
                ncpy(p1,get_word(line,1),BFSZ-1);
                if (*cmd=='#' || *cmd==0) continue;
                if (strcmpnc(cmd,"upto")==0) gotup = TRUE;
                else if (strcmpnc(cmd,"remember")==0) expire_remember(s_line(expire,i));
                else if (strcmpnc(cmd,"reserve")==0) pile_reserve = atoi(p1);
                else if (strcmpnc(cmd,"pile")==0) expire_pile_add(line);
                else if (strcmpnc(cmd,"total")==0) ;
                else emsg("expire: pile, syntax error in expire.conf line %d (%s)\n",i,cmd);
        }
	if (npile>0) if (gotup) {
		emsg("FATAL: Cannot define pile AND upto rules in expire.conf\n");
		dnews_ex("Cannot define pile AND upto rules in expire.conf");
	}
	pile_tot = 0;
	for (i=0; i<npile; i++) {
		pile_xmax[pile_n[i]] = pile_max[i];
		pile_xdays[pile_n[i]] = pile_days[i];
		pile_xsort[pile_n[i]] = pile_sort[i];
		pile_xunread[pile_n[i]] = pile_unread[i];
	}
	unread_pile = -1;
	for (i=0; i<top_pile; i++) {
		if (pile_xmax[i]>=0) pile_tot += pile_xmax[i];
		if (pile_xunread[i]) unread_pile = i;
	}
	imsg("Unread pile is %d\n",i);
/*	//pile_tot += pile_reserve;*/
	pile_tot = 100;
	if (npile>0) return TRUE;
	return FALSE;
}
double expire_pile_get(int i)
{
	double n;
	if (pile_xmax[i]<0) return -1;
	if (pile_tot==0) pile_tot = 100;
	n = pile_xmax[i] / pile_tot;
	return n;
}
double expire_pile_days(int i)
{
	return pile_xdays[i];
}
int expire_pile_sort(int i)
{
	return pile_xsort[i];
}
int expire_pile_unread(int i)
{
	return pile_xunread[i];
}
void expire_pile_free(void)
{
	int i;
	for (i=0; i<npile; i++) {
		myfree(pile_match[i]);
		pile_match[i] = NULL;
	}
	npile = 0;
}
void expire_pile_add(char *rec2)
{
	char *groups="";
	int days= -1;
	int n,sort,unread;
	double use;
        char *s;
        char bf[BFSZ],wild[BFSZ];
        char rec[1000];
	static int done_warn;

        ncpy(rec,rec2,999);
        if (remember==0 ) {
                if (!done_warn) wmsg("expire: Must define remember at top of expire file\n");
		done_warn = TRUE;
        }
        ncpy(bf,rec,BFSZ-1);
        
        s = strtok(rec," \t\n\r"); if (s==NULL) return;
        s = strtok(NULL," \t\n\r"); if (s==NULL) return;
	n = atoi(s);
	if (top_pile<=n) top_pile = n+1;
	use= -1; days= -1; sort = FALSE;
	unread = FALSE;
	s = strtok(NULL," \t\n\r"); if (s==NULL) {emsg("fatal: groups must be second word in expire.conf (%s)\n",bf); return; }
	for (;s!=NULL;) {
		if (strcmpnc("groups",s)==0) {
			s = strtok(NULL," \t\n"); if (s==NULL) {emsg("fatal3: groups must be second word in expire.conf (%s) \n",bf); return; }
			ncpy(wild,s,400);			
			strlwr(wild);
		} else if (strcmpnc("use",s)==0) {
			s = strtok(NULL," \t\n"); if (s==NULL) {emsg("fatal3: use must be second word in expire.conf (%s) \n",bf); return; }
			use = atof(s);
		} else if (strcmpnc("sort",s)==0) {
			sort = TRUE;
		} else if (strcmpnc("unread",s)==0) {
			unread = TRUE;
		} else if (strcmpnc("days",s)==0) {
			s = strtok(NULL," \t\n"); if (s==NULL) {emsg("fatal3: use must be second word in expire.conf (%s) \n",bf); return; }
			days = atoi(s);
		} else {
			emsg("Invalid word in pile line in expire.conf (%s)\n",s);
			emsg("FATAL Invalid line in expire.conf %s\n",rec2);
			dnews_ex("Invalid line in expire.conf, cannot continue");
		}
		s = strtok(NULL," \t\n"); 
	}
	pile_match[npile] = mystrdup(wild,SD_EXPIRE);
	pile_n[npile] = n;
/*
	if (npile != (n-1)) {
		dnews_ex("FATAL: Invalid PILEs in expire.conf, they must go 1,2,3,4 with no gaps ");	
	}
*/
	if ((n<0) || (n>30)) {
		emsg("Pile value bad must be in the range 1...30 (%s)\n",rec2);
		return;
	}
	pile_max[npile] = use;
	pile_days[npile] = days;
	pile_sort[npile] = sort;
	pile_unread[npile] = unread;
	npile++;
}
int expire_ispile(void)
{
	return ispile;
}
void gage_setpile(int g);

void expire_pile_load(void)
{
	int g;
	ispile = FALSE;
	expire_pile_free();
	if (!expire_pile_read()) return;
	ispile = TRUE;
	imsg("ispile = true\n");
	for (g=0; g<group_n(); g++) {
		gage_setpile(g);
	}
}
void gage_setpile(int g)
{
	int n,i;
	char gname[BFSZ];
	ncpy(gname,group_name(g),300);
	strlwr(gname);
	n = 0;
	if (!ispile) return;
	for (i=0; i<npile; i++) {
		if (group_match(pile_match[i],gname)) n = pile_n[i];
	}
	gage_sethash(g,n);	
}

void expire_reserve(char *line)
{
	pile_reserve = atoi(get_word(line,1));
}
void expire_load(void)
{
	
        if (expire!=NULL) s_free(expire);
        expire = s_alloc();
        s_read(expire,file_config(DNEWS_EXPIRE));
	expire_cache_clear();
	expire_pile_load();
}
int split_check(int g)
{
	if ((g<0) || (g>=max_groups)) return FALSE;
	return issplit[g];
}
void split_find(void)
{
	int i;
	int addit;
	char split_list[10000];
	char cur[BFSZ];
	char *s;
	char bf[BFSZ];
	int ng = group_n();
	int nfound = 0;

	strcpy(cur,"");
	expire_load();
	strcpy(split_list,"");
	if (!config_true(C_DB_SPLIT)) return;
	imsg("Finding split rules in expire.conf\n");
	for (i=0; i<s_len(expire); i++) {
                s = get_word(s_line(expire,i),0);
                if (*s=='#' || *s==0) continue;
                if (strcmpnc(s,"upto")!=0) continue;
		ncpy(bf,s_line(expire,i),BFSZ-1);
		addit = FALSE;
		s = strtok(bf," \t\r\n");
		for (;s!=NULL;) {
			if (strcmpnc(s,"split")==0) addit = TRUE;
			if (strcmpnc(s,"groups")==0) {
				s = strtok(NULL," \t\r\n");
				if (s!=NULL) ncpy(cur,s,BFSZ-1);
				else strcpy(cur,"");
			}
			s = strtok(NULL," \t\r\n");
		}
		if (addit) {
			if (strlen(split_list)!=0) strcat(split_list,",");
			strcat(split_list,cur);
		}
		if (strlen(split_list)>5000) break;
	}
	ncpy(cur,split_list,100);
	imsgf("Split rules (%s) \n",cur);
	strlwr(split_list);
	for (i=0; i<ng; i++) {
		if (group_match(split_list,group_namelc(i))) {issplit[i] = TRUE; nfound++;}
		else issplit[i] = FALSE;
	}
	imsgf("Finished finding split rules, found %d groups\n",nfound);
}
/* upto 90% groups * keep 1 normal 2 purge 33 maxitems 100 minitems  0  */
void expire_upto(char *rec2)
{
        char *s;
        char bf[BFSZ],wild[BFSZ];
        char rec[1000];
	static int done_warn;

        ncpy(rec,rec2,999);
        if (remember==0 ) {
                if (!done_warn) wmsg("expire: Must define remember at top of expire file\n");
		done_warn = TRUE;
        }
        ncpy(bf,rec,BFSZ-1);
        
        s = strtok(rec," \t\n"); if (s==NULL) return;
        s = strtok(NULL," \t\n"); if (s==NULL) return;
        if (pused<atoi(s)) {
                s = strtok(NULL," \t\n"); if (s==NULL) {emsg("fatal: groups must be second word in expire.conf (%s)\n",bf); return; }
                if (strcmpnc("groups",s)!=0) { emsg("fatal2: groups must be second word in expire.conf (%s)\n",bf); return; }
                s = strtok(NULL," \t\n"); if (s==NULL) {emsg("fatal3: groups must be second word in expire.conf (%s) \n",bf); return; }
                ncpy(wild,s,400);
                if (group_match(wild,curgrp)) {
                        expire_set(bf);
                } 
        }
}
/* upto 90% groups * keep 1 normal 2 purge 33 maxitems 100 minitems  0  */
void expire_set(char *rec)
{
        char cmd[BFSZ];  
        char *s;

        /* Setup defaults to expire nothing */
        maxitems = 10000; minitems = 0;
        keep = 0; xnormal = 9999999; purge = 99999;
	unread = 100000;

        s = strtok(rec," \t\n"); if (s==NULL) return;
        s = strtok(NULL," \t\n"); if (s==NULL) return;
        s = strtok(NULL," \t\n"); if (s==NULL) return;
        s = strtok(NULL," \t\n"); if (s==NULL) return;
        s = strtok(NULL," \t\n"); if (s==NULL) return;
        
        for (;s!=NULL;) {
                ncpy(cmd,s,BFSZ-1);
		strlwr(cmd);
		if (strcmp(cmd,"split")==0) goto nextone;

                s = strtok(NULL," \t\n"); 
                if (s==NULL) {emsg("Syntax error in expire.conf\n"); return;}
                                
                if (strcmp(cmd,"keep")==0) keep = (int) (atof(s)*24);
                else if (strcmp(cmd,"purge")==0) purge = (int) (atof(s)*24);
                else if (strcmp(cmd,"default")==0) xnormal = (int) (atof(s)*24);
                else if (strcmp(cmd,"maxitems")==0) maxitems = atoi(s);
                else if (strcmp(cmd,"unread")==0) unread = (int) (atoi(s)*24);
                else if (strcmp(cmd,"minitems")==0) minitems = atoi(s);
                else { emsg("Syntax error in expire.conf {%s} \n",cmd); } 
nextone:        s = strtok(NULL," \t\n"); 
        }       
/*      dmsg("Expire set minmax %d-%d keep, normal purge %d %d %d \n",
                minitems,maxitems,keep,normal,purge);
*/
}
void expire_remember(char *s)
{
        remember = atoi(get_word(s,1));
}
void expire_total(char *s)
{
        total_disk = atoi(get_word(s,1));
        pused = (diskuse_get()/10)/total_disk;
}
extern int xtime_now;
static int too_old_ignore;
void too_old_off(void)
{
	too_old_ignore = TRUE;
}
static int useage;
int set_useage(int x)
{
	useage = x;
	return TRUE;
}

int expire_toold(char *dte)
{
        int posted;
        int toold = atoi(config_get(C_TOO_OLD));

	if (too_old_ignore) return FALSE;
        if (toold==0) toold = 35;
	if (useage>0) toold = useage; useage = 0;
        posted = parsedate(dte);
        if (posted< ( xtime_now-toold*60*60*24 )) {
		imsg("too_old %d<%d days %d\n",posted, ( xtime_now-toold*60*60*24 ),toold);
                return TRUE;
        } 
	if (!config_true(C_FUTURE_OK)) {
	  if ((posted-(60*60*24*4))> xtime_now ) {
		imsg("Dated in the future, %s (%d %d %d)\n",dte,posted,xtime_now,lib_gmtime());
                return TRUE;
          } 
	}
        return FALSE;
}
static int gupto,g_max;
void expire_purgelostx(void);
void expire_purgelost(void) /* Used for rebuild_piles among other things */
{
	int start = lib_gmtime();
	for (;((lib_gmtime()-start)==0) && (inpurge);) {
	 	expire_purgelostx();
	}
}

/* For each valid 'fid' call db_addlost(), when done set stuff */
static int npurged;
void expire_purgelostx(void) 
{
	static int infid;
	int r;
	static int curfid = -1;

	if (curfid<0) {
		npurged = 0;
		db_rebuild_start(); /* Delete current .idx files etc... */
		imsg("rebuild_start_done\n");
                nntp_setsleep(0); /* Start of addlost operation */
		imsg("setsleep 0\n");
		g_max = db_fidn();
		imsg("db_fidn returned\n");
	}
again:;
	if (infid) {
	 	r = db_addlost(curfid);
		gupto = curfid;
		if (r>=0) return; /* Still more to do for this one */
	}
	npurged++;
	curfid = db_nextused(curfid);
	infid = TRUE;
	if (curfid>=0) goto again; 

	infid = FALSE;
	curfid = -1;
        nntp_setsleep(3);               
        inpurge = FALSE;
        s_nhistdel = 0;
	cmd_pause(FALSE,"Rebuilding index finished");
	db_wasused_clear();
        s_nhist = 0;
	db_set_rebuild(FALSE); 
	db_set_rebuild_fast(FALSE); 
	db_set_addlost(FALSE); 
	db_set_renumber(FALSE);
	db_set_regid(FALSE);
	wmsg("Now running check_index to complete rebuild operation\n");
	db_check_index();
	wmsg("Rebuild completed\n");
	item_stopfeeds(FALSE);
        suck_expire_end(); /* Set suck loop to idle */
        return;
}
/* reindex */
void expire_reindex(void)
{
        static Sary *ls;        
        static int range;
        static int i;
        static int upto,ingroup;
        static char bf[BFSZ];    
        static int g;
	int g_start,g_end;
	static int start_time;
	start_time = lib_gmtime();

next_one:
        gupto = g;
        if (ingroup) goto cont;

        if (g==0) {
                nntp_setsleep(0); 
        }
        if (g>=group_n()) {
                g = 0;
                nntp_setsleep(3);               
                inpurge = FALSE;
		inreindex = FALSE;
                suck_expire_end(); /* Set suck loop to idle */
                item_setaddlost(FALSE);
		if (ls!=NULL) s_free(ls); ls = NULL;	
                return;
        }
	if (ls==NULL) ls = s_alloc();
        s_wild(ls,sys_gdir(g),"*.itm");  /* Returns list without paths */
        ingroup = TRUE;
        upto = 0;
        return;
cont:
        if (upto>=s_len(ls)) {
                upto = 0;
                ingroup = FALSE;
                group_setrange(g,-1);
                g++;
		goto ret;
        }       
        ncpy(bf,s_line(ls,upto),BFSZ-1);
        range = atoi(bf);                       
	imsg("reindex: Processing {%s} {%s} %d \n",group_name(g),bf,range);
/*       	db_convert3(g,range); */

	g_start = group_getstart(g);
	g_end = group_getend(g);
/*	for (i=0; i<99; i++) {
		if (db_exists(g,range+i)) {
			if (g_start > range+i) g_start = range+i;
			if (g_end < range+i) g_end = range+i;
		}
	}
	if (g_start!=group_getstart(g)) {
	 	imsg("findlost: Moving start {%s} from %d -> %d \n",group_name(g),group_getstart(g),g_start);
		group_setstart(g,g_start);
	}
	if (g_end!=group_getend(g)) {
		imsg("findlost: Moving end {%s} from %d -> %d \n",group_name(g),group_getend(g),g_end);
		group_setend(g,g_end);
	}
  */

        upto++;
ret:;
	if ((lib_gmtime()-start_time)<1) goto next_one;/*loop*/
	return;
}
void expire_findlost(void)
{
        static Sary *ls;        
        static int range;
        static int i;
	static int g_start,g_end;
        static int upto,ingroup;
        static char bf[BFSZ];    
        static int g;
	static int start_time;
	start_time = lib_gmtime();

next_one:
        gupto = g;
        if (ingroup) goto cont;
       
        if (g==0) {
                nntp_setsleep(0); 
        }
        if (g>=group_n()) {
                g = 0;
                nntp_setsleep(3);               
                inpurge = FALSE;
		infindlost = FALSE;
                suck_expire_end(); /* Set suck loop to idle */
                item_setaddlost(FALSE);
	        if (ls!=NULL) s_free(ls); ls = NULL;
                return;
        }
	if (ls==NULL) ls = s_alloc();
        s_wild(ls,sys_gdir(g),"*.idx");  /* Returns list without paths */
        ingroup = TRUE;
        upto = 0;
        return;
cont:
        if (upto>=s_len(ls)) {
                upto = 0;
                ingroup = FALSE;
                group_setrange(g,-1);
                g++;
		goto ret;
        }       
        ncpy(bf,s_line(ls,upto),BFSZ-1);
        range = atoi(bf);                       
	dmsg("findlost: Processing {%s} {%s} %d \n",group_name(g),bf,range);
	g_start = group_getstart(g);
	g_end = group_getend(g);
/*	for (i=0; i<99; i++) {
		if (db_exists(g,range+i)) {
			if (g_start > range+i) g_start = range+i;
			if (g_end < range+i) g_end = range+i;
		}
	}
	if (g_start!=group_getstart(g)) {
	 	imsg("findlost: Moving start {%s} from %d -> %d \n",group_name(g),group_getstart(g),g_start);
		group_setstart(g,g_start);
	}
	if (g_end!=group_getend(g)) {
		imsg("findlost: Moving end {%s} from %d -> %d \n",group_name(g),group_getend(g),g_end);
		group_setend(g,g_end);
	}
  */
        upto++;
ret:;
	if ((lib_gmtime()-start_time)<1) goto next_one;
	return;
}
static int xover_a;
void expire_doxover(void)
{
        static int i;
        static int g;
	static int xtot;
	static int start_time;
	static int inxov;
	int tt;
	char bf[BFSZ];
	start_time = lib_gmtime();

	if (inxov) goto xov_again;

next_one:
        gupto = g;
        if (g==0) {
		t_xover = 0;
		xtot = xov_total_get();
		xov_total_clear();
		xover_a = 0;
                nntp_setsleep(0); 
        }
	if (g<group_n()) for (;(!group_isxover(g)) && (g<group_n());) {
		g++;
		if (g>=group_n()) break;
	}
        if (g>=group_n()) {
          	xtot = xov_total_get() - xtot;
		imsg("xover: Xover data now takes %d MB changed by %dK \n",xov_total_get()/1000,xtot);
		if (!config_true(C_XOVER_MOVED)) {
			diskuse_add(xtot*1000);		
		}
		g = 0;
                nntp_setsleep(3);               
		end_time = lib_gmtime(); /* Record end of expire */ 

                strcpy(expire_end,lib_now());
                lprintf(bf,BFSZ-1,"Daily expire report, start %d%% end %d%%",used_start,used_end); 
                if (config_true(C_REPORT_OLD)) lprintf(bf,BFSZ-1,"Daily expire report"); 
                if (!get_expire_mode()) dnews_manager_report(bf);
                stats_clear();
		feed_statsclear();

                inpurge = FALSE;
		inxover = FALSE;
                suck_expire_end(); /* Set suck loop to idle */
                return;
        }
	inxov = TRUE;
xov_again:
	tt = lib_gmtime();
	if (item_expire_xover(g)) {
		inxov = FALSE;
	}
	t_xover += lib_gmtime() - tt;
	if (inxov) {
		return;
	}
	xover_a++;
        g++;
	if ((lib_gmtime()-start_time)<1) goto next_one;
	return;
}


void expire_refeed(void)
{
        static Sary *ls;        
        static int range;
        static int i;
        static int ingroup;
        static char bf[BFSZ];    
        static char upto[BFSZ];    
        static int estate;
	static h_type uptoi;
	char *h;

        if (estate==0) {
		gupto = 0;
                uptoi = 0;
                s_nold=s_nhistdel=s_nart=s_nartdel=s_nrealdel=0;
                s_time = lib_gmtime();
		end_time = 0;
                nntp_setsleep(0); /* Keep calling unless you get packets */
                strcpy(expire_started,lib_now());
                h = hist_line(0);
                if (h==NULL) {emsg("Nothing to expire\n"); goto fail;}
		refeed_line(h);
                uptoi = hist_upto();
                h = hist_next();  gupto++;
                if (h==NULL) {
                        estate = 3; goto ret;
                }
                copy_mid(upto,h);
                estate=1;
                goto ret;
        }
        if (estate==1) {
                h = hist_line(uptoi);
                dmsg("expire: restart search from {%s} \n",h);
                if (uptoi!=0) for (i=0; i<200; i++) {
                        if (h==NULL) break;
                        if (strstr(h,upto)!=NULL) goto found_restart;
                        h = hist_next();
                }
                emsg("expire: Failed to find restart position, %d {%s} \n",uptoi,upto);
                hist_line(uptoi); h = hist_next();
                goto notfound;
found_restart:  dmsg("expire: refeed Found restart position {%s} \n",h);
notfound:       if (h!=NULL) dmsg("expire: Restarting at {%s} {%s} \n",upto,h);
                if (h==NULL) {
                        emsg("expire: Failed to find restart position {%s} \n",upto);
                         goto fail;
                }
                for (i=0;h!=NULL;i++) {
                        refeed_line(h); uptoi = hist_upto();
                        h = hist_next(); gupto++;
                        if (h==NULL) break;
                        if (i>100) {
                                copy_mid(upto,h);
                                goto ret; 
                        }
                }
                estate=3;
        }       
        if (estate==3) {
fail:           suck_expire_end(); /* Set suck loop to idle */
                nntp_setsleep(3); /* Switch back to sleeping mode */
                strcpy(expire_end,lib_now());
    		inrefeed = FALSE;
                estate = 0;
                goto ret;
        }
ret: return;

}

void expire_purgelinks(void)
{
        static int i;
        static int upto,ingroup;
        static char bf[BFSZ];    
	static int g;
      	int upto_end;

        gupto = g;
        dmsg("expire: purgelinks g=%d upto=%d ingroup=%d \n",g,upto,ingroup);
        if (ingroup) goto cont;

        if (g==0) {
                nntp_setsleep(0); /* Switch back to sleeping mode */
        }
next:;
        if (g>=group_n()) {
                g = 0;
                nntp_setsleep(3);               
                inpurge = FALSE;
		inlinks = FALSE;
                s_nhistdel = 0;
                s_nhist = 0;
                suck_expire_end(); /* Set suck loop to idle */
                return;
        }
        ingroup = TRUE;
        upto = group_getfrom(g);
	if (upto>group_getend(g)) {
	 	g++;
		goto next;
	}
        group_setrange(g,0);
cont:
	
	upto_end = (upto / 100)*100 + 99;
	if (upto_end>group_getend(g)) upto_end = group_getend(g);
/* 	db_readtest(g,upto,upto_end);*/
	upto = upto_end + 1;
	if (upto>group_getend(g)) {ingroup = FALSE; g++;}
}

void expire_upgradex(void);
void expire_upgrade(void)
{
 	int start = lib_gmtime();
	for (;(lib_gmtime()-start<3) && inupgrade;) {
		expire_upgradex();	
	}
}
int db_reindex(void);
void expire_resort(void)
{
	
 	int start = lib_gmtime();
	int xdelay;
        static int first=TRUE;
        int r; /*= db_reindex();*/
        
        if (first) {
            tell_master("stop_write_dbi_index");
            db_flush2();
        }

        r = db_reindex();
        if (first) {
            t_resort = 0;
            first = FALSE;
        }
        for (;(lib_gmtime()-start<3) && (r>=0);) {
            r = db_reindex();   
        }
	xdelay = lib_gmtime() - start;
	t_resort += xdelay;
	if (xdelay > 5) imsg("expire_resort: Delay %d seconds\n",xdelay);
	if (r<0) {
		first = TRUE;
	 	inresort = FALSE;
                nntp_setsleep(3);               
                suck_expire_end(); /* Set suck loop to idle */
		if (!donetoday) suck_doxover();
                donetoday = TRUE;
                /* brian */
                if (get_expire_mode()) {
                    db_flush2();
                    db_clear_cache("expire3");
                    db_write_last();
                    tell_master("reread_dbi_index");
                }
                imsg("Resort Index Finished %d\n",r);
	}
}
void expire_upgradex(void)
{
        static int islink;
        static int upto,ingroup,start,end;
	char bf[BFSZ];
        static int g;
	FILE *f;
        
        gupto = g;
        dmsg("expire: upgrade g=%d upto=%d ingroup=%d \n",g,upto,ingroup);
        if (ingroup) goto cont;

        if (g==0) {
		cmd_pause(TRUE,"Upgrading database to v4 format");
		hist_delete();
                nntp_setsleep(0);
		f = fopen(UPGRADE_DAT,"r"); 
		if (f!=NULL) {
			fgets(bf,BFSZ-10,f);
			g = atoi(bf)+1; /* Skip one just incase */
		 	emsg("Found %s restarting upgrade from %d %s, previous group may not be completed\n",UPGRADE_DAT,g,group_name(g));
			fclose(f);
		}
        }
        if (g>=group_n()) {
		imsgf("Upgrade completing...\n");
                g = 0;
                nntp_setsleep(3);               
                inupgrade = FALSE;
                suck_expire_end(); /* Set suck loop to idle */
 		remove(UPGRADE_DAT);
		db_check_index();
		db_correct_used();
		cmd_pause(FALSE,"Upgrade finished");
		wmsg("Upgrade finished, sorting new index file... (this will take 10 minutes)");
		suck_resort();
                return;
        }
	remove(UPGRADE_DAT);
	f = fopen(UPGRADE_DAT,"w"); 
	if (f==NULL) filemsg("Can't open upgrade.dat\n");
	else {
	 	fprintf(f,"%d\n",g);
		fclose(f);
	}
        start = group_getstart(g);
        end = group_getend(g);
        upto = start;
	if (end-start > 100000) {
		emsg("Range of items in group %s is wrong, adjusting during rebuild\n",group_name(g));
		upto = end-100000;
	}
        ingroup = TRUE;
        return;
cont:
        if ((upto/100)>(end/100)) {  
                upto = 0;
                ingroup = FALSE;
                g++;
                return;
        }       
        if (start<end) {
	        imsg("expire: Upgrading group file {%s}  %d \n",group_name(g),upto);
	       	db_convert(g,upto);
	}
        upto += 100;
}
void expire_setpurge(void)
{
        inpurge = TRUE;
}
void expire_setrefeed(int f)
{
	refeed_num = f;
        inrefeed = TRUE;
}
void expire_setlinks(void)
{
        inlinks = TRUE;
}
void expire_setdoxover(void)
{
        inxover = TRUE;
}
void expire_setupgrade(void)
{
        inupgrade = TRUE;
}
void expire_setreindex(void)
{
        inreindex = TRUE;
}
void expire_setresort(void)
{
        inresort = TRUE;
}
void expire_setfindlost(void)
{
        infindlost = TRUE;
}
int expire_upgrading(void)
{
	return inupgrade;
}
char *expire_getmode(void)
{
        static char bf[BFSZ];
        strcpy(bf,"");
        if (inrefeed) lprintf(bf,BFSZ-1,"(Refeed %d/%d) ",gupto,s_nhist);
        if (inpurge) lprintf(bf,BFSZ-1,"(Rebuild_index %d/%d) ",npurged,g_max);
        if (inlinks) lprintf(bf,BFSZ-1,"(Purge %d/%d) ",gupto,group_n());
        if (inxover) lprintf(bf,BFSZ-1,"(Expire Xover %d/%d) ",xover_a,group_nxover());
        if (inupgrade) lprintf(bf,BFSZ-1,"(Upgrade %d/%d) ",gupto,group_n());
        if (inresort) lprintf(bf,BFSZ-1,"(Resort Index)");
        if (inreindex) lprintf(bf,BFSZ-1,"(Reindex %d/%d) ",gupto,group_n());
        if (infindlost) lprintf(bf,BFSZ-1,"(Findlost %d/%d) ",gupto,group_n());
        return bf;
}

int refeed_line(char *h)
{
	char mid[BFSZ];
	char bf[BFSZ];
	char agroup[BFSZ];
	int num;
	int i;
	char *s;
	static char *groups[100];
	

       	ncpy(mid,h,BFSZ-1);
	dmsg("expire_line: %s\n",mid);
       	s = strchr(mid,'\t'); if (s!=NULL) *s = 0;
        s = hist_groupfirst(h);
	if (s==NULL) return FALSE;
	ncpy(agroup,s,BFSZ-50);
	num = hist_groupnextnum();
        s = hist_groupfirst(h);	
	for (i=0;(s!=NULL) && (i<20);i++) {
		if (groups[i]==NULL) groups[i] = mymalloc(100);
		ncpy(groups[i],s,99);
		s = hist_groupnext();
	}
	groups[i] = NULL;


	if (feed_test(refeed_num,mid,"",groups,NULL,"",1000,NULL)) {
	     	lprintf(bf,BFSZ-1,"%s:%d",agroup,num);
                feed_send(refeed_num,mid,bf,0);
		return TRUE;
	}
	return FALSE;
}
static char *stat_names[20];
static int stat_total[20];
static int nstat;
int ex_stat_find(char *name);
void ex_clear(void)
{
	int i;
	for (i=0; i<nstat; i++) stat_total[i] = 0;
}
int ex_stat_find(char *name)
{
	int i;
	for (i=0; i<nstat; i++) {
		if (strcmp(stat_names[i],name)==0) {
			return i;
		}
	}
	if (nstat>18) return 0;
	stat_names[i] = mystrdup(name,SD_EXPIRE);
	return nstat++;	
}
void ex_start(char *name)
{
	int i = ex_stat_find(name);
	stat_total[i] -= lib_gmtime();	
}
void ex_stat_add(char *name, int n)
{
	int i = ex_stat_find(name);
	stat_total[i] += n;
}
void ex_end(char *name)
{
	int i = ex_stat_find(name);
	stat_total[i] = lib_gmtime() + stat_total[i];
}
char *ex_stat_name(int i)
{
	return stat_names[i];
}
int ex_stat_total(int i)
{
	return stat_total[i];
}
int ex_stat_n(void)
{
	return nstat;
}




#define SIZE (199*4)
typedef struct MLIST {char *data; int left; struct MLIST *next;} MList;
void *getmem_malloc(void **xtop, int size)
{
	MList **top = (MList **) xtop;
	MList *h = *top;
	char *p;
	if (h==NULL) {
addone:		h = (MList *) malloc(sizeof(MList));
		if (h==NULL) return NULL;
		h->data = (char *) malloc(SIZE);
		if (h->data==NULL) return NULL;
		h->left = SIZE;
		h->next = *top;
		*top = h;
	}
	if (size>SIZE) return NULL;
	if (h->left < size) goto addone;
	p = h->data + SIZE - h->left;
	h->left -= size;
	return (void *) p;
}
void getmem_free(void **xtop)
{
	MList **top = (MList **) xtop;
	MList *h,*next;

	for (h=*top; h!=NULL; h=next) {
		next = h->next;
		free(h->data);
		free(h);
	}
	*top = NULL;
}

