/* Copyright (C) 1995 NetWin World, New Zealand, All rights reserved. */
#include <stdlib.h>
#include <stdio.h>
#include <ctype.h>
#include <string.h>
#include <assert.h>
#include <errno.h>
#include "dnews.h"
#include "emsg.h"
#include "sary.h"
#include "str.h"
#include "chan.h"
#include "lib.h"
#include "feed.h"
#include "suck.h"
#include "expire.h"
#define IN_GROUP TRUE
#include "group.h"
#include "item.h"
#include "xhash.h"
void lib_msleep(int x);

/* Routines to deal with the list of groups */
/* name end start moderated cached feedend */
#ifndef unix
#define GROUPID TRUE
#endif

#ifdef NETWARE
#define GROUPID TRUE
#endif 

/* name to from mod cache feedupto cachdays */
/* alt.drugs 20 10 y c 1000 23 */
/*
        mod =  'y' 'm' 'd' 'g' 'e'     Yes can post, Moderated, Deleted, gone, deleted-moderated
*/
typedef struct {
        char *name; 
        int from,to,life,catchup,range,groupid,xxx; /* xxx to make groupid safer */ 
        int fend[4];
        int nread,size,rsize;
        short lastmsg;
        char mod,cache; 
        char *title;
        char xover;
        char changed;
        char binsucked;
        char mrej;
	char isxsort;
} GRec;

void glog(char * arg_list, ...);
int xover_days(void);

static int names_changed;
int group_nameok(char *s);
static GRec **groups;
void group_master(int x);
int group_set(GRec *r, char *s);
static int ngroups2;
static int dummy1;
static int ngroups;
static int dummy3;
int ghash_find(char *gname);
int ghash_add(char *s,int g);
void db_init(void);
static int is_changed;

#define xisspace(x) ((x==' ') || (x=='\t'))
#define DNEWS_ACTIVE file_work("active.dat")
#define DNEWS_MERGE file_work("active.add")
#define DNEWS_ACTIVE_TMP file_work("active.new")
static int group_changed;
static int isver2 = FALSE;
static int uptime;
static int gconvert;
static int read_v2;
static int read_v3;
static int read_v4;
extern int xmax_groups;
static int greadok=TRUE;
int group_bounds(int x);
static int changed_date;


static int *inupdate;
static int *gid_to_g;
static char *gid_isused;
static int *gtimes_time;
static int *gtimes_group;
static int *g_p;
static double *gage_avg;
static int *gage_n,*gage_h;
#undef mymalloc
#undef myfree
#define mymalloc(x) mymalloc2(x,M_GROUP)
#define myfree(x) myfree2(x,M_GROUP)

int group_malloc_groups(void)
{
        unsigned long x;
        inupdate = (int *) mymalloc(max_groups *sizeof(inupdate[0]));
        gid_to_g = (int *) mymalloc(xmax_groups * sizeof(int));
        gid_isused = (char *) mymalloc(xmax_groups);
        gtimes_time = (int *) mymalloc(max_groups * sizeof(int));
        gtimes_group = (int *) mymalloc(max_groups * sizeof(int));
        g_p = (int *) mymalloc(max_groups * sizeof(int));
        x = (unsigned long) mymalloc((max_groups+20) * sizeof(double));
        if ((x&1)==1) x ++;
        if ((x&2)==2) x += 2;
        if ((x&4)==4) x += 4;
        gage_avg = (double *) x;
        gage_n = (int *) mymalloc(max_groups * sizeof(int));
        gage_h = (int *) mymalloc(max_groups * sizeof(int));
        
        return TRUE;
}


void group_read(void)
{
        int i,cnt;
        char inbuff[BFSZ];
        int g;
        FILE *f;
        static int done;
        if (done) return;
        greadok = FALSE;        
        done = TRUE;
        lib_tzinit();
        uptime = lib_gmtime();

        for (i=0; i<xmax_groups; i++) gid_to_g[i] = -1;
        read_v2 = FALSE;
        if (groups==NULL) groups = (GRec **) mymalloc(max_groups*sizeof(GRec *));
        memset(groups,0,max_groups*sizeof(GRec *));

        read_v2 = FALSE; read_v3 = FALSE; read_v4 = FALSE;
        f = xfopen(DNEWS_ACTIVE,"r");
        if (f==NULL) {
                if (config_true(C_ACTIVE_REQUIRE)) {
                        filemsg("Couldn't open %s\n",DNEWS_ACTIVE);
                        emsg("Exiting because active.dat is not available\n");
                        dnews_ex("active.dat could not be read");
                }
                if (errno==ENOENT) {
                        filemsg("Couldn't open %s\n",DNEWS_ACTIVE);
                        f = xfopen(DNEWS_ACTIVE_TMP,"r");
                        if (f!=NULL) {
                                emsg("Found (%s) will use that instead, hope this is ok\n",DNEWS_ACTIVE_TMP);
                                goto noskip;
                        }
                }
                filemsg("Couldn't open %s\n",DNEWS_ACTIVE);
                goto skip_read;
        }
noskip:
        for (cnt=0;!feof(f);cnt++) {
          if (fgets(inbuff,BFSZ-1,f)!=NULL) {
#ifdef NETWARE
                if ((100*(cnt/100)) == (100.0* (double) (cnt/100.0))) 
                        lib_doevents();
                        /* Allow low priority jobs to run? */ 
#endif
#ifdef Mac
                if ((100*(cnt/100)) == (100.0* (double) (cnt/100.0))) 
                        lib_doevents();
                        /* Allow low priority jobs to run? */ 
#endif
                i = strlen(inbuff);
                if (i>0) {
                 if (inbuff[i-1]=='\n') inbuff[i-1] = 0;

                 if (cnt==0) if (strncmp(inbuff,"DNEWS V2.0",10)==0) {
                                isver2 = TRUE;                  
                                read_v2 = TRUE;
                 } 
                 if (cnt==0) if (strncmp(inbuff,"DNEWS V3.0",10)==0) {
                                isver2 = TRUE;                  
                                read_v3 = TRUE;
                 } 
                 if (cnt==0) if (strncmp(inbuff,"DNEWS V4.0",10)==0) {
                                isver2 = TRUE;                  
                                read_v4 = TRUE;
                 } 
                 dmsg("groupadd: %s\n",inbuff);
                 if ((!isver2) || (cnt>0))  {
                        g = group_add(inbuff);
                        if (gconvert) if (g>=0) {
                                groups[g]->from = groups[g]->to + 1;
                        }               
                 }
                }
          } else {
                  if (!feof(f)) {
                        filemsg("Error reading active.dat file, dnews cannot continue until this is fixed\n");  
                        dnews_ex("cannot_read_active");
                  }
          }
        }
        xfclose(f); f = NULL;
skip_read:
        greadok = TRUE;
        imsgf("Read %d groups of %d allowed (max_groups)\n",ngroups,max_groups);
        group_times_load();
        empty_read();
        if (read_v2 || read_v3) {
                emsg("Warning: Attempting to upgrade database to V4 format, you cannot switch back to dnews 2.x or 3.x\n");
                emsg("The upgrade will take approx 20-90 minutes per GIG of news\n");
                emsg("News will slowly become available during this period but posting will not be allowed\n");
                suck_upgrade();
        } else if (lib_exists(UPGRADE_DAT)) {
                emsg("Found upgrade file %s, system must have crashed during upgrade?\n",UPGRADE_DAT);
                emsg("Will attempt to restart upgrade process\n");
                suck_upgrade();
        }
	is_changed = FALSE;
        group_read_changed();
        if (!expire_ispile()) gage_init();
}
int group_correct(char *gline);
void active_catchup(void)
{
        /* From the current point, read 5000 active.dat entries */
        int i;
        static int upto;
        static int last_time;
        int mod;
	static int old_mod;
        FILE *f;
        char bf[BFSZ];

        if (upto==0) {
                if ((lib_gmtime()-last_time)<60) return;
                last_time = lib_gmtime();
        }

        
        dmsg("group: Scanning active.dat for changes upto %d\n",upto);
        f = fopen(DNEWS_ACTIVE_TMP,"r");
        if (f!=NULL) {
                imsg("group: Not scanning active.dat as it is being re-written\n");
                fclosep(f);
                return;
        }

	if (upto==0) {
		mod = lib_file_mod(DNEWS_ACTIVE);
		if (mod==old_mod) {
			imsg("group: Not scanning active.dat because it hasn't changed\n");
			return ;
		}
		old_mod = mod;
	}
/*        if (upto==0) if (mod<changed_date) {
                imsg("group: Not scanning active.dat because active.pat is newer\n");   
                return ;
        }

 */       
        f = fopen(DNEWS_ACTIVE,"r");
        if (f==NULL) {
                filemsg("Could not read %s\n",DNEWS_ACTIVE);
                return;
        }

        /* Open and seek to current point */
        if (upto!=0) {
                upto -= 100;
                if (upto<0) upto = 0;
                fseek(f,upto,SEEK_SET);
        } else fgets(bf,BFSZ-10,f); /* Swallow DNEWS v4.0 header */ 
        for (i=0;(!feof(f)) && (i<2000);) {
                if (fgets(bf,BFSZ-10,f)==NULL) break;
		if (strncmp(bf,"DNEWS V4",8)==0) continue;
                group_correct(bf);
        }
        if (feof(f)) upto = 0;
        else upto = ftell(f);
        fclosep(f);
        dmsg("group: Scanning completed\n");
}
void db_clear_group(int g);
int group_correct(char *gline)
{
/*
Find group, if it doesnt exist call add,
If it does exist, correct  from,to,range,cache,mod,gid,xover

  


microsoft.public.de.german.windowsxp.sonstiges 33507 14244 y n fend=0

life 0 range 18067 fend 0 0 0 gid=47187 xover=16 lastmsg=7

*/
        char bf[BFSZ];
	char mod[100];
        int isnew=FALSE;
        char *s;
        int g;
        GRec *r;
        ncpy(bf,gline,BFSZ-1);
        s = strtok(bf," \t\n\r"); if (s==NULL) return FALSE;
        g = group_find(s);
        if (g<0) {
		imsg("group scan %.60s %.100s\n",s,gline);
                g = group_add(gline);
		/* If readonly slave 0 then tell slaves about it */
                if (g>=0) if (is_readmaster()) {
			r = groups[g];
			mod[0] = r->mod;
			mod[1] = 0;
			slave_group_new(group_name(g),"master",mod,"",group_getgroupid(g));
		}
                return TRUE;
        } 
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r = groups[g];
        r->to = atoi(s);
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->from = atoi(s);
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->mod = *s;
        if (*s==0) r->mod = 'y'; /* Default */
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->cache = *s;
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->fend[0] = atoi(s);
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->life = atoi(s);
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        if (atoi(s)!=r->range) {
                imsg("Found %d new items in %s using index_changes\n",atoi(s) - r->range,group_name(g));
		if (!config_true(C_INDEX_CHANGES)) {	
	                db_clear_group(g);
	                isnew = TRUE;
		}
        }
        r->range = atoi(s); /* Default */
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->fend[1] = atoi(s);
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->fend[2] = atoi(s);
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->fend[3] = atoi(s);
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->groupid = atoi(s); 
        if ((r->groupid<0) || (r->groupid>xmax_groups) ) r->groupid = 0;
        gid_to_g[r->groupid] = g;

	if (!config_true(C_INDEX_CHANGES)) {	
	        if (isnew) group_master(g);
	}
        
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->xover = atoi(s); /* 1 = xover cmd yesterday */
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->lastmsg = atoi(s); /* 1 = 1day ago .*/
        return TRUE;
}

int group_add(char *s)
{
        GRec *rec;
        static GRec *nextrec;
        
        if (ngroups+100> max_groups) {
                emsg("Too many groups in active.dat, max %d, increase max_groups in dnews.conf \n",max_groups-200); 
                return -1;
        }
	is_changed = TRUE;
        group_changed = TRUE;
        if (nextrec==NULL) nextrec = mykeep(sizeof(*rec));
        rec = nextrec;
        if (!group_set(rec,s)) return -1;
        ghash_add(rec->name,ngroups); 
        rec->groupid = getagid(ngroups,rec->groupid,s);
	rec->isxsort = FALSE;
        if (ngroups<0) {
                emsg("Memory corruption error, ngroups=%d\n",ngroups);
                dnews_exit("Memory corrupt","group.c",9999);
        }
        nextrec = NULL;
        groups[ngroups++] = rec;
        if (ngroups>ngroups2) ngroups2 = ngroups;
        return ngroups-1;
}
int group_new_ok(char *s)
{
	char *w;
	w = config_get(C_GROUP_NEW);
	if (strlen(w)==0) return FALSE;
	if (group_match(w,s)) return TRUE;
	return FALSE;
}
int group_new(char *s, char *user, char *mod, char *title)
{
        int r;

        group_changed = TRUE;
        
        r = group_find(s);
        if (r<0) {
		if (!group_new_ok(s)) if (!group_wanted(s)) {
                        imsg("Cannot create (%s) due to ME feed\n",s);
                        return -1;
                }
                r = group_add(s);
		is_changed = TRUE;
                if (r>=0) {
                        group_times(r,s,user,lib_gmtime());
			feed_addgroup(r);
                }
        }
        if (r>=0) {
                if (strlen(title)>0) group_settitle(r,title);
                group_setmod(r,mod);
                slave_group_new(s,user,mod,title,group_getgroupid(r));
                gage_setpile(r);
		if (strcmp(groups[r]->name,s)!=0) {
			imsg("Updating case of news group name (%s)->(%s)\n",groups[r]->name,s);
			groups[r]->name = mystrkeep(s);
		}
        }
        return r;
}
static FILE *f_times;
static int ngtimes,got_gtimes;
void group_fixtimes(int age)
{
        FILE *f;
        int i;

        if (age>0) age = -age;
        if (is_readonly()) return;
        ngtimes = 0;
        got_gtimes = FALSE;
        group_times_flush();
        f = fopen_log(DNEWS_ACTIVE_TIMES,"w");
        if (f==NULL) {filemsg("Can't reset %s\n",DNEWS_ACTIVE_TIMES); return;}
        xfclose(f); f = NULL;
        for (i=0; i<group_n(); i++) {
                group_times(i,group_name(i),"reset",lib_gmtime()+age);
        }
        group_times_flush(); /* So the file can be read */
}
void group_times(int g, char *gname, char *user, int t)
{
        group_times_add(g, t);
        if (is_readonly()) return;
        if (f_times==NULL) f_times = fopen_log(DNEWS_ACTIVE_TIMES,"a");
        if (f_times==NULL) {filemsg("Unable to open %s \n",DNEWS_ACTIVE_TIMES); return;}      
        fprintf(f_times,"%s %d %s\n",gname,t,user);
}
int group_times_add(int g, int t)
{
        static int done_error;
        if ((ngtimes+200)>max_groups) {
                if (!done_error) emsg("active.times is too big, please use command: tellnews fixtimes -100\n");
                done_error = TRUE;
        } else {
                gtimes_time[ngtimes] = t;
                gtimes_group[ngtimes++] = g;
        }
        return TRUE;
}
void group_times_load(void)
{
        char bf[BFSZ];
        char gname[BFSZ];
        char rest[BFSZ];
        char *s;
        FILE *f;
        int g;

        if (got_gtimes) return;
        got_gtimes = TRUE;
        imsg("Reading Group times files\n"); 
        f = fopen_log(DNEWS_ACTIVE_TIMES,"r");
        if (f==NULL) {filemsg("Can't read %s\n",DNEWS_ACTIVE_TIMES); return;}
        for (;!feof(f);) {
          if (fgets(bf,BFSZ-10,f)!=NULL) {
                s = strtok(bf," \t\n");        
                if (s==NULL) continue;
                g = -1;
                ncpy(gname,s,BFSZ-1);
                s = strtok(NULL," \t\n");
                if (s==NULL) continue;
                ncpy(rest,s,BFSZ-1);                
                g = group_find(gname);
                if ((g>=0)) {
                        group_times_add(g,atoi(rest));
                }
          }
        }
        xfclose(f); f = NULL;
        if ((ngtimes+1200)>max_groups) {
                wmsg("Removing %s as it is now too big (not serious)\n",DNEWS_ACTIVE_TIMES);
                remove(DNEWS_ACTIVE_TIMES);
        }
        imsg("Group times read successfully\n"); 
}
int group_times_n(void)
{
        return ngtimes;
}
int group_times_g(int n)
{
        return gtimes_group[n];
}
int group_times_time(int n)
{
        return gtimes_time[n];
}
int group_times_after(int t)
{
    char tmpstr[BFSZ];
        int i;
        time_t ut = (time_t) t;
        dmsg("group: group_times %d %d %d (%s) \n",t,gtimes_time[0],ngtimes,safe_ctime(&ut,tmpstr));
        for (i=0; i<ngtimes; i++) if (gtimes_time[i]>t) return i;
        return ngtimes;
}
void group_times_flush(void)
{
        if (f_times!=NULL) xfclose(f_times);
        f_times = NULL;
}
char *group_name_gid(int gid)
{
        int g = gid_to_group(gid);
        if (g<0) return "deleted_group";
        return group_name(g);
}
void group_null_name(int chan)
{
	int g;
	char bf[BFSZ];
	int ngood,nbad;
	ngood=nbad=0;
	for (g=0; g<ngroups; g++) {
	        if (groups[g]==NULL) continue;
		if (groups[g]->name==NULL) {
			lprintf(bf,BFSZ-1,"Null one [%d]\r\n",g);
			chan_write_s(chan,bf);
			nbad++;
		} else ngood++;
	}
	lprintf(bf,BFSZ-1,"Totals %d/%d max %d gidmax %d\n",nbad,ngood+nbad,max_groups,xmax_groups);
	chan_write_s(chan,bf);
}

int group_set(GRec *r, char *s)
{
        char bf[300];

        group_changed = TRUE;
        ncpy(bf,s,290); s = bf;
        memset(r,0,sizeof(*r));
        gconvert = TRUE;
        r->to = 2;      /* Defaults leave room for fake article*/
        r->from = 3;
        r->mrej = 0;
        r->binsucked = FALSE;
        r->changed = FALSE;
        r->mod = 'y'; /* Default */
        r->cache = 'n'; /* Default */
        r->range = 0; /* Default */
        r->groupid = -1;
        r->lastmsg = 0;
        r->xover = 100;
        s = strtok(s," \t\n"); if (s==NULL) return FALSE;
        r->name = NULL;
        r->title = NULL;
        if (!group_nameok(s)) {
                wmsg("group: rejecting, illegal character in newsgroup name {%s} \n",s);
                return FALSE;
        }
        if (group_find(s)>=0) {
                emsg("group: removing duplicate entry (%s)\n",s);
                return FALSE;
        }
        r->name = mystrkeep(s);
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->to = atoi(s);
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->from = atoi(s);
                if (r->from<1 || r->to<=1) {
                        imsg("Patched from counter for group {%s}\n",r->name);
                        r->from = 2; 
                        if (r->to<=1) {
                                r->from = 2; 
                                r->to = 1; 
                        }
                }
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->mod = *s;
        if (*s==0) r->mod = 'y'; /* Default */
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->cache = *s;
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->fend[0] = atoi(s);
        gconvert = FALSE;
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->life = atoi(s);
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->range = atoi(s); /* Default */
#ifdef GROUPID        
        if (read_v2) {
                s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
                r->groupid = atoi(s); /* groupid (used by netware only) */
        }
#endif
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->fend[1] = atoi(s);
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->fend[2] = atoi(s);
        s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
        r->fend[3] = atoi(s);
        if (!read_v2) {
                s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
                r->groupid = atoi(s); 
                s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
                r->xover = atoi(s); /* 1 = xover cmd yesterday */
                s = strtok(NULL," \t\n"); if (s==NULL) return TRUE;
                r->lastmsg = atoi(s); /* 1 = 1day ago .*/
        }

        return TRUE;
}
int gid_to_group(int gid)
{
        if (gid<0 || gid>=xmax_groups) {emsg("Corruption in GID's %d \n",gid); return -1;}
        return gid_to_g[gid];
}
char *gid_find_group(int gid)
{
        int g;
        if (gid<0 || gid>=xmax_groups) {emsg("Corruption in GID's in db %d\n",gid); return "";}
        g = gid_to_g[gid];
        if (g<0) return NULL;
        return group_name(g);   
}
int gid_find_g(char *s)
{
        int g = group_find(s);
        return group_getgroupid(g);
}
int getagid(int g, int x, char *gname)
{
        if (x>=xmax_groups) {
                emsg("FATAL, group %d, gid=%d faulty, correcting\n",g,x);
                x = 0;
        }
        x = next_groupid(x);
        if ((x<0) || (x>xmax_groups)) {
                emsg("FATAL: Too many groups and GID values ran out %d, see rebuid_gids\n",x);
                dnews_ex("getagid failed");
        }
        if (gid_to_g[x]!= -1) {
                emsg("Active.dat corrupt, duplicate GID value line %d and %d both use gid %d, adding (%s) other (%s)\n",g,gid_to_g[x],x,gname,group_name(gid_to_g[x]));
                if (!is_readonly()) {
                 x = next_groupid(0);
                 emsg("changing line %d to use gid %d\n",g,x);
                 emsg("This could result in a few cross posts disappearing or being mixed up\n");
                }
        }
        if ((x<0) || (x>xmax_groups)) {
                emsg("FATAL: Too many groups and GID values ran out %d, see rebuid_gids\n",x);
                return x;
        }
        gid_to_g[x] = g;
        gid_isused[x] = TRUE;
        return x;
}
static int gbigest;
static int set_gid=FALSE;
static int set_gid_val;
void group_slavegid(int g)
{
        set_gid = TRUE;
        set_gid_val = g;
}
int free_gid(void);
int next_groupid( int x)
{
        if (set_gid) {
                set_gid = FALSE;
                return set_gid_val;
        }
        if ((x<=0) || (x>=xmax_groups)) {
                if (gbigest>max_groups) {
                        return free_gid();
                }
                x = ++gbigest;
                if (x>gbigest) gbigest = x;
                return x;
        }
        if (x>gbigest) gbigest = x;
        return x;               
}
void db_scan_gids(char gid_isused[],int maxgid);
int free_gid(void)
{
        /* Search index for free gid's */
        static int done_init;
        static int f=1;
        if (!done_init) {
                db_scan_gids(gid_isused,xmax_groups);
                done_init = TRUE;
        }
        /* Now search for free one */
        for (;f<xmax_groups;f++) {
                if (gid_isused[f]==0) {
                        imsg("group: reusing free gid %d\n",f);
                        return f;
                }
        }
        /**/
        dnews_ex("No GIDS free, you must delete some groups and/or expire the items within deleted groups\n");
        /**/
        return 0;
}

int group_lastgid(void)
{
        return gbigest;
}
int group_writex(int isbackup);
void group_monthly(void)
{
        group_writex(2);
}
int group_write_backup(void)
{
        return group_writex(1);
}
int group_write(void)
{
        db_flush();
        empty_write();
	is_changed = FALSE;
        return group_writex(0);
}       
typedef struct GINFO {int type; int gid; int from; int to; int range; char mod,cache,fill1,fill2; int fill3[3];} Ginfo; 
#define FILE_CHANGED file_work("active.pat")
#define CHANGED_TYPE 0x9238
int group_write_changed(void)
{
        int n;
        int i;
	int nwrote = 0;
        GRec *r;
        Ginfo g;
        FILE *f;
	int start,delay;
        db_flush();

        g.type = CHANGED_TYPE;
        if (is_readonly()) return TRUE;
        if (is_slave()) return TRUE;
	
	if (is_changed) {
		imsg("Groups created so writing new active.dat\n");
		group_write();
		return TRUE;
	}
	start = lib_gmtime();
	remove(FILE_CHANGED);
        f = fopen(FILE_CHANGED,"wb");
        if (f==NULL) {
                filemsg("Could not write to changes file\n");
                return FALSE;
        }
        for (i=0; i<ngroups; i++) {
                r = groups[i];
                if (r==NULL) continue;
                if (r->changed) {
			nwrote++;
                        g.gid = r->groupid;
                        g.from = r->from;
                        g.to = r->to;
                        g.range = r->range;
                        g.mod = r->mod;
                        g.cache = r->cache;
                        n = fwrite(&g,sizeof(g),1,f);
                        if (n!=1) {
                                filemsg("Could not write to changes file %s\n",FILE_CHANGED);
                                break;
                        }
                }
        }
        fclosep(f);
	delay = lib_gmtime()-start;
	if (delay>2) imsg("Time writing active.dat changes %d seconds,  %d written\n",delay,nwrote);
        return TRUE;
}
/*
        If date of file has changed, read it.
*/
int group_read_changed(void)
{
        int n;
        GRec *r;
        int grp;
        Ginfo g;
        FILE *f;
        int mod;
        int nread;
        int isnew;

        mod = lib_file_mod(FILE_CHANGED);
        if ((mod-changed_date)<=0) return TRUE;
        changed_date = mod;
        f = fopen(FILE_CHANGED,"rb");
        if (f==NULL) {
                filemsg("Could not read %s\n",FILE_CHANGED);
                return FALSE;
        }
        nread = 0;
        for (;!feof(f);) {
                isnew = FALSE;
                nread++;
                n = fread(&g,sizeof(g),1,f);
                if (n<=0) break;
                if (g.type!=CHANGED_TYPE) continue;
                grp = gid_to_group(g.gid);
                if (grp<0) continue;
                r = groups[grp];
                if (r->range!=g.range) {
			if (!config_true(C_INDEX_CHANGES)) {	
	                        imsg("Found %d new items in %s\n",g.range - r->range,group_name(grp));
		                 db_clear_group(grp);
	                         isnew = TRUE;
			} else {
	                        imsg("Found %d new items in %s - not clearing cache\n",g.range - r->range,group_name(grp));
			}
                }
                r->range = g.range;
                r->from = g.from;
                r->to = g.to;
                r->mod = g.mod;
                r->cache = g.cache;
		if (!config_true(C_INDEX_CHANGES)) {	
	                if (isnew) group_master(grp);
		}
        }
        fclosep(f);
        imsg("Read %d records from active.pat file\n",nread);
        return TRUE;
}
#ifdef WIN32
void sleep(int x);
#endif
int group_writex(int isbackup)
{
        FILE *fout;
        GRec *r;
        int i;
        int worsti,worst;
        int nfail=0;
        char bf[400];
        char tmpname[BFSZ];
        int status;

        worsti = -1; worst = 0;
        if (is_readonly()) return TRUE;
        if (ngroups==0) {
                imsg("group: Can't write what we haven't even read in\n");
                return TRUE;
        }
        if (!greadok) {
                emsg("group: Group read failed so not writing faulty list \n"); 
                emsg("group: You may need to edit active.dat by hand to fix the faulty lines\n");       
                emsg("group: Use 'debug groupadd:' to see the faulty line on start\n"); 
                return TRUE;
        }
        if (ngroups<ngroups2) {
                emsg("FATAL: ngroups has decreased %d %d \n",ngroups,ngroups2);
                emsg("FATAL: please send dnews.log to support@netwinsite.com\n");
                dnews_ex("ngroups faulty");
        }
        strcpy(tmpname,DNEWS_ACTIVE_TMP);
        if (isbackup==1) strcpy(tmpname,DNEWS_ACTIVE_BACKUP);
        if (isbackup==2) strcpy(tmpname,DNEWS_ACTIVE_BACKUP2);
        imsg("group: Group Write %s\n",tmpname);
        fout = fopen_log(tmpname,"w");
        if (fout==NULL) {
                filemsg("Unable to OPEN %s trying again\n",tmpname);
                sleep(1);
                fout = fopen_log(tmpname,"w");
                if (fout!=NULL) goto xcontinue;
                if (isbackup==0) dnews_ex("Dnews cannot continue if it can't write to its active.dat file\n");
                return TRUE;
        }
xcontinue:
        fprintf(fout,"DNEWS V4.0\n"); 
        for (i=0; i<ngroups; i++) {
                /* if (group_inupdate(i)) group_setcached(i,TRUE);*/
                group_setupdate(i,FALSE);
                r = groups[i];
                if (r==NULL) continue;
                if (r->name==NULL) continue;
                if (isbackup==0) r->changed = FALSE;
                if (r->mod!='g') { 
                  status = fprintf(fout,"%s %d %d %c %c %d %d %d %d %d %d %d %d %d\n",
                                   r->name,r->to,r->from,r->mod,r->cache,
                                   r->fend[0],r->life,r->range,
                                   r->fend[1],r->fend[2],r->fend[3],
                                   r->groupid,r->xover,r->lastmsg);
                  if (status<5) {
                        filemsg("Unable to write records %s ",tmpname);
                        xfclose(fout); fout = NULL;
                        return FALSE;
                  }
                }
                if (r->mrej>0) {
                        if (worst<r->mrej) {
                                worst = r->mrej;
                                worsti = i;
                        }
                }
        }
        i = group_find("comp.lang.c");
        if (i>=0) if (group_ismod(i)) {
                status_recent("ERROR: Refused %d unapproved posts in moderated group %s (see tellnews showmod in updates.htm)",group_getmrej(i),group_name(i));
        }
        if (xfclose(fout)!=0) {
		fout = NULL;
		return FALSE;
	}
	fout = NULL;
        if (isbackup==0) {
again:          if (remove(DNEWS_ACTIVE)!=0) if (errno!=ENOENT) {
                        nfail++;
                        filemsg("Remove failed for active.dat, trying again\n");
#ifdef WIN32
                        lib_msleep(100);
                        if (errno!=ENOENT) if (nfail<30) goto again;
#else
                        lib_sleep(1);
                        if (errno!=ENOENT) if (nfail<3) goto again;
#endif
                        filemsg("Giving Up, could not remove (%s)\n",DNEWS_ACTIVE);
                }
                if (nfail>0) emsg("Remove succeeded! slave must have released file\n");
                strcpy(bf,DNEWS_ACTIVE_TMP);
                if (rename(bf,DNEWS_ACTIVE)!=0) {
                  filemsg("Unable to rename %s to %s, will try later\n",bf,DNEWS_ACTIVE);
                  return TRUE;  
                 /* dnews_ex("unable to RENAME active file, CANNOT CONTINUE ");*/
                }
                remove(FILE_CHANGED);
                group_names_write();
        }
        imsg("group: Group write done\n");
        return TRUE;
}
#define FILE_INN file_log("active.inn")
void group_export(void)
{
        FILE *f;
        GRec *r;
        int i;
        char bf[400];
        char pad[20];
       
        if (ngroups==0) return; 
        f = fopen(FILE_INN,"w");
        if (f==NULL) {
                filemsg("Unable to OPEN %s \n",FILE_INN);
                return ;
        }
        for (i=0; i<ngroups; i++) {
          if (!group_isdeleted(i)) {
                r = groups[i];
                fprintf(f,"%s ",r->name);

                lprintf(bf,400-1,"%d",r->to);
                memset(pad,'0',10);
                strcpy(pad+10-strlen(bf),bf);
                fprintf(f,"%s ",pad);

                lprintf(bf,400-1,"%d",r->from);
                memset(pad,'0',10);
                strcpy(pad+10-strlen(bf),bf);
                fprintf(f,"%s ",pad);

                fprintf(f,"%c\n",r->mod);
          }
        }
        fclosep(f);
        imsg("group: Active.inn written\n");
        return ;
}
void group_addmrej(int x)
{
        if (x<0 || x>=ngroups) {
                return ;
        }
        if (groups[x]==NULL) return ;
        if (groups[x]->mrej<120) groups[x]->mrej++;
}
int group_getmrej(int x)
{
        if (x<0 || x>=ngroups) {
                return 0;
        }
        if (groups[x]==NULL) return 0;
        return groups[x]->mrej;
}
int group_binsucked(int x)
{
        int r;
        if (x<0 || x>=ngroups) {
                return TRUE;
        }
        if (groups[x]==NULL) return TRUE;
        r = groups[x]->binsucked;
        groups[x]->binsucked = TRUE;
        return r;
}
char *group_name_x(int x, char *file, int line)
{
        if (ngroups==0) group_read();
        if (x<0 || x>=ngroups) {
                emsg("group: Invalid group number %d %s:%d\n",x,file,line);
                return "";
        }
        if (groups[x]==NULL) return "";
	if (groups[x]->name==NULL) {
                emsg("group: Invalid group name, should be impossible %d %s:%d\n",x,file,line);
                return "";
	}
        return groups[x]->name;
}
char *group_namelc_x(int x,char *file, int line)
{
        static char bf[BFSZ];
        if (ngroups==0) group_read();
        if (x<0 || x>=ngroups) {
                emsg("group: Invalid group numberlc %d %s:%d\n",x,file,line);
                return "";
        }
        if (groups[x]==NULL) return "";
        ncpy(bf,groups[x]->name,BFSZ-1);
        strlwr(bf);
        return bf;
}
int group_getend(int x)
{       
        if ((x>=ngroups)||(x<0)) {emsg("getend out of range \n"); return 0;}
        if (groups[x]==NULL) return 0;
        return groups[x]->to;
}
int group_getfake(int x)
{
        if ((x>=ngroups)||(x<0)) return 0;
        if (groups[x]==NULL) return 0;
        return groups[x]->from - 1;
}


int group_getgroupid(int x)
{
        int g=x;
        if (x<0 || x>=ngroups) return 0;
        if (groups[x]==NULL) return 0;
        x = groups[x]->groupid;
        if ((x<0) || (x>=xmax_groups)) {
                emsg("group gid is corrupt %d=%d\n",g,x);
                dnews_ex("Group gid is corrupt");
                return 0;
        }
        return x;   
}
int group_getfrom(int x)
{
        if (groups[x]==NULL) return 0;
        return groups[x]->from;
}
int group_getstart(int x)
{

        int start;
        if ((x>=ngroups)||(x<0)) {emsg("getstart out of range \n"); return 0;}
        if (groups[x]==NULL) return 0;
        start  = groups[x]->from;
        if (!group_nodownload()) if (!group_iscached(x) || group_inupdate(x)) start--;
        if (start<0) start = 0;
        return start;
}
int group_downmsg(int g, int n)
{
        if (g<0) return FALSE;
        if (n >= groups[g]->from) return FALSE;
        if (n < (groups[g]->from - 1)) return FALSE;
        if (!group_nodownload()) if (!group_iscached(g) || group_inupdate(g)) return TRUE;
        return FALSE;
}
int group_ismod(int x)
{
        if ((x>=ngroups)||(x<0)) {emsg("ismod out of range \n"); return 0;}
        if (groups[x]==NULL) return FALSE;
        if (groups[x]->mod=='m') return TRUE;
        return FALSE;
}
int group_isdeleted(int x)
{
        if ((x>=ngroups)||(x<0)) {emsg("isdeleted out of range \n"); return 0;}
        if (groups[x]==NULL) return TRUE;
        if (groups[x]->mod=='e') return TRUE;
        if (groups[x]->mod=='d') return TRUE;
        if (groups[x]->mod=='g') return TRUE;
        return FALSE;
}
int group_remove(int x)
{
        if (x<0) {
                emsg("Group_remove[%d] failed\n",x); return FALSE;
        }
        if ((groups[x]->to - groups[x]->from + 1)<=0) {
                wmsg("REMOVING group {%s} \n",group_name(x));
                groups[x]->mod = 'g';   /*Set to 'gone' */
                ghash_del(group_namelc(x));
                return TRUE;
        }
        return FALSE; /* We can't deleted it until it's empty (via expire)*/
}
int group_remove_now(int x)
{
     if ((x>=ngroups)||(x<0)) {emsg("removenow out of range\n"); return TRUE;}
        if (groups[x]==NULL) return TRUE;
     groups[x]->mod = 'g';   /*Set to 'gone' */
     ghash_del(group_namelc(x));
     return TRUE;
}
void group_setlife(int x, int d)
{
        if ((x>=ngroups)||(x<0)) {emsg("setlife out of range\n"); return ;}
        if (groups[x]==NULL) return ;
        groups[x]->life = d;
}
int group_getlife(int x)
{
        if ((x>=ngroups)||(x<0)) {emsg("getlife out of range\n"); return 0;}
        if (groups[x]==NULL) return 0;
        return groups[x]->life;
}
extern int nntp_item;
static char *xref_group[100];
static int xref_num[100];
static int nxref;
static int use_xref=FALSE;
void group_setxreplic(char *s)
{
        group_xref_clear();
        if (strlen(s)==0) {
                use_xref = FALSE;
                return;
        }
        use_xref = TRUE;
        group_xref(s);
}
int group_isdownloading(int group, int num)
{
        int start,iscached;
        start = group_getstart(group);
        iscached = group_iscached(group);

        if (!group_nodownload()) return FALSE;

        if ((num==start || num+1==start) ) {
          if ( (group_inupdate(group) && num==start) ||
               (iscached && (num==start-1)) || 
               (!iscached && (num==start))  ) {
                  return TRUE;
          }
        }
        return FALSE;
}    
void group_use_xref(int t)
{
        use_xref = t;
}
int group_xref_ary(int n, int grps[], int nums[])
{
        int i;
        nxref = 0;
        for (i=0; (i<n) && (nxref<99); i++) {
                if (xref_group[nxref]==NULL) xref_group[nxref] = mymalloc(100);
                ncpy(xref_group[nxref],group_name(grps[i]),99);
                strlwr(xref_group[nxref]);
                xref_num[nxref] = nums[i];
                nxref++;
        }
        return TRUE;
}
int group_xref(char *xref) /* Xref: site group:num group:num group:num */
{
        char *s;
        char *s2;
        char bf[BFSZ];
        if (strlen(xref)==0) return TRUE;
        nxref = 0;
        for (s = strtok(xref," ,\t\n"); (s!=NULL) && (nxref<99) ; s = strtok(NULL," ,\t\n")) {
                ncpy(bf,s,BFSZ-1);
                s2 = strchr(bf,':');
                if (s2==NULL) continue;
                *s2 = 0;
                if (xref_group[nxref]==NULL) xref_group[nxref] = mymalloc(100);
                ncpy(xref_group[nxref],bf,99);
                strlwr(xref_group[nxref]);
                xref_num[nxref] = atoi(s2+1);
                nxref++;
        }
        return TRUE;
}
#define fhead(z) {s = strstr(bf,z); if (s!=NULL) s+= strlen(z);}
int group_find_file(char *s)
{
        char bf[BFSZ];
        char *lasts=NULL;
        ncpy(bf,s,BFSZ-1);
        

        for (s=bf;*s!=0;s++) {
                if (*s == '/') {*s = '.'; lasts = s;}
                if (*s == '\\') {*s = '.'; lasts = s;}
        }
        if (lasts==NULL) return -1;
        *lasts = 0;
        strlwr(bf);
        fhead("spool.news");
        if (s==NULL) fhead("spool.dnews");
        if (s==NULL) fhead("news.old");
        if (s==NULL) fhead("news");
        if (s==NULL) fhead("dnews");
        if (s==NULL) return -1;
        return group_find(s+1);
}
void group_xref_clear(void)
{
        nxref = 0;
}
int group_here_get(void)
{
        return xref_num[0]; 
}
int group_xref_here(int g, int num)
{
        nxref = 0;
        if (xref_group[nxref]==NULL) xref_group[nxref] = mymalloc(100);
        ncpy(xref_group[nxref],group_namelc(g),99);
        xref_num[nxref] = num;
        nxref++;
        return TRUE;
}
#define XOVER_DAYS 30
int xover_days(void)
{
	static int nn;
	int n;
	if (nn>0) return nn;
	n = atoi(config_get(C_XOVER_DAYS));
	if (n==0) n = XOVER_DAYS;
	nn = n;
	return n;
}
int group_bounds(x)
{
        if ((x>=ngroups)||(x<0)) {emsg("group_bounds %d\n",x); return TRUE;}
        if (groups[x]==NULL) {emsg("group_null %d\n",x); return TRUE;}
        return FALSE;

}
static int nohits;
int group_xov_nohits(int x)
{
	nohits = x;
	return TRUE;
}
int group_hitxover(int x)
{
	int r;
	static int last;
        if (group_bounds(x)) {emsg("Hitxover bad number\n"); return 0;}

	r = groups[x]->range;
	if (r<0) r = group_getend(x)-group_getstart(x) + 1;
	if (groups[x]->xover!=0) if (last!=x) imsg("xover: hitxover %d (%s) range=%d,%d nohits=%d xover=%d\n",x,group_name(x),groups[x]->range,r,nohits,groups[x]->xover);
	last = x;
        if (r > 20) {
		if (!nohits) {
	                if (groups[x]->xover > 0) slave_xoverhit(x);
	                groups[x]->xover = 0; 
		}
        }
        if (groups[x]->xover<=xover_days()) return TRUE;
        return FALSE;
}
void group_regid(void)
{
        int i;
        for (i=0; i<xmax_groups; i++) gid_to_g[i] = -1;
        for (i=0; i<xmax_groups; i++) gid_isused[i] = FALSE;
        for (i=0; i<ngroups; i++) {
                groups[i]->groupid = i+1;
                gid_to_g[i+1] = i;
        }
        gbigest = ngroups+1;
}
int group_nxover(void)
{
        int n=0;
        int i;
        for (i=0; i<ngroups; i++) if (group_isxover(i)) n++;
        return n;
}
int group_isxover(int x)
{
        if ((x>=ngroups)||(x<0)) {emsg("isxover FATAL %d\n",x); return FALSE;}
        if (groups[x]==NULL) return FALSE;
        if (groups[x]->xover > xover_days()) return FALSE;
        return TRUE;
}
void group_setxsort(int x, int t)
{
        if ((x>=ngroups)||(x<0)) return;
	groups[x]->isxsort = t;
}
int group_getxsort(int x)
{
        if ((x>=ngroups)||(x<0)) return TRUE;
	return groups[x]->isxsort;
}
int group_getxover(int x)
{
        if ((x>=ngroups)||(x<0)) {emsg("getxover FATAL %d\n",x); return 5;}
        if (groups[x]==NULL) return 0;
        return groups[x]->xover;
}
void group_setxover(int x, int days)
{
        if ((x>=ngroups)||(x<0)) {emsg("setxover FATAL %d\n",x); return ;}
        if (groups[x]==NULL) return;
        groups[x]->xover = days;
}
int group_getnext(int x)
{
        int i;
        int num= -1;
        if ((x>=ngroups)||(x<0)) {emsg("getnext FATAL %d\n",x); return 5;}
        if (groups[x]==NULL) return 0;
        groups[x]->lastmsg = 0;
        if (ishdr_old()) {
                if (!hdr_matchgroup(group_namelc(x))) {
			if (config_true(C_REPLICATE) || use_xref ) goto replicate;
			goto normal;
		}
                goto replicate;
        }
        if (config_true(C_REPLICATE) || use_xref) {
replicate:
                for (i=0; i<nxref; i++) {
                        if (strcmpnc(group_namelc(x),xref_group[i])==0) {
                                num = xref_num[i];
                                break;
                        }
                }
                if (num<0) {
                        imsg("Cross post problem, with replicate {%s} rejecting item %d %d\n",group_name(x),groups[x]->to,nxref);
                        if (nxref>0) imsg("Original Upstream (%s:%d)\n",xref_group[0],xref_num[0]);	
			return -1;
/*                        group_addrange(x,1);
                        return ++(groups[x]->to);*/
                }
                if (groups[x]->to < num) groups[x]->to = num;
                if (groups[x]->from > num) groups[x]->from = num;
                group_addrange(x,1);
		if (!config_true(C_INDEX_CHANGES)) {	
	                group_master(x);
		}
                return num;
        } else {
normal:         group_addrange(x,1);
                num = ++ (groups[x]->to);
		if (!config_true(C_INDEX_CHANGES)) {	
	                group_master(x);
		}
                return num;
        }
}
int group_feedend(int x, int i)
{
        if ((x>=ngroups)||(x<0)) {emsg("FEEDEND FATAL\n"); return 0;}
        if (groups[x]==NULL) return 0;
        if (i>3) i = 3;
        if (i<0) {emsg("feedend faulty %d:%d\n",x,i); return 0; }
        return groups[x]->fend[i];
}
void group_setfeedend(int x, int f, int i)
{
        if ((x>=ngroups)||(x<0)) {emsg("SETFEEDEND FATAL\n"); return ;}
        if (groups[x]==NULL) return ;

        if (f>3) f = 3;
        if (f<0) {emsg("setfeedend faulty %d:%d=%d\n",x,f,i); return ; }

        groups[x]->fend[f] = i;
}
void group_setfeedendif(int x, int f, int i)
{
        if ((x>=ngroups)||(x<0)) {emsg("SETFEEDEND FATAL\n"); return ;}
        if (groups[x]==NULL) return ;

        if (f>3) f = 3;
        if (f<0) {emsg("setfeedend faulty %d:%d=%d\n",x,f,i); return ; }
	if (groups[x]->fend[f] < i) groups[x]->fend[f] = i;
}
void group_addrange(int x, int i)
{
        if ((x>=ngroups)||(x<0)) {emsg("ADDRANGE FATAL\n"); return;}
        if (groups[x]==NULL) return ;

        if (groups[x]->range>=0) groups[x]->range += i;
}
void group_master(int x)
{
        groups[x]->changed = TRUE;
        if (!is_slave()) {
                slave_fixgroup(x,groups[x]->from,groups[x]->to,groups[x]->range,groups[x]->cache,groups[x]->mod);
        }
}
void group_setrange(int x, int i)
{
        int old;
        if ((x>=ngroups)||(x<0)) {emsg("SETRANGE FATAL\n"); return ;}
        if (groups[x]==NULL) return ;
        old = groups[x]->range;
        groups[x]->range = i;
	if (!config_true(C_INDEX_CHANGES)) {	
	        if (old!=i) group_master(x);
	}
}

void group_fixup(int gid, int i)
{
        int g;
        if ((gid<0) || (gid>=xmax_groups)) return;
        g = gid_to_g[gid];
        if (g<0) return;
        if (group_bounds(g)) {emsg("group fixup %d %d\n",gid,g); return;}
        if (groups[g]->from < i) groups[g]->from = i;
        if (groups[g]->to > i) groups[g]->to = i;
        groups[g]->range++; 
}
int group_fixrange(void)
{
/*        int i;
        for (i=0; i<group_n(); i++) groups[i]->range = -1;
*/        return TRUE;    

}
int group_rawrange(int x)
{
        if ((x>=ngroups)||(x<0)) {emsg("GETRANGE FATAL\n"); return 0;}
        if (groups[x]==NULL) return 0;
/*      if (groups[x]->range < 0) return 0;*/
        return groups[x]->range;
}
int group_getrange(int x)
{
        int r;
        if ((x>=ngroups)||(x<0)) {emsg("GETRANGE FATAL\n"); return 0;}
        if (groups[x]==NULL) return 0;

        if (group_iscached(x) || group_nodownload() ) {
                if (groups[x]->range<0) groups[x]->range = item_findnum(x,&groups[x]->from,&groups[x]->to);
                r = groups[x]->range;
		if (!group_nodownload()) if (!group_iscached(x) || group_inupdate(x)) r++;
        } else {
                if (groups[x]->range > 0) return groups[x]->range+1;
                r = group_getend(x)-group_getstart(x) + 1;
        }
        return r;
}
void group_fix_byindex(int x, int from, int to, int nitems)
{
        if ((x>=ngroups)||(x<0)) {emsg("SETEND FATAL\n"); return;}
        if (groups[x]==NULL) return; 
        groups[x]->to = to;
        groups[x]->from = from;
        groups[x]->range = nitems;	
}
void group_setend(int x, int i)
{
        int old;
        if ((x>=ngroups)||(x<0)) {emsg("SETEND FATAL\n"); return;}
        if (groups[x]==NULL) return; 
        old = groups[x]->to;
        groups[x]->to = i;
	if (!config_true(C_INDEX_CHANGES)) {	
	        if (old!=i) group_master(x);
	}
}
void group_setstart(int x, int i)
{
        if ((x>=ngroups)||(x<0)) {emsg("SETSTART FATAL\n"); return;}
        if (groups[x]==NULL) return ;
        groups[x]->from = i;
        if (i>groups[x]->to) groups[x]->range = 0; /* Range must be 0 */
}
void slave_tellmaster_cached(int x);
void group_setcached_hit(int g)
{
        int life=atoi(config_get(C_LIFE));
        if (g<0) return;
        imsg("setcached hit from readonly slave (%s) \n",group_name(g));
        if (!group_iscached(g)) {
                if (!group_inupdate(g)) {
                        suck_queue_anyone(group_name(g),0);
                        group_setupdate(g,TRUE);
                }
        }

        if (group_getlife(g)<life) group_setlife(g,life);
        group_setcached(g,TRUE);
}
void group_setcached_slave(int g)
{
        int life= atoi(config_get(C_LIFE));
	int wanted;
        if (!is_slave()) return;
	wanted = (life+1)/2;
	if ((life-2) > wanted) wanted = life-2;
        if (group_getlife(g)<(life+1)/2) {
                group_setlife(g,life);
                group_setcached(g,TRUE);
                slave_tellmaster_cached(g);
        }
}
void readonly_setcached(int x);
void group_setcached(int x, int iscached)
{
        if ((x>=ngroups)||(x<0)) {emsg("SETCACHED FATAL\n"); return;}
        if (groups[x]==NULL) return ;
        if (is_reader()) if (!is_slave()) if (iscached) if (groups[x]->cache != 'c') {
                imsg("Readonly setcached %s\n",group_name(x));
                readonly_setcached(x);
        }
        if (iscached) groups[x]->cache = 'c';
        else groups[x]->cache = 'n';
        group_master(x);
}
void group_setcachedc(int x, int iscached)
{
        if ((x>=ngroups)||(x<0)) {emsg("SETCACHED FATAL\n"); return;}
        if (groups[x]==NULL) return ;
        groups[x]->cache = (char) iscached;
}
void group_setmod(int x, char *s)
{
        if ((x>=ngroups)||(x<0)) {emsg("SETmod FATAL\n"); return;}
        if (group_bounds(x)) {emsg("group setmod %d\n",x); return;}

        if (*s==0) return;
        if (*s=='=') return;
        if (*s=='g') {
                emsg("Cannot remove group with setmod() call g=%d \n",x);
                return; 
        }
        if (groups[x]->mod!=*s) groups[x]->changed = TRUE;
        groups[x]->mod = *s;
}
void group_addsize(int grp, int sz)
{
        if ((grp<0) || (grp>=ngroups)) return;
        if (groups[grp]==NULL) return ;
        groups[grp]->size += (sz/100);
}
/* Disable posting, don't delete until group is empty from expire working. */
/* this makes delete a very 'safe' operation */
/* Note: expire doesn't currently delete the empty group, must add this !!! */

void group_delete_x(int x, char *file, int line)
{
        char oldmod;
        if (x<0) return;
	glog("%s:%d rmgroup %s\n",file,line,group_name(x)); 
        oldmod = groups[x]->mod;

        if (groups[x]->mod == 'm') groups[x]->mod = 'e'; /* Deleted moderated group */
        else groups[x]->mod = 'd';
        if (groups[x]->mod!=oldmod) group_master(x);
}
void group_undelete(int x)
{
        char oldmod;
        if (x<0) return;
        oldmod = groups[x]->mod;
	if (groups[x]->mod == 'm') return;
        if (groups[x]->mod == 'e') groups[x]->mod = 'm'; /* Deleted moderated group */
        else groups[x]->mod = 'y';
        if (groups[x]->mod!=oldmod) group_master(x);
}
int group_iscached(int x)
{
        int r;
        if ((x>=ngroups)||(x<0)) {emsg("iscached FATAL\n"); return FALSE;}
        if (groups[x]==NULL) return FALSE;

        if (ngroups==0) group_read();
        if (x>=ngroups) {emsg("iscached out of range \n"); return FALSE;}
        if (config_true(C_CACHEDALL)) {
                if (group_isdeleted(x)) return FALSE;
                return TRUE;
        }
        if (groups[x]->cache=='c') r = TRUE;
        else r = FALSE;
        return r;
}
int group_iscachedreal(int x)
{
        int r;
        if ((x>=ngroups)||(x<0)) {emsg("iscached FATAL\n"); return FALSE;}

        if (ngroups==0) group_read();
        if (groups[x]==NULL) return FALSE;
        if (x>=ngroups) {emsg("iscachedreal out of range \n"); return FALSE;}
        if (groups[x]->cache=='c') r = TRUE;
        else r = FALSE;

        if (group_isdeleted(x)) return FALSE;
        return r;
}
int group_n(void)
{
        if (ngroups==0) group_read();
        return ngroups;
}
char *group_getlist(int x) /*return in LIST format */
{
        static char bf[BFSZ];
        GRec *r;
        if (x<0 || x>=ngroups)  {
                emsg("group: list cmd Asked for non existent group %d/%d \n",x,ngroups);
                return "";
        }
        if (groups[x]==NULL) return "";
        r = groups[x];
        if (r->name==NULL) {
                emsg("group: Asked for null group %d/%d \n",x,ngroups);
                return "";
        }

        if (group_isdeleted(x)) {
                /* dmsg("group: list cmd referred to a deleted group %s \n", r->name);*/
                return "";
        }
        lprintf(bf,BFSZ-1,"%s %d %d %c\015\012",r->name,r->to,group_getstart(x),r->mod);

        return bf;
}
char *group_getlistx(int x) /*return in LIST format */
{
        static char bf[BFSZ];
        GRec *r;
        if (x<0 || x>=ngroups)  {
                emsg("group: list cmd Asked for non existent group %d/%d \n",x,ngroups);
                return "";
        }
        if (groups[x]==NULL) return "";
        r = groups[x];
        if (r->name==NULL) {
                emsg("group: Asked for null group %d/%d \n",x,ngroups);
                return "";
        }

        if (group_isdeleted(x)) {
                /* dmsg("group: list cmd referred to a deleted group %s \n", r->name);*/
                return "";
        }
        lprintf(bf,BFSZ-1,"%s %d %d %c %d\015\012",r->name,r->to,group_getstart(x),r->mod,group_getrange(x));

        return bf;
}
char *group_getlist2(int x) /*return in LIST format */
{
        static char bf[BFSZ];
        int range;
        GRec *r;
        if (x<0 || x>=ngroups) {
                emsg("group: list cmd Asked for non existent group %d/%d \n",x,ngroups);
                return "";
        }
        if (groups[x]==NULL) return "";

        r = groups[x];
        if (r->name==NULL) { 
                emsg("group: Asked for null group %d/%d \n",x,ngroups);
                return "";
        }

        if (group_isdeleted(x)) return "";

        if (groups[x]->range<0) range = group_getend(x)-group_getstart(x) + 1;
        else range = groups[x]->range;
	if (!group_nodownload()) 
           if (!group_iscached(x) || group_inupdate(x)) range++;
  
        lprintf(bf,BFSZ-1,"%s %d %d %c %d\015\012",r->name,r->to,group_getstart(x),r->mod,range);

        return bf;
}
static int lastall;
int group_match_none(void)
{
        return lastall;
}
int group_match(char *wild, char *grp2)
{
        char bf[1000];
        char grp[BFSZ];
        int m=TRUE,lastm;
        char *s,*p,*w;
        int done,len;
        int all;

        lastall = FALSE;
        ncpy(grp,grp2,BFSZ-10); strlwr(grp);
        lastm = FALSE;
        p = wild;
        done = FALSE;
        for (;!done;) {
                s = strchr(p,',');
                if (s==NULL) {s = wild + strlen(wild); done = TRUE;}
                if (*p==0) break;
                len = s - p;
                if (len>100) len = 100;
                strncpy(bf,p,len); bf[len] = 0;
                strlwr(bf); w = bf;

                m = TRUE;
                all = FALSE;
                if (*w=='!') {
                        w++;
                        m = FALSE;
                }
                if (*w=='@') {
                        w++;
                        m = FALSE;
                        all = TRUE;
                }
                if (match_wild(w,grp)) { lastm = m; lastall = all;} 

                p = s + 1;
        }
        return lastm;
}
int old_group_match(char *wild, char *grp2)
{
        char bf[10000];
        char grp[BFSZ];
        int m=TRUE,lastm;
        char *s;

        ncpy(grp,grp2,BFSZ-1); strlwr(grp);
        lastm = FALSE;
        ncpy(bf,wild,10000-1);
        s = strtok(bf,",");
        for (;s!=NULL;) {
                m = TRUE;
                if (*s=='!') {
                        s++;
                        m = FALSE;
                }
                if (match_wild(s,grp)) { lastm = m; } 
                s = strtok(NULL,",");
        }
        return lastm;
}
int group_inupdate(int g)
{
        if (g<0) {imsg("In update, g = %d\n",g); return FALSE;}
        return inupdate[g];
}
int group_iscached2(int x)
{
        if (group_inupdate(x)) return FALSE;
        if (group_iscached(x)) return TRUE;
        return FALSE;
}
int group_iscached3(int x) /* if updating group then accept items */
{
        if (group_inupdate(x)) return TRUE;
        if (group_iscached(x)) return TRUE;
        return FALSE;
}

void group_setupdate(int g,int v)
{
        if (g<0) {
                emsg("Inupdate failed [%d] = %d\n",g,v);
                return;
        }
        inupdate[g] = v;
}
/*      For statistical measurements of news reading                    */
/*              day/hour:min:sec/user/ipname/group/number               */
static int unknown=FALSE;
void group_unknown(int x)
{
        unknown = x;
}
static FILE *fused;
char *used_fname(void);
int group_used(int chan, int grp, int num, int sz)
{
        time_t ts;
	char *gname;
	struct tm atm;
        struct tm *t; 
        int life=atoi(config_get(C_LIFE));
        ts = time(NULL);
        t = safe_localtime(&ts,&atm);

        cstat_addnread(chan,1);

        if ((grp>=0) && (grp<ngroups)) {
                if (!unknown) {
                        groups[grp]->nread += 1;
                        groups[grp]->rsize  += sz/100;
                        if (group_getlife(grp)<life) {
                                group_setcached_slave(grp);
                                group_setlife(grp,life);
                        }
                        group_setcached(grp,num);
                } else imsg("Use not logged as article referred by MSG ID %s:%d\n",group_name(grp),num);
        }
        user_addinfo(chan,1,0,0);
        if (!config_true(C_LOGUSE)) return TRUE; 
        /* if (is_slave()) return TRUE;*/
        if (fused==NULL) fused = fopen_log(file_log(used_fname()),"a");
        if (fused==NULL) {filemsg("Unable to open %s \n",file_log(used_fname())); return TRUE;}

	if (grp>=0) gname = group_name(grp); else gname="finish_cmd";

        fprintf(fused,"%04d%02d%02d/%02d:%02d:%02d/%s/%s/%s/%d/%d\n"
                ,t->tm_year+1900,t->tm_mon+1,t->tm_mday
                ,t->tm_hour,t->tm_min,t->tm_sec
                ,chan_getuser(chan),chan_getipname(chan),gname,num,sz);
        return TRUE;
}
char *used_fname(void)
{
        static char bf[BFSZ];
        time_t ts;
	struct tm atm;
        struct tm *t; 
        int life=atoi(config_get(C_LIFE));
        ts = time(NULL);
        t = safe_localtime(&ts,&atm);                       

        if (is_slave()) {
                lprintf(bf,BFSZ-1,"u_%04d%02d%02d.log_%d",t->tm_year+1900,t->tm_mon+1,t->tm_mday,slave_n());
        } else {
                lprintf(bf,BFSZ-1,"u_%04d%02d%02d.log",t->tm_year+1900,t->tm_mon+1,t->tm_mday);
        }
        return bf;
}
void group_used_flush(void)
{
        if (fused!=NULL) {
                fflush(fused);
                xfclose(fused); fused = NULL;
        }
        fused = NULL;
}
void group_uncache_most(void)
{
        int i;
        int nun=0;
        int life = atoi(config_get(C_LIFE))+4;
        if (life==4) life = 30;
        if (ngroups==0) group_read();
        for (i=0; i<ngroups; i++) {
                if (group_iscached(i)) if (group_getlife(i)<life) {
                        imsg("group: Un-Caching group (%s) life was %d \n",group_name(i),group_getlife(i));
                        group_setcached(i,FALSE);
                        group_setlife(i,0);
                        nun++;
                }
        }
        emsg("group: Uncached %d groups due to suck_maxcache being exceeded\n",nun);
}
int group_nodownload(void)
{
	return config_true(C_CACHED_HIDE);	
}
void group_clearxsort(void)
{
	time_t now;
	int i;
        struct tm *t; 
	struct tm atm;
        now = time(NULL);
        t = safe_localtime(&now,&atm);
        if (t->tm_wday!=2) return;
	imsgf("Clearing xover sorting flags\n");
        for (i=0; i<ngroups; i++) {
		group_setxsort(i,FALSE);
	}
}
void group_age(void) /* Called once a day*/
{
        int i;
        int x;
        int age;
	
	
	if (ngroups==0) group_read();
        wmsg("group: Aging groups one day, ngroups %d \n",ngroups);
        for (i=0; i<ngroups; i++) {
		group_setxsort(i,FALSE);
                groups[i]->lastmsg++;
                age = group_getlife(i);
                if (age>0) age--;
	        if (age<=0) if (config_true(C_CACHE_ONXOVER)) {
			if (group_isxover(i)) age=3;
		}
                if (age==0) {
			if (!config_true(C_CACHEDALL)) 
				if (group_iscached(i)) imsg("group: Un-Caching group (%s)\n",group_name(i));
                        group_setcached(i,FALSE);
                } else group_setcached(i,TRUE);
                group_setlife(i,age);
                x = group_getxover(i); /* Age days since xover command */
                if ((x>=xover_days()) && (x<=xover_days()+5)) {
			if (config_true(C_XOVER_XHASH)) xhash_delete(group_getgroupid(i));
                        imsg("Removing old .xov file {%s} \n",file_xover(i));
                        remove(file_xover(i));
                }
                if (x<100) x++;
                group_setxover(i,x);
        }
        group_write();
        dmsg("group: New group file written %d lines \n",ngroups);
}
void group_cacheall(int days) /* Called once a day*/
{
        int i;
        if (days==0) days=365;
        if (ngroups==0) group_read();
        wmsg("group: Setting all groups to cached for %d days \n",days);
        for (i=0; i<ngroups; i++) {
                group_setcached(i,TRUE);
                group_setlife(i,days);
        }
        group_write();
        imsg("group: New group file written %d lines \n",ngroups);
}

void group_names_load(void)
{
        char bf[BFSZ];
        static int loaded;
        FILE *f;
        if (loaded) return;
        loaded = TRUE;
        
        f = fopen(DNEWS_ACTIVE_NAMES,"r");
        if (f==NULL) {
                imsg("Unable to read (%s), not serious\n",DNEWS_ACTIVE_NAMES);
                return;
        }
        for (;!feof(f);) {
                if (fgets(bf,BFSZ-10,f)==NULL) break;
                group_names_addfix(bf);
        }
        fclosep(f);
}
int gname_isok(char *s, Sary *names);
int group_fixnames(void)
{
        return TRUE;
}
int group_names_addfix(char *bf)
{
        char name[BFSZ],title[BFSZ];
        int g;
        char *s;
        s = strtok(bf,"\t "); if (s==NULL) return -1;
        ncpy(name,s,BFSZ-1);
        s = strtok(NULL,"\r\n"); if (s==NULL) return -1;
        if (s==NULL) return -1;
        for (;*s!=0;s++) if (!isspace(*s)) break;
        ncpy(title,s,BFSZ-1);
        if (strchr(name,' ')!=NULL) return -1;
        g = group_find(name);
        if (g>=0) group_settitle(g,title);
        names_changed = TRUE;
        return g;
}
int group_names_add(char *s)
{
        group_names_load();
        names_changed = TRUE;
        return group_names_addfix(s);
}
int group_settitle(int i,char *s)
{
        group_names_load();
        if (i<0) return i;
        if (i>=max_groups) return i;
        if (groups[i]==NULL) return i;
        groups[i]->title = mystrkeep(s);                        
        names_changed = TRUE;
        return i;
}
char *group_gettitle(int i)
{
        group_names_load();
        if (i<0) return NULL;
        return groups[i]->title;
}
void group_names_write(void)
{
        char *t;
        FILE *f;
        int i;
        if (!names_changed) return;
        group_names_load();
        if (is_readonly()) return;
        imsg("Writing active.names file\n");
        f = fopen(DNEWS_ACTIVE_NAMES,"w");
        if (f==NULL) {
                filemsg("Can't create (%s)\n",DNEWS_ACTIVE_NAMES);
                return;
        }
        for (i=0; i<group_n(); i++) {
                t = group_gettitle(i);
                if (t!=NULL) 
                  fprintf(f,"%s\t%s\n",group_name(i),t);
        }
        fclosep(f);
        imsg("Writing active.names file done\n");
        names_changed = FALSE;
}
void group_setcatchup(int x, int f, int v)
{
        if ((x>=ngroups)||(x<0)) {emsg("SETCATCHUP FATAL\n"); return;}
        if (groups[x]==NULL) return ;

        groups[x]->catchup = v;
}
void group_usecatchup(int x, int f)
{
        if ((x>=ngroups)||(x<0)) {emsg("USECATCHUP FATAL\n"); return;}
        if (groups[x]==NULL) return ;
        if (group_feedend(x,f) > groups[x]->catchup) {
                emsg("group: Ignoring catchup value group %s [%d] fend %d > catchup %d \n",group_name(x),f,group_feedend(x,f),groups[x]->catchup);
                return;
        }
        group_setfeedend(x,f,groups[x]->catchup);
        wmsg("group: Catching up group %s fend %d \n",group_name(x),group_feedend(x,f));
}

int group_cmp(const void *pa, const void *pb);
int group_find_fast(char *s);
void group_sort(void);

int group_find(char *s)
{
        char bf[101];
        
        if (s==NULL) return -1;
        if (ngroups==0) group_read();
        ncpy(bf,s,100);
        s = strchr(bf,' ');
        if (s!=NULL) *s = 0;
        strlwr(bf);

        return ghash_find(bf);

/*      return group_find_fast(bf);*/
}
int group_cmp(const void *pa, const void *pb)
{
        int *a = (int *) pa;
        int *b = (int *) pb;
        return strcmp( groups[*a]->name, groups[*b]->name);
}
void group_sort(void)
{
        int i;
        static int done_init;

        if (!group_changed) return;
        group_changed = FALSE;
        imsg("Group sort\n");
        for (i=0; i<ngroups; i++) g_p[i] = i;           
        qsort(g_p,ngroups,sizeof(int),group_cmp);
        imsg("Group sort done\n");

}
int group_ord(int i)
{
        return g_p[i];
}
int group_find_fast(char *s)
{
        int *r;
        GRec tmp;

        group_sort();

        tmp.name = mystrdup(s,SD_GROUP);
        groups[ngroups] = &tmp; g_p[ngroups] = ngroups;
        r = bsearch(&ngroups, g_p, ngroups,sizeof(int),group_cmp);

        mystrfree(tmp.name,SD_GROUP);
        if (r==NULL) return -1;
        return *r;
}
int group_wanted(char *s)
{
        return feed_wanted(s);
}

int group_nameok(char *s)
{
        static int done_init;
        static int mask[300];
        int i;
        if (!done_init) {
                for (i=1; i<32; i++) {
                        mask[i] = TRUE;         
                }
                for (i=1; i<256;i++) {
                        if (strchr(",:;~!@#%^/\\*()[]{}?<>| \t",i)!=NULL) mask[i] = TRUE;
                }
                done_init = TRUE;
        }
        if (strlen(s)>98) return FALSE; /* Maximum length of group name */

        if (config_true(C_NON_ASCII)) {
                for (;*s!=0;s++) {
                        if (xisspace(*s)) return FALSE;
                }
        } else {
                for (;*s!=0;s++) {
                        if (mask[(unsigned char) (*s)]) return FALSE; 
                }
        }
        return TRUE;
}

#define GHASH_SIZE 12000 /* Seems to work fine with smallish values. */
typedef struct Gref {int grp; struct Gref *next;} GREF;
static GREF *ghash[GHASH_SIZE];

int ghash_add(char *gname2, int grp)
{
        int h;
        GREF *x;
        char gname[BFSZ];
        int i=0;
        ncpy(gname,gname2,BFSZ-10);
        strlwr(gname);
        if (grp<0) return FALSE;
        if (grp>=max_groups) return FALSE;
        h = str_hash(gname,GHASH_SIZE);
        x = mykeep(sizeof(GREF));
        x->next = ghash[h];
        x->grp = grp;
        ghash[h] = x;
        return FALSE;
}
int ghash_find(char *gname)
{
        int h = str_hash(gname,GHASH_SIZE);
        GREF *x;

        for (x=ghash[h];x!=NULL;x=x->next) {
                if (x->grp<0) break;
                if (strcmpnc(gname,groups[x->grp]->name)==0) return x->grp;
        }
        return -1;
}
int ghash_del(char *gname)
{
        int h = str_hash(gname,GHASH_SIZE);
        int i=0;
        GREF *x,*last;
        last = NULL;
        for (x=ghash[h];x!=NULL;x=x->next) {
                if (strcmpnc(gname,group_name(x->grp))==0) {
                        if (last==NULL) ghash[h] = x->next;
                        else last->next = x->next; /* mem leak */ 
                        return TRUE;    
                }
                last = x;
        }
        return FALSE;
}
int group_nc(void)
{
        int i;
        int nc=0;
        for (i=0; i<group_n(); i++) if (group_iscachedreal(i)) nc++;
        return nc;
}
int group_nx(void)
{
        int i;
        int nc=0;
        for (i=0; i<group_n(); i++) if (group_isxover(i)) nc++;
        return nc;
}
char *group_status(void)
{
        static char bf[1200];
        int tup;

        tup = lib_gmtime()-uptime;

        lprintf(bf,1200-1,"Groups Xover %d %dMB, Cached %d of %d, GID %d,  Uptime %d day(s) %d hour(s), XOver Cache %.2f%%\015\012\n",group_nx(),xov_total_get()/1000,group_nc(),group_n(),group_lastgid(),tup/(60*60*24),(tup/(60*60)) % 24,(xcache_hitrate()*100));
        return bf;

}
char *uptime_status(void)
{
        static char bf[1200];
        int tup;

        tup = lib_gmtime()-uptime;

        lprintf(bf,1200-1,"Uptime %d day(s) %d hour(s)\015\012\n",tup/(60*60*24),(tup/(60*60)) % 24);
        return bf;

}

static char *nodup_used;
int group_nodup(void)
{
        int i;
        int r;
        int ndup = 0;
        if (nodup_used==NULL) nodup_used = mymalloc(max_groups);        
        memset(nodup_used,0,max_groups);
        for (i=0; i<group_n(); i++) {
                r = group_find(group_name(i));
                if (r<0) {
                        emsg("nodup: Group find failed, dnews bug ?  {%s}\n",group_name(i));
                } else {
                        if (nodup_used[r]) {
                                imsg("Duplicate group deleted {%s}\n",group_name(i));
                                groups[i]->mod = 'g'; ndup++;
                        }
                        nodup_used[r] = TRUE;
                }
        }
        return ndup;

}
void group_merge(char *p1) /* Merge a bunch of groups from another active file */
{
        int i,cnt;
        char inbuff[BFSZ];
        char bf[BFSZ];
        char *s;
        FILE *f;
        char fname[BFSZ];
        int g;
        if (strlen(p1)==0) p1 = DNEWS_MERGE;
        if (strchr(p1,'/')==NULL) if (strchr(p1,':')==NULL) if (strchr(p1,'\\')==NULL) {
                p1 = file_work(p1);
        }
        ncpy(fname,p1,BFSZ-1);

        f = xfopen(fname,"r");
        if (f==NULL) {
                filemsg("Couldn't open %s\n",fname);
                return;
        }
        for (cnt=0;!feof(f);cnt++) {
          if (fgets(inbuff,BFSZ-1,f)!=NULL) {
                i = strlen(inbuff);
                if (i>0) if (inbuff[i-1]=='\n') inbuff[i-1] = 0;
                strcpy(bf,inbuff);
                s = strtok(inbuff," \t,\n\015\012");
                if (s!=NULL) if (strlen(s)<80) {
                        if (group_find(s)<0) {
                          if (!group_wanted(s)) {
                                imsg("Skipping group that doesnt match me feed (%s) \n",s);
                          } else {
                                imsg("Adding group {%s} \n",s);
                                g = group_add(bf);
                                if (g>=0) {
                                        group_times(g,s,"group_merge",lib_gmtime());
                                }
                                if (gconvert) if (g>=0) {
                                        groups[g]->from = groups[g]->to + 1;
                                }               
                          }
                        }
                }
          }
        }
        xfclose(f); f = NULL;
	feed_mask_reload();

}
int cmp_nread(const void *aa, const void *bb);
int cmp_nread(const void *aa, const void *bb)
{
        int a = *((const int *)aa);
        int b = *((const int *)bb);

        return (groups[b]->nread - groups[a]->nread);
}
int cmp_size(const void *aa, const void *bb);
int cmp_size(const void *aa, const void *bb)
{
        int a = *((const int *)aa);
        int b = *((const int *)bb);

        return (groups[b]->size - groups[a]->size);
}
void stats_groups_clear(void)
{
	int i;
	for (i=0; i<ngroups; i++) {
                groups[i]->nread = 0;
		groups[i]->size = 0;
		groups[i]->rsize = 0;

	}
}
String *stats_groups_read(char *wild, int chan)
{
        int *p;
        int zi;
        char bf[BFSZ];
        static String *stat;
        int i,ii,ni;
        if (stat==NULL) stat = str_new();

        /*RHP cant have a static data >32k so need malloc for p array*/

	group_read_nread();

        
	p = mymalloc(max_groups*sizeof(int));
        if (p==NULL) {emsg("group: Memory  allocation failure\n"); return 0;}/*RHP is return 0 ok?*/

        for (i=0; i<ngroups; i++) p[i] = i;
        ni = 40; if (ni>ngroups) ni = ngroups;
        if (strcmp(wild,"*")==0) ni = ngroups;

        qsort(p,ngroups,sizeof(int),cmp_nread);

        str_copysn(stat,"Top 40 groups sorted by number of articles read\015\012");
        str_addsn(stat,"                                  Group    NRead      KBytes-Read Received\015\012");
	if (chan>=0) {chan_write(chan,stat); str_clear(stat);}
        for (i=0,zi=0; i<ngroups; i++) {
                if (zi>ni) break;
                ii = p[i];
                if (strlen(wild)>0) if (!group_match(wild,group_name(p[i]))) continue;
                lprintf(bf,BFSZ-1,"%39s  %7d  %9.0fk %9.0fk\015\012",group_name(p[i])
                        ,groups[ii]->nread,(double) ((double) groups[ii]->rsize)/10.0,(double) ((double) groups[ii]->size)/10.0);
                if (!group_isdeleted(ii))  {
			str_addsn(stat,bf);
                	 zi++;
		}
        }

        qsort(p,ngroups,sizeof(int),cmp_size);

        str_addsn(stat,"\015\012Top 40 groups sorted by kbytes received since startup\015\012");
        str_addsn(stat,"                                  Group    NRead      KBytes-Read Received\015\012");
        for (zi=0,i=0; i<ni; i++) {
                if (zi>ni) break;
                ii = p[i];
                if (strlen(wild)>0) if (!group_match(wild,group_name(p[i]))) continue;
                lprintf(bf,BFSZ-1,"%39s  %7d  %9.0fk %9.0fk\015\012",group_name(p[i])
                        ,groups[ii]->nread,(double) ((double) groups[ii]->rsize)/10.0,(double) ((double) groups[ii]->size)/10.0);
                 if (!group_isdeleted(ii)) {
			 str_addsn(stat,bf);
	                zi++;
			if (chan>=0) {chan_write(chan,stat); str_clear(stat);}
		}
        }
        if (p!=NULL) myfree(p);
	group_reset_rsize();
	if (chan>=0) {chan_write(chan,stat); str_clear(stat);}
        return stat;
}
/* Dealing with empty news groups to get them out of active file */
#define FILE_EMPTY file_work("empty.dat")
#define FILE_EMPTY_NEW file_work("empty.new")
static FILE *emptyf;
typedef struct EREC {char *name; char mod; struct EREC *next;} Erec;
#define ENHASH 5000
static Erec *empty[ENHASH]; 
static int empty_changed=FALSE;
Erec *empty_find(char *xname);
int empty_addif(char *xname, char *mod)
{
        Erec *r = empty_find(xname);
        if (r!=NULL) {
                r->mod = *mod;
                empty_changed = TRUE;
        } else {
                empty_add(xname,mod);
        }
        return TRUE;
}
int empty_add_mem(char *name, char *mod);
int empty_add(char *xname, char *mod)
{
        char name[BFSZ];
        ncpy(name,xname,BFSZ-1); strlwr(name);
        if (emptyf==NULL) {
                emptyf = myfopen(FILE_EMPTY,"a");
                if (emptyf==NULL) {
                        filemsg("Cannot append to %s\n",FILE_EMPTY);
                        return TRUE;
                }
        }
        fprintf(emptyf,"%s %s\n",name,mod);
        empty_add_mem(name,mod);
        return TRUE;
}
int empty_add_mem(char *name, char *mod)
{
        Erec *r;
        int h;
        h = str_hash(name,ENHASH);
        r = mykeep(sizeof(Erec));
        r->name = mystrkeep(name);
        r->mod = *mod;
        r->next = empty[h];
        empty[h] = r;
        return TRUE;
}
int empty_isempty(char *s)
{
        Erec *r = empty_find(s);
        if (r==NULL) return FALSE;
        if (r->mod=='z') return FALSE;
        return TRUE;
}
int empty_unempty(char *s)
{
        /* Find group, add it to real groups, 
        set it as non empty*/
        int g;
        char m[10];
        Erec *r = empty_find(s);

        if (s==NULL) return FALSE;
        g = group_find(r->name);
        imsg("Setting empty group to unhidden (%s)\n",s);
        m[0] = r->mod; m[1] = 0;
        if (g<0) {
                group_new(s,"unempty",m,"");
        } else {
                group_setmod(g,m);
        }
        r->mod = 'z';
        empty_changed = TRUE;
        return TRUE;
}
Erec *empty_find(char *xname)
{
        int h;
        Erec *r;
        char name[BFSZ];
        ncpy(name,xname,BFSZ-1); strlwr(name);

        h = str_hash(name,ENHASH);
        for (r=empty[h];r!=NULL;r=r->next) {
                if (strcmp(r->name,name)==0) return r;
        }
        return NULL;
}
int empty_close(void)
{
        if (emptyf==NULL) return TRUE;
        fclosep(emptyf); emptyf = NULL;
        return TRUE;
}
int empty_read(void)
{
        FILE *f;
        char mod[BFSZ],group[BFSZ];
        char *s;
        char bf[BFSZ];
        int nempty = 0;
        f = fopen(FILE_EMPTY,"r");
        if (f==NULL) return FALSE;
        imsgf("Reading (%s) ...\n",FILE_EMPTY);
        for (;!feof(f);) {
                if (fgets(bf,BFSZ-10,f)==NULL) break;
                s = strtok(bf," \t\n\r"); if (s==NULL) continue;
                ncpy(group,s,BFSZ-1);
                s = strtok(NULL," \t\n\r"); if (s==NULL) continue;
                ncpy(mod,s,BFSZ-1);
                empty_add_mem(group,mod); nempty++;
        }
	fclosep(f);
        imsgf("Read info on %d empty groups\n",nempty);
        return TRUE;
}
int empty_n(void)
{
        /* If empty items 'deleted', write new empty.dat file */
        Erec *r;
        int n= 0;
        int i;
        for (i=0; i<ENHASH; i++) {
                r = empty[i];
                for (;r!=NULL; r= r->next) {
                        if (r->mod!='z') n++;
                }
        }
        return n;
}
int empty_write(void)
{
        /* If empty items 'deleted', write new empty.dat file */
        Erec *r;
        char bf[BFSZ];
        int i;
        int nsent = 0;
        FILE *f;
        empty_close();
        if (!empty_changed) return  TRUE;
        imsg("Writing %s\n",FILE_EMPTY);
        f = fopen(FILE_EMPTY_NEW,"w");
        if (f==NULL) {
                filemsg("Cannot append to %s\n",FILE_EMPTY_NEW);
                return FALSE;
        }
        empty_changed = FALSE;
        for (i=0; i<ENHASH; i++) {
                r = empty[i];
                for (;r!=NULL; r= r->next) {
                        if (r->mod != 'z') {/* No longer empty */
                         if (fprintf(f,"%s %c\n",r->name,r->mod)<0) {
                                filemsg("Cannot write new empty groups file\n");
                                fclosep(f);
                                return FALSE;
                         }
                         nsent++;
                        }
        
                }
        }
        if (fclosep(f)!=0) {
                filemsg("Cannot write new empty groups file\n");
                return FALSE;
        }
        strcpy(bf,FILE_EMPTY);
        remove(bf);
        rename(FILE_EMPTY_NEW,bf);
        imsg("Wrote %s, %d lines\n",FILE_EMPTY,nsent);
        return TRUE;
}
int empty_unhide(int chan, char *wild)
{
        /* If empty items 'deleted', write new empty.dat file */
        Erec *r;
        char bf[BFSZ];
        int ndone = 0;
        int i;
        if (strlen(wild)==0) wild = "*";
        for (i=0; i<ENHASH; i++) {
                r = empty[i];
                for (;r!=NULL; r= r->next) {
                 if (r->mod!='z') {
                        if (group_match(wild,r->name)) {
                                empty_unempty(r->name);
                                ndone++;
                        }
                 }
                }
        }
        lprintf(bf,BFSZ-1,"unhiding %d empty groups\r\n",ndone);
        chan_write_s(chan,bf);
        return TRUE;
}
int empty_hide(int chan, int n)
{
        char bf[BFSZ];
        char m[10];
        int i;
        int ndel=0;

        lprintf(bf,BFSZ-1,"Currently %d empty groups are hidden\r\n",empty_n());
        chan_write_s(chan,bf);
        for (i=0; i<ngroups; i++) {
          if (groups[i]->range <= 0) {
            if (groups[i]->lastmsg>=n) {
                m[0] = groups[i]->mod; m[1] = 0;
                empty_addif(group_name(i),m);
                imsg("removing empty group (%s) \n",group_name(i));
                ndel++;
                group_remove_now(i);
            }
          }
        }
        lprintf(bf,BFSZ-1,"Hiding %d new empty groups\r\n",ndel);
        chan_write_s(chan,bf);
        return TRUE;
}
int empty_count(int chan, int n)
{
        char bf[BFSZ];
        int i;
        int ndel=0;

        lprintf(bf,BFSZ-1,"Currently %d empty groups are hidden\r\n",empty_n());
        chan_write_s(chan,bf);
        for (i=0; i<ngroups; i++) {
          if (groups[i]->range <= 0) {
            if (groups[i]->lastmsg>=n) {
                ndel++;
            }
          }
        }
        lprintf(bf,BFSZ-1,"%d empty groups  are not hidden\r\n",ndel);
        chan_write_s(chan,bf);
        return TRUE;
}
#define FILE_AGE file_work("active.age")
/* gage_n gage_avg gage_hash */ 
void gage_init(void)
{
        FILE *f;
        char bf[BFSZ];
        int n,gid;
        char *s;
        int g;
        double avg;

        int i;

        for (i=0; i<max_groups; i++) {
                gage_n[i] = 0;
                gage_avg[i] = 0;
        }
        f = fopen(FILE_AGE,"r");
        if (f==NULL) {
                fimsg("Could not read (%s) not serious\n",FILE_AGE);
                return;
        }

        for (;!feof(f);) {
                if (fgets(bf,BFSZ-10,f)==NULL) break;
                s = strtok(bf," \n"); if (s==NULL) continue;
                gid = atoi(s);
                s = strtok(NULL," \n"); if (s==NULL) continue;
                n = atoi(s);
                s = strtok(NULL," \n"); if (s==NULL) continue;
                avg = atof(s);
                g = gid_to_group(gid);
                if (g<0) continue;
                gage_n[g] = n;
                gage_avg[g] = avg;
        }
        fclosep(f);
        gage_spread();
}
int get_sort_piles_nf(void) /* Number of open file handles */
{
	return expire_npiles_nf();	
}

int get_sort_piles_np(void) /* Number of piles */
{
        int n;
        if (expire_ispile()) return expire_npiles();
        n = atoi(config_get(C_DB_SORT_PILES));
        if (n==0) n = 5; /* Should allow up to 30 */
        if (n>30) n = 30;
        return n;
}

/*

age[1-60]

Add up the total for each day, then pick the 'n' most popular
days.

*/
#define MAXDAYS 60
void gage_spread(void)
{
        int i,j;
        int a,maxi;
        int n,changed,x;
        int np=0;
        int np2;
        int table[MAXDAYS+1];
        int p2[MAXDAYS+1];
        int p[MAXDAYS+1];
        int piles[MAXDAYS+1];

        if (expire_ispile()) return;

        n = get_sort_piles_np();
        for (i=0; i<MAXDAYS; i++) piles[i] = 0;

        for (i=0; i<ngroups; i++)  {
                a = (int) (gage_avg[i]+.5);
                if (gage_n[i]==0) a = MAXDAYS-1;
                if (a>MAXDAYS-1) a = MAXDAYS-1;
                piles[a]++;
        }
        for (i=0; i<n; i++) {
                maxi = 0;
                for (j=0; j<MAXDAYS; j++) {
                        if (piles[j]>piles[maxi]) maxi = j;
                }
                piles[maxi] = 0;
                p[np++] = maxi;
        }
        /* Sort 'p' and create a table */

        for (changed=TRUE;changed;) {
                changed = FALSE;
                for (i=0; i<np-1; i++) {
                        if (p[i]>p[i+1]) {
                                x = p[i];
                                p[i] = p[i+1];
                                p[i+1] = x;
                                changed = TRUE;
                        }
                        
                }
        }
        p2[0] = p[i];
        for (np2=1,i=0; i<np; i++) {
                if (p2[np2-1]==p[i]) continue;
                p2[np2++] = p[i];
        }
        for (j = 0; j<MAXDAYS; j++) {
                for (i=0; i<np2; i++) {
                        if (j<=p2[i]) {goto xx;} 
                }
                i = np2-1;
xx:             table[j] = i;
        }
      /*  for (i=0; i<MAXDAYS; i++) {
                imsg("Table [%d] = %d\n",i,table[i]);
        }
	*/
        for (i=0; i<ngroups; i++) {
                a = (int) (gage_avg[i]+.5);
                if (a>=MAXDAYS) a = MAXDAYS-1;
                if (gage_n[i]==0) a = MAXDAYS-1;
                gage_h[i] = table[a];
        }

}
int gage_get(int g)
{
        return (int) (gage_avg[g] + .5);
}
int gage_hash(int g)
{
        if (gage_h[g]==0) {
		gage_setpile(g);
                imsg("gage_hash zero? %s is now %d\n",group_name(g),gage_h[g]);
        }
        return gage_h[g];
}
void gage_sethash(int g, int x)
{
        gage_h[g] = x;
}
void gage_add(int g, int age)
{
        int n;
        if ((g<0)|| (g>=max_groups)) return;
        age = age / (60*60*24);
        n = gage_n[g];
        if (n<100) n++;
        if (age<0) age = 0;
        gage_n[g] = n;
        gage_avg[g]  = gage_avg[g] * ((n-1.0)/n) + (1.0/n)*age;
}
void gage_write(void)
{
        FILE *f;
        int i;
        GRec *r;


        f = fopen(FILE_AGE,"w");
        if (f==NULL) {
                filemsg("Could not write to (%s) \n",FILE_AGE);
                return;
        }

        for (i=0; i<ngroups; i++) {
                r = groups[i];
                if (r==NULL) continue;
                fprintf(f,"%d %d %g\n",r->groupid,gage_n[i],gage_avg[i]);
        }
        fclosep(f);
        gage_spread();
}
int expire_pile_unread(int i);
int expire_unread_pile(void);
/* Choose which pile this item should be in, if expire_any true then
choose shortest pile, otherwise choose longest pile */
int db_ispile(void);
static int gage_next;
void gage_choose(int grp[],int ngrp)
{
        int i;
        int best = -1;
        int h;

	if (!db_ispile()) return;
 
	if (config_true(C_EXPIRE_ANY)) {
                for (i=0; i<ngrp; i++) {
                        h = gage_hash(grp[i]) ;
                        if (h>best) {best = h;}
                }
                gage_next = best;
        } else {
                best = get_sort_piles_np()-1;
                for (i=0; i<ngrp; i++) {
                        h = gage_hash(grp[i]) ;
                        if (h<best) {best = h;}
                }
                gage_next = best;
        }
	if (expire_unread_pile()>0) {
                for (i=0; i<ngrp; i++) {
			if (group_iscached(grp[i])) goto isok;
                }
		gage_next = expire_unread_pile();
	}
isok:;
}
int gage_getnext(void)
{
	int nf = expire_pile_nopen(gage_next);
	static int z;
	z++;
	return expire_pile_base(gage_next) + (z % nf);
}

static int towrite;
int group_write_changed_later(void)
{
        towrite = lib_gmtime()+1;
        return TRUE;
}
int group_backgnd(void)
{
        if (towrite>0) if (lib_gmtime()>=towrite) {
                group_write_changed();
                towrite = 0;
        }
        return TRUE;
}
#define FILE_GLOG file_work("group.log")
void glog(char * arg_list, ...)
{
        va_list arg_ptr;
        char *format;
        int len;
	static FILE *f;
	char output[4000];
        va_start(arg_ptr, arg_list);
        format = arg_list;
        len = vsnprintf(output, BFSZ-1, format, arg_ptr);
        output[BFSZ-1] = 0;
	if (f==NULL) {
		if (lib_length_file(FILE_GLOG)>1000000) {
			remove(FILE_GLOG);	
		}
		f = myfopen(FILE_GLOG,"a");
	}
	if (f!=NULL) fprintf(f,"%s",output);
}


struct WriteData {int nread; int rsize; int gid;} wdata;
char *file_nread(int n)
{
	char bf[BFSZ];
	static char fname[BFSZ];
	lprintf(bf,BFSZ-1,"nread_%d.dat",n);
	ncpy(fname,file_work(bf),200);
	return fname;
}
void group_write_nread(void) 
{
	static int last;
	int t = lib_gmtime();
	if ((t-last)<(60*60)) return;
	last = t;
	group_write_nread_now();
}
void group_write_nread_now(void) 
{
	static int last;
	int i;
	FILE *f;
	int n=0;
	imsg("Writing list of read groups to a file for this slave\n");	
	remove(file_nread(slave_n()));
	f = fopen(file_nread(slave_n()),"wb");
	if (f==NULL) return;

	for (i=0; i<ngroups; i++) {
		if (groups[i]==NULL) continue;
		if (groups[i]->nread==0) continue;
		wdata.nread = groups[i]->nread;
		wdata.rsize = groups[i]->rsize;
		wdata.gid = groups[i]->groupid;
		fwrite(&wdata,sizeof(wdata),1,f);
		n++;
	}
	imsg("Wrote %d records\n",n);
	fclosep(f);
}

void group_read_nread(void)
{
	int i;
	if (atoi(config_get(C_SLAVE_N))<2) return;
	if (is_slave()) return;
	/* Write out totals for us */
	group_write_nread_now(); 
	/* Read  totals in for each slave */
	for (i=0; i<ngroups; i++) {
		if (groups[i]==NULL) continue;
		groups[i]->rsize = 0;
		groups[i]->nread = 0;
	}
	for (i=0; i<atoi(config_get(C_SLAVE_N)); i++) {
		group_read_rsize(i);
	}
}
void group_read_rsize(int n)
{
	FILE *f;
	int g;
	f = fopen(file_nread(n),"rb");
	if (f==NULL) return;
	for (;!feof(f);) {
		if (fread(&wdata,sizeof(wdata),1,f)<1) break;
		g = gid_to_group(wdata.gid);
		if (g<0) continue;
		if (g>group_n()) continue;
		if (groups[g]==NULL) continue;
		groups[g]->rsize += wdata.rsize;
		groups[g]->nread += wdata.nread;
	}
	fclosep(f);
}
void group_reset_rsize(void)
{
	int i;
	if (atoi(config_get(C_SLAVE_N))<2) return;
	if (is_slave()) return;
	for (i=0; i<ngroups; i++) {
		if (groups[i]==NULL) continue;
		groups[i]->rsize = 0;
		groups[i]->nread = 0;
	}
	group_read_rsize(0);
}

int group_matchmod(int chan,char *fname)
{
	char bf[BFSZ],out[BFSZ],*p,*gname,gmod;
	FILE *f;
	int g,m,fixit,n=0;
	if(f=fopen(fname,"r")) {
		while(fgets(bf,BFSZ-1,f)) {
			fixit=FALSE;
			if(p=strrchr(bf,' ')) gmod=*(p+1);
			else continue;
			if(!(gname=strtok(bf," "))) continue;
			g=group_find(gname);
			if(g>=0) {
				m=group_ismod(g);
				if(gmod=='y' && m==TRUE) fixit=TRUE;
				else if(gmod=='m' && m==FALSE) fixit=TRUE;

				if(fixit) {
					lprintf(out,BFSZ-1,"Correcting moderation setting for %s to %c)\r\n",gname,gmod);
					chan_write_s(chan,out);
					imsg(out);
					group_setmod(g,&gmod);
					n++;
				}
			} else {
				imsg("group_matchmod - Group %s does not exist, ignoring\r\n",gname);
			}
		}
		fclose(f);
		return n;
	}
	lprintf(out,BFSZ-1,"group_matchmod - Could not open %s (%s)\r\n",fname,strerror(errno));
	chan_write_s(chan,out);
	imsg(out);
	return -1;
}