#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <ctype.h>
#include "mutex.h"
#include "emsg.h"
#include "str.h"
#include "cmd.h"
#include "sary.h"
#include "group.h"
#include "dnews.h"
#include "lib.h"
#include "item.h"
#include "chan.h"
#include "hist.h"
#include "expire.h"
#include "feed.h"
#include "post.h"
#include "suck.h"
#include "chan_dmy.h"
#include "db.h"
#include "pull.h"
#include "ml.h"
#include "xhash.h"
#ifdef KEYLIB
#include "keylib.h"
#endif

#undef mymalloc
#undef myfree
#define mymalloc(x) mymalloc2(x,M_SUCK)
#define myfree(x) myfree2(x,M_SUCK)

enum {F_IDLE,F_WAIT, F_OPEN,F_OPEN_AUTHUSER,F_OPEN_AUTHPASS,F_OPEN_MODE
        ,F_OPEN2,F_LIST,F_LIST2,F_LIST3,F_DOLIST
        ,F_UPDATE,F_FORGROUP,F_GET,F_GROUPNEXT,F_EXPIRE
        ,F_FEED,F_FEED_OPEN,F_FEED_AUTHUSER,F_FEED_AUTHUSER2
        ,F_FEED_AUTHPASS,F_FEED_MODE    
        ,F_FEED_NEXT,F_FEED_SEND
        ,F_FEED_SEND2,F_FEED_CLOSE
        ,F_MAIL_OPEN,F_MAIL_NEXT,F_MAIL_FROM,F_MAIL_RCPT,F_MAIL_DATA
        ,F_MAIL_SEND,F_MAIL_SEND2,F_MAIL_CLOSE,F_GENERATE
	,F_STREAM_GROUP, F_STREAM_NEXT
	,F_FORGROUP2,F_HEADER_EXPIRE
	,F_STREAM_GROUP2
	,F_OPEN_DOMODE
	,F_OPEN_END
	,F_OPEN_AUTHUSER2
	,F_OPEN_AUTHPASS2


};
char *f_text[]={"F_IDLE","F_WAIT","F_OPEN","F_OPEN_AUTHUSER"
        ,"F_OPEN_AUTHPASS","F_OPEN_MODE"
        ,"F_OPEN2","F_LIST","F_LIST2","F_LIST3","F_DOLIST"
        ,"F_UPDATE","F_FORGROUP","F_GET","F_GROUPNEXT","F_EXPIRE"
        ,"F_FEED","F_FEED_OPEN","F_FEED_AUTHUSER","F_FEED_AUTHUSER2"
        ,"F_FEED_AUTHPASS","F_FEED_MODE"
        ,"F_FEED_NEXT","F_FEED_SEND"
        ,"F_FEED_SEND2","F_FEED_CLOSE"
        ,"F_MAIL_OPEN","F_MAIL_NEXT","F_MAIL_FROM","F_MAIL_RCPT","F_MAIL_DATA"
        ,"F_MAIL_SEND","F_MAIL_SEND2","F_MAIL_CLOSE", "F_GENERATE"
	,"F_STREAM_GROUP", "F_STREAM_NEXT"
	,"F_FORGROUP2","F_HEADER_EXPIRE"
	,"F_STREAM_GROUP2"
	,"F_OPEN_DOMODE"
	,"F_OPEN_END"
	,"F_OPEN_AUTHUSER2"
	,"F_OPEN_AUTHPASS2"
};
void feed_flushuucp(void);


/* Find name of feeder and user/password */
static char *suck_name[50];
static char *suck_user[50];
static char *suck_pass[50];
static char *suck_groups[50];
static char *suck_file[50];
static int nsuck=0;
void suck_convert_file(char *fname);
void suck_convert(void);
int suck_infile(int n, char *gname);
static int list_start;

int suck_bygroup(void);
void xinitem_clear(void);
char *hdr_link(char *feed, char *group, int n);
void suck_write_s(int t,char *bf);


void suck_isat(char *gtime);
void suck_trylater(char *s);
int suck_dontretry(char *s);
int suck_retryfind(char *s);
static int nextloop;
static int expire_pid = -1;

void suck_load(void);
void suck_addfeed(char *s);
void suck_feeder(char *gname);
void suck_feeder_name(char *gname);
void wcache_shrink(void);
int suck_update_groups_nq(int f);

void do_trim(void);
void uucp_suck(void);
void suck_setfeedend(char *s);

void suck_sched(void);
void tell_user(String *xhead, String *body, char *status);
int suck_ismail(void);
void db_kill_lost_bkgnd(void);
void remove_header(String *xhead,char *hname);
void suck_second(void);
char *email_fixn(int upto, char *list);
char *suck_getnext(void);  /*Pop item off bottom of queue.*/
void gen_line(char *rec);
int suck_queue_feed(char *grp, int pri,  int feed);
void gen_free(void);
void do_generate(void);
static int nntp_bad,nntp_err;
int suck_post_moderated(String *anitem);

static int state,endwait,failwait,setchan;
static int t;
static int item_i,oldlimit;
static String *newlist;
static char nntp_status[1000];
static int open_end;
/* Naughty use of fixed size array. */

static char pop_feed;
static int nque;
static int sched_expire,sched_update;

char *email_fix(char *s);
void gen_addline(char *rec);
void do_names(void);
static int xstat_start,stat_n;
static double stat_size;
static int diskfull=FALSE;
void suck_posts(void);
int feed_reader(void);

static char *gname;
static int isheader;
static char **que;
static int *que_feed;
static int *que_pri;
static int doskip = TRUE;
char *chan_showcon(void);


/* master command prototypes */
static int spawned_expire = FALSE;
static int isexpire;
void do_master_command(char *cp);

void do_reuse_buckets(void);
void do_history_flush_pause(void);
void do_reread_history(void);
void do_stop_write_dbi_index(void);
void do_reread_dbi_index(void);
void do_expire_done(void);
/*void do_slave_flush_unpause(void);*/
void do_slave_flush_pause(void);

struct master_command
{
    char *cmd;
    int len;
    void (*func)(void);
};

static struct master_command mastcmds[] =
{
    {"reuse_buckets",	    13,do_reuse_buckets},
    {"history_flush_pause", 19,do_history_flush_pause},
    {"reread_history",      14,do_reread_history},
    {"stop_write_dbi_index",20,do_stop_write_dbi_index},
    {"index_pause",11,do_index_pause},
    {"index_unpause",13,do_index_unpause},
    {"reread_dbi_index",    16,do_reread_dbi_index},
    {"expire_done",         11,do_expire_done},
    /*{"slave_flush_unpause", 19,do_slave_flush_unpause},*/
    {"slave_flush_pause",   17,do_slave_flush_pause},

    {NULL,0,NULL}
};

/* master command functions */

int suck_expire_only(void)
{
   	imsg("expirer: expire processing started\n");
	if (state==F_EXPIRE) {emsg("Expire already running\n"); return FALSE;}
	expire_groups("");
	expire_clear();
	state=F_EXPIRE;
	while (state != F_IDLE)
	{
	    sched_expire = FALSE;
	    expire_do();
	    expire_write_current_status("");
	    if (sched_expire) state=F_EXPIRE;
	}
	tell_master("expire_done");
	imsg("expirer: expire processing done!\n");
	return TRUE;
}
int spawn_expire_dieing(char *reason)
{
	char bf[BFSZ];
	if (get_expire_mode()) {  /* We are a spawned expire process */
	    emsg("Attempting to tell master that we have finished\n");
	    readonly_unpause();
            tell_master("history_flush_unpause");
	    lprintf(bf,BFSZ-1,"FATAL ERROR during spawned expire (%s)\n",reason);
	    expire_write_current_status(bf);
	    tell_master("expire_done");
	}
	return TRUE;
}

void suck_set_expire_pid(int procid) 
{
    expire_pid = procid;
} 

int spawn_kill(int x);
int suck_expire_exit(void) 
{
    if (expire_pid!=-1) {
	imsg("spwan_kill (%d)\n",expire_pid);
	spawn_kill(expire_pid);    
    }
    return TRUE;
}

void do_master_command(char *cp) {
    int i;
    
    for (i=0;mastcmds[i].cmd!=NULL;i++)
    	if (strncmp(cp,mastcmds[i].cmd,mastcmds[i].len)==0) break;
    if (mastcmds[i].cmd==NULL) {
	emsg("master: command not valid (%.40s)\n",cp);
	signal_command_done();
    }
    else {	
	imsg("master: initiating expire command function (%.40s)\n",mastcmds[i].cmd);
	mastcmds[i].func();
    }
}

void do_reuse_buckets(void) {
    db_reuse_buckets();
    signal_command_done();
}

void do_history_flush_pause(void) {
    /* flush history and close history files and pause */
    imsgf("expire: external expire asked master to close history file\n");
    hist_flush();
    hist_close();
    readonly_pause(); /* Tell all reading servers to pause and close files */
    /* Added by chrisp as seemed to be missing */
    slave_flush(TRUE);

    signal_command_done();
    signal_wait(); /* Wait till we are told to continue;*/
    imsg("master: continuing after history_flush_unpause command - rereading history \n");
    readonly_unpause(); /* Tell all reading servers to pause and close files */
    hist_reload();
    slave_flush(FALSE);
    imsg("master: reread of history completed\n");
}

void do_reread_history(void) 
{
    /* re-read history files */
    imsg("master: obsolete command, should not occur now\n");
    hist_reload();
    signal_command_done();
}

void do_stop_write_dbi_index(void) 
{
    db_set_stopdbi(TRUE);
    db_flush2();
    db_set_in_resort();
    signal_command_done();
}
void do_index_pause(void) 
{
    imsg("expire: Told us to stop writing to the dbi.idx file\n");
    db_set_stopdbi(TRUE);
    db_clear_cache("index_pause");
    db_flush2();
    db_setpatching(TRUE);
    db_set_in_resort();
    signal_command_done();
}
void db_index_unpause(void);
void do_index_unpause(void) 
{
    db_set_stopdbi(FALSE);
    db_flush2();
    db_index_unpause();
    signal_command_done();
}

void do_slave_flush_pause(void) {
    db_flush2();
    db_slave_flush_pause();
    db_master_dbi_close();
    signal_command_done();
    signal_wait(); /* Wait till we are told to continue;*/
    db_slave_flush_unpause();
    imsg("master: received slave_flush_unpause command\n");
}

/*void do_slave_flush_unpause(void) {
    db_slave_flush_unpause();
    signal_command_done();  
    
          if (fdbi>=0) dbi_close(fdbi); 
    fdbi = -1;

}*/

void do_reread_dbi_index(void) 
{
    db_set_stopdbi(FALSE);
    db_flush2();
    db_clear_in_resort();
    db_clear_cache("reread_dbi_index");
    signal_command_done();
}

void suck_set_expire_pid(int procid);
void do_expire_done(void) 
{
    imsg("do_expire_done called\n");
    xtell_check(); /* Patch any xover files not yet patched */
    imsg("do_expire_done db_flush2\n");
    db_flush2();
    imsg("do_expire_done signal_command done\n");
    signal_command_done();
    imsg("do_expire_done manager report\n");
    dnews_manager_report(expire_subj());
    imsg("do_expire_done sleep\n");
    /* re-enable spawning of external expires */
    lib_sleep(1);
    imsg("do_expire_done expire_exit\n");
    suck_expire_exit();	/* kill the expire process if it didn't die already */
    imsg("do_expire_done Set expire pid\n");
    spawned_expire = FALSE;
    suck_set_expire_pid(-1); /* assuming the spawned expire has closed by now */
    imsg("Check buckets in case expire deleted something\n");
    db_check_buckets_noerror();
    imsg("Checking index so item ranges are correct\n");
    db_bucket_ranges_clear();
    db_check_index();
    
    stats_clear();
    feed_statsclear();

    xtell_master_reset();

    if (hist_ispage()) hist_expire_oldest_file();

    imsg("do_expire_done FINISHED\n");
}

int suck_spawned_expire(void) {
    return spawned_expire;
}
int ml_newday(void);

int show_chan_status(void);
void tomail_idle(void);
void getmail_idle(void);
void suck_idle(void)
{
	time_t now;
        struct tm *t; 
	struct tm atm;
	static int last,last_status;

        doskip = !config_true(C_SUCK_NOSKIP);
	stat_start(X_NEWDAY);
	ml_newday();
	stat_end(X_NEWDAY);
	stat_start(X_XHASH);
	xhash_close();
	stat_end(X_XHASH);
	stat_start(X_CHECK);
	xtell_check();
	stat_end(X_CHECK);
	stat_start(X_TOMAIL);
	tomail_idle();
	stat_end(X_TOMAIL);
	stat_start(X_GETMAIL);
	getmail_idle();
	stat_end(X_GETMAIL);
        now = time(NULL);
        t = safe_localtime(&now,&atm);
	if ((now-last_status)>60) {
		last_status = now;
		if(!is_readonly()) show_chan_status();
		if (atoi(config_get(C_INPUT_WINDOW))>0) nntp_input_window(atoi(config_get(C_INPUT_WINDOW)));
	        sponser_init(); /* Fetch any sponser changes this week */	
	}

	if (config_true(C_DISKIO_MIN)) return;

	if ((t->tm_min % 5) == 0) if (last!=t->tm_min) {
		last = t->tm_min;
		imsg("stats: %s\n", chan_showcon());

	}
}
int show_chan_status(void)
{
	String *str;
	char bf[BFSZ];
	str = dnews_status();
	
	lprintf(bf,BFSZ-1,"DNEWS %s %s\r\n",DNEWS_VERSION,lib_strdate());
	str_adds(str,bf,strlen(bf));	
	remove(file_log("status.log"));
	str_write(str,file_log("status.log"));

	return TRUE;
}
int suck_malloc_groups(void)
{
	que_feed = (int *) mymalloc(max_groups *sizeof(int));
	que_pri = (int *) mymalloc(max_groups *sizeof(int));
	que = (char **) mymalloc(max_groups *sizeof(char *));
	return TRUE;
}
int suck_expire_internal(void)
{
	imsg("expire: starting one in this process\n");
	    if (state==F_EXPIRE) {emsg("Already doing expire\n"); return FALSE;}
	    expire_groups("");
	    expire_clear();
	    sched_expire = /* isexpire = */ TRUE;
	    return TRUE;
}

int suck_expire_groups(char *s)
{
	imsg("Starting expire_groups to delete (%s) \n",s);
	suck_expire_internal();
	expire_groups(s);
	return TRUE;
}

int suck_expire(void)
{       
	if (is_readonly()) {
		imsg("slave: can't run an expire\n");
		return FALSE;
	}
	if (is_spawn_expire())	{
	    if (spawned_expire) {
		imsg("master: external expire process already spawned\n");
	    }
	    else {
		xtell_master_reset();
		expire_set_start();
		imsg("spawning external expire process\n");
		remove_tellmast_dat();
	        suck_set_expire_pid(spawn_dnews_expire());
		if (expire_pid==-1)
		    emsg("could not spawn expire process\n");
	        else {
		    imsg("external expire process successfully spawned (pid=%d)\n",expire_pid);
		    spawned_expire = TRUE;
		}
	    }
	}
	else
	{
		imsg("expire: starting one in this process\n");
	    if (state==F_EXPIRE) {emsg("Already doing expire\n"); return FALSE;}
	    expire_groups("");
	    expire_clear();
	    sched_expire = /* isexpire = */ TRUE;
	}
        return TRUE;
}
int is_expiring(void)
{
	if (state==F_EXPIRE) return TRUE;
	if (spawned_expire) return TRUE;
	return FALSE;
}
void expire_setdoxover(void);
int suck_doxover(void)
{       
	if (state==F_EXPIRE) {emsg("Already doing expire\n"); return FALSE;}
        sched_expire = TRUE;
        expire_setdoxover();
        return TRUE;
}
int suck_purgelost(void)
{       
	if (state==F_EXPIRE) {emsg("Already doing expire\n"); return FALSE;}
        sched_expire = TRUE;
        expire_setpurge();
        return TRUE;
}
int suck_refeed(int f)
{       
	if (state==F_EXPIRE) {emsg("Already doing expire\n"); return FALSE;}
        sched_expire = TRUE;
        expire_setrefeed(f);
        return TRUE;
}
int suck_purgelinks(void)
{       
	if (state==F_EXPIRE) {emsg("Already doing expire\n"); return FALSE;}
        sched_expire = TRUE;
        expire_setlinks();
        return TRUE;
}
int suck_addlost(void)
{       
	if (state==F_EXPIRE) {emsg("Already doing expire\n"); return FALSE;}
        sched_expire = TRUE;
        expire_setpurge();
        return TRUE;
}
int suck_resort(void)
{       
	if (state==F_EXPIRE) {emsg("Already doing expire\n"); return FALSE;}
        sched_expire = TRUE;
        expire_setresort();
        return TRUE;
}
int suck_upgrade(void)
{       
	if (state==F_EXPIRE) {emsg("Already doing expire\n"); return FALSE;}
        sched_expire = TRUE;
        expire_setupgrade();
        return TRUE;
}
int suck_reindex(void)
{       
	if (state==F_EXPIRE) {emsg("Already doing expire\n"); return FALSE;}
        sched_expire = TRUE;
        expire_setreindex();
        return TRUE;
}
int suck_findlost(void)
{       
	if (state==F_EXPIRE) {emsg("Already doing expire\n"); return FALSE;}
        sched_expire = TRUE;
        expire_setfindlost();
        return TRUE;
}
void suck_expire_end(void)
{
        state = F_IDLE;
}
static int inlist;
static int feed_nocheck;
int suck_update_n(int suckn)
{
	int i;
	int nadded=0;

	if (is_readonly()) return TRUE;

	if (suckn<0 || suckn>40 || suckn>=nsuck) {
		emsg("Suckn out of range %d\n",suckn);
		return TRUE;
	}
	if (suck_groups[suckn]==NULL) {
		emsg("suck_update_n for suck that doesn't exist (%d) \n",suckn);
		return TRUE;
	}
	dmsg("suck: update%d\n",suckn);
	/* Special case of empty queue, just chuck them in */
	if (nque<5) feed_nocheck = TRUE;
	imsg("suck: suck%d %.80s\n",suckn+1, suck_groups[suckn]);
        for (i=0; i<group_n(); i++) if (group_iscachedreal(i)) if (!group_isdeleted(i)) {
                if (suck_infile(suckn,group_name(i))) if (group_match(suck_groups[suckn],group_name(i))) {
			if (pull_true(suckn)) {
				if (strcmp(group_name(i),"alt.binaries.sounds.country.mp3")==0) sucklog("pull: Adding %s to pull queue %d\n",group_name(i),suckn);
				pull_add(suckn,i,FALSE);
			} else {
				suck_queue_feed(group_name(i),1,suckn);
                		group_setcatchup(i,suckn,group_feedend(i,suckn));
			}
			nadded++;
		}
        }
	feed_nocheck = FALSE;
	if (suckn>0) imsg("suck: UPDATE_AT%d  Added %d groups to queue\n",suckn+1,nadded);
	return TRUE;
}
void suck_que_agroup(int grp)
{
	if (suck_groups[0] == NULL) return;
	if (strlen(config_get(C_NNTP_SUCK1))==0) return;
	if (suck_infile(0,group_name(grp)))
          if (group_match(suck_groups[0],group_name(grp))) {
		if (!pull_suck(0,grp,FALSE)) {
			suck_queue_feed(group_name(grp),1,0);
		}
    	}
}

int suck_update_cached(void)
{       
        if (suck_bygroup()) return TRUE;
        return suck_update_groups_n(0);
}
static int trim_only;
int suck_set_trimonly(int x)
{
	trim_only = TRUE;
	return TRUE;
}	

static int get_n;
static int intrim = FALSE;
int suck_update_groups_n(int n)
{
	if (n>=nsuck) {
		emsg("Invalid suck feed number %d\n",n);
		return TRUE;
	}
	get_n = n;
        sched_update = TRUE;
        inlist = TRUE;
        return TRUE;
}
static int header_next;
static char active_host[BFSZ];
static int getactive;
int suck_getactive(char *s)
{
	getactive = TRUE;
	strcpy(active_host,s);
	return TRUE;
}
static int trim_n;
static int instat,stat_loop,infindfirst,infind_loop;
static char match_site[BFSZ];
static char match_groups[BFSZ];
int suck_matchsite(char *site, char *groups)
{	
	ncpy(match_site,site,BFSZ-1);
	ncpy(match_groups,groups,BFSZ-1);
	inlist = TRUE;
	intrim = TRUE;
        sched_update = TRUE;
	return TRUE;
}

int suck_trim_groups_n(int n)
{
	if (n>=nsuck) {
		emsg("Invalid suck feed number %d\n",n);
		return TRUE;
	}
	get_n = n;
        sched_update = TRUE;
        inlist = TRUE;
	intrim = TRUE;
	trim_n = n;
        return TRUE;
}
static int donames;
int suck_update_names(int x)
{       
        int r = suck_setstate(F_LIST);
	if ((x-1)>=nsuck) {
		emsg("Invalid suck feed number %d\n",x);
		return TRUE;
	}
	get_n = x-1;
	if (get_n<0) get_n = 0;
        if (r) donames = TRUE;
        return r;
}
int suck_setstate(int s)
{
        if (state!=F_IDLE) return FALSE;
        state = s;
        return TRUE;
}
int suck_dofeed(char *feedname)
{
        if (!feed_start(feedname)) return FALSE; /*Nothing to do for some reason*/
        state = F_FEED;
        return TRUE;
}
int suck_domail(char *feedname)
{
        if (!feed_start(feedname)) return FALSE; /*Nothing to do for some reason*/
        state = F_MAIL_OPEN;
        return TRUE;
}
void docheck(int nextf, int nextc, int failf);
void docheck(int nextf, int nextc, int failf)
{
        chan_setstate(t,C_CHECK);
        state = F_WAIT;
        failwait = failf;
        endwait = nextf;
        setchan = nextc;
}
static int doexit;
void suck_setexit(void)
{
        doexit = TRUE;
}
void show_stack(void)
{
        static int sz;
        if (dnews_stackused()>sz) {
                sz = dnews_stackused();
                dmsg("lib: Stack used %d bytes\n",sz);
        }            
}
static int gg,gnum;
static int isafeed = FALSE;
static int isamail = FALSE;
static String *xhead,*xbody;
static int inactive;
static int just_started;
static char lastsuck[BFSZ];
static char lastsuck2[BFSZ];
static char cur_feeder[BFSZ],cur_user[BFSZ],cur_pass[BFSZ],cur_open[BFSZ];
static int liststart;
static int liststart_g;
static int suck_batchn,suck_batchsize;
void fts_background(void);
void id_background(void);
static xgroup_end;
#define timecheck(name) if ((lib_gmtime()-xnow)>2) wmsg("slow: %s %d seconds\n",name,lib_gmtime()-xnow); xnow = lib_gmtime();
static int  item_end;
int bsort_idle(void);
void ncmd_settiming(int x);
void getmail_update(void);

static char cur_group[BFSZ];
void db_srt_flush(void); 
void slave_isopen(void);
void suck_do(void)
{
        char bf[1000];
	char bf2[BFSZ];
        int ok;
	double oldp;
        char rep_id[200],rep_from[200],rep_subj[200];
        static int fend;
        static int isopen,done_helo;
        static int gtotal;
	static int ispostonly;
        static int time_flush,time_fast,time_flush2,time_flush3,time_flush4,time_two;
        double elapsed;
        String *xstr;
	static int loaded;
	static int tout,i;
	static int last_sec;
	static int toi;
	char *cp;
	char mailat[BFSZ];
	int autoex;
	int xnow = lib_gmtime();
        if (!loaded) suck_load(); loaded = TRUE;
		
	vtrace();
        show_stack();
#ifdef NETWARE
        if (doexit) dnews_nwexit();
#endif
        if (doexit) dnews_ex("Exit flagged, probably service exit");
	id_background();
	if (xnow!=last_sec) {
		/* if i'm the master and expire spawned */
		vtrace();
	    	if ((!is_readonly())&&spawned_expire) {
		    /* check for a command from the spawned expire process once a second */
		    if ((cp=check_master_command())!=NULL) {
			do_master_command(cp);		
		    }
		}
		thin_release();
		stat_start(X_FLUSH1);
		stat_start(X_XIDLE);
		xidle_do();
		stat_end(X_XIDLE);
		thin_release();
		timecheck("xidle_do");
		if (is_readmaster()) readonly_check();
		suck_convert();
		thin_release();
		timecheck("suck_convert");
		
		vtrace();
		live_idle();

		timecheck("live_idle");
		thin_release();
		stat_start(X_DB_IDLE);
	        db_idle(); /* should really be in new  idle loop*/ 
		stat_end(X_DB_IDLE);
		timecheck("db_idle");
		thin_release();
		if (!lib_ispause()) if (!is_readonly()) uucp_suck();
		timecheck("uucp_suck");
	        suck_sched(); /* Check if we should be doing anything */

		thin_release(); 
		
/**/		suck_idle();
		vtrace();
		thin_release();
		timecheck("suck_sched");

/**/		suck_second();

		timecheck("suck_second");
		thin_release();
		ncmd_backgnd();

		timecheck("ncmd_backgnd");
		group_backgnd();

		thin_release();
		timecheck("group_backgnd");
		pop_backgnd();
		
		thin_release();
		vtrace();
		stat_start(X_SLAVESND);
		
		slave_flushsnd();
		stat_end(X_SLAVESND);
		
		stat_start(X_CANCEL);
		db_cancel_backgnd(); 
		
		thin_release();
		stat_end(X_CANCEL);
		if ((state!=F_EXPIRE) && (!sched_expire)) {
			db_sort_bk();
		} else {
			db_srt_flush(); /* if we are doing one, stop */ 
		}
		dmsg("nocem: Background\n");
		if (!is_readonly()) nocem_background();
		bsort_idle();
		thin_release();
		
		stat_end(X_FLUSH1);
		vtrace();
		hque_bkgnd();
		
		if (is_readmaster()) group_read_changed();
		else {
		  if (config_true(C_SLAVE_BYFILE)) {
			if (is_readonly()) group_read_changed();			
		  } 
		}
		thin_release();
		
		slave_checkpause();
		vtrace();
		last_sec = lib_gmtime();
		
	} else {
	 	if (state==F_IDLE) return; /* Only to idle check 1/second */
	}
        if (xhead==NULL) {
                xhead = str_new(); 
                xbody = str_new();
        }
	if (xnow-time_two>1) {
		time_two = xnow;
		if (is_mastermaster()) suck_posts();
	}

        /* Flush critical stuff every 10 seconds */
        if (xnow-time_fast > 10) {
		stat_start(X_FLUSH10A);
		vtrace();	
		if (!is_readonly()) xidle_do_file();
		stat_end(X_FLUSH10A);
		stat_start(X_FLUSH10B);
		if (!is_readonly()) if (!config_true(C_DISKIO_MIN)) users_idle();
		users_idle2();
		stat_end(X_FLUSH10B);
		stat_start(X_FLUSH10C);
		ncpy(mailat,config_get(C_GETMAIL_AT),100);
		if (strlen(mailat)==0) strcpy(mailat,"*3 *");
		if (conf_isat(mailat)) getmail_update();
		vtrace();
		stat_end(X_FLUSH10C);
		stat_start(X_FLUSH10D);
		if (!is_readonly()) filter_idle();
		vtrace();
		stat_end(X_FLUSH10D);
		stat_start(X_FLUSH10E);
		pull_idle();
		feed_idle();
		stat_end(X_FLUSH10E);
		stat_start(X_FLUSH10F);
		vtrace();   /* IT DIED BETWEEN HERE	*/
		ncmd_settiming(config_true(C_TIMING));

		wcache_shrink();
		db_idle_slow();
		stat_end(X_FLUSH10F);
		stat_start(X_FLUSH10G);
		if ((state==F_IDLE) || (state==F_EXPIRE))  ras_hangup();
		suck_batchn = atoi(config_get(C_SUCK_BATCHN));
		suck_batchsize = atoi(config_get(C_SUCK_BATCHSIZE));
		vtrace();    /* AND HERE               */
                chan_purge2();
		vtrace();
                time_fast = xnow;
                emsg_flush();
		timecheck("emsg_flush");
		tout = atoi(config_get(C_SUCK_TIMEOUT));
		if (tout==0) tout = 300;

		autoex = atoi(config_get(C_AUTOEXPIRE));
		if (!db_ispile())
		  if (autoex>0) if (state!=F_EXPIRE) if (lib_pused() > autoex) {
			wmsg("Auto Expire, usage %d%% is > %d%%\n",lib_pused(),autoex);
			suck_expire();
		}
		mem_check();
		vtrace();
		stat_end(X_FLUSH10G);
		stat_start(X_FLUSH10H);
 		chan_purge_ihave();
		if (is_slave()) slave_isopen(); /* Check the master link is working */
		if (is_readmaster()) group_read_changed();
		stat_end(X_FLUSH10H);
		stat_start(X_FLUSH10I);
		if (is_readmaster()) active_catchup();
		if (is_mastermaster()) suck_posts();
		vtrace();
		if (is_mastermaster()) readonly_readxhit();
		readonly_readsetcached();
		vtrace();
		out_flush();
		
		stat_end(X_FLUSH10I);
        }
	if (suck_batchn==0) suck_batchn = 100;
	if (suck_batchsize==0) suck_batchsize = 5000000;

        if (xnow-time_flush > 60) {
		stat_start(X_FLUSH60);
		
		vtrace();
		item_xover_reset();
		ml_idle();
		group_write_nread(); 
		vtrace();
		ml_flush();
		vtrace();
		spam_postage();
		if (is_slave()) hist_close(); /* Just to make sure */
		readonly_unpause();
		vtrace();
                dmsg("suck: flush 60 \n");
                time_flush = xnow;
                item_flush();
		timecheck("item_flush");
		vtrace();
                dmsg("suck: flush 60 part 2 \n");
                chan_purge();
		vtrace();
		dmsg("suck: Chan purge done\n");
		timecheck("chan_purge");
                group_times_flush();
		vtrace();
		timecheck("times_flush");
                diskuse_flush();
                mem_flush();    /* Free heap space if OS supports this function*/
		vtrace();
		timecheck("mem_flush");
                p_flush(); /* Compress strings */
		timecheck("p_flush");
		vtrace();
		fts_background();
		timecheck("fts_background");
		vtrace();
	        group_used_flush();
		timecheck("group_used_flush");
		vtrace();
		stats_show_changes();
		
		vtrace();
		timecheck("stats_show_changes");
		if (is_readmaster())  hist_catchup();
                if (is_mastermaster()) group_write_changed();
		if (!is_readonly()) db_kill_lost_bkgnd();
		stat_end(X_FLUSH60);
	}
        if ((xnow-time_flush2 > 180) || (is_master() && (xnow-time_flush2 > 60))) {
		strcpy(lastsuck2,"");		
		stat_start(X_FLUSH180);
		vtrace();
                dmsg("suck: flush 300 part 2 \n");
                time_flush2 = xnow;
		vtrace();
                feed_flush();
		timecheck("feed_flush");
                hist_flush(); 
		vtrace();
		timecheck("hist_flush");
		db_flush(); 
		timecheck("db_flush");
		vtrace();
		db_do_later();
		vtrace();
		feed_flushuucp2();
                group_write_changed();
		stat_end(X_FLUSH180);
        }       
        if (xnow-time_flush3 > 800) {
		stat_start(X_FLUSH800);
		vtrace();
		dmsg("suck: suck 20 minutes\n");
		feed_flushuucp2();
                time_flush3 = xnow;
                group_write_changed();
		timecheck("group_write");
		vtrace();
		dmsg("suck: fts_reload\n");
		if (strlen(config_get(C_FTS_DIR))>0) fts_reload();
		dmsg("suck: fts_reload_done\n");
		idx_write_flush("suck.c 800 second flush");
	 	stat_end(X_FLUSH800);
        }      

	if (xnow-time_flush4>(60*120)) {
		time_flush4 = xnow;
		group_write();
	}
        dmsg("lib: suck_do, State {%s}\n",f_text[state]);  
        if (isopen) if (!chan_isopen(t)) {
           imsg("suck: Drat channel closed or failed to open properly or timed out\n");
           if ((state!=F_EXPIRE) && (state!=F_GENERATE)) state = F_IDLE;
           isopen = FALSE;
	   chan_close(t);
	   ncpy(cur_group,"",BFSZ-1);
           lprintf(lastsuck,BFSZ-1,"info: (%s) NNTP closed unexpectedly {%s} check dnews.log\015\012",lib_hour(),cur_open);
           if (gname!=NULL) {
           	gg = group_find(gname);
	   	if (gg>=0) group_setupdate(gg,FALSE);
	   }
	   if (isamail) if (gname!=NULL) suck_trylater(gname);
        }

again:
	if (is_readonly()) return;
	vtrace();
        dmsg("lib: suck_do, State {%s} \n",f_text[state]);  
        switch (state) {
            case F_IDLE:
                donames = FALSE;
                inlist = FALSE;
		inactive = FALSE;
		cur_group[0] = 0;
		if (!sched_expire) cmdq_do();
		if ((sched_expire)&&(is_spawn_expire())&&isexpire) {
		    if (db_ispile()) {
			suck_expire();
			sched_expire = isexpire = FALSE;
		    }
		    else {
			emsg("not using piles - won't spawn expire process\n");
			imsg("switch to piles to enable spawned expiration - initiating normal expire\n");
		    }
		}
                if (sched_expire) { state=F_EXPIRE; isexpire = FALSE; break;}
                if (!lib_checkspace()) 
                	if (!suck_ismail()) break;
			else imsg("Attempting to send email despite disk shortage\n");

                if (sched_expire) {
                        if (isopen) chan_close(t);
                        isopen = FALSE;
                        state = F_EXPIRE;
                        break;
                }
		if (getactive) {
			getactive = FALSE;
                        if (isopen) chan_close(t);
			isopen = FALSE;
			inactive = TRUE;
	                sched_update = FALSE;
	                state = F_OPEN; /*Open the channel*/
	                ncpy(cur_feeder,active_host,100);
	                ncpy(cur_user,"" ,100);
	                ncpy(cur_pass,"" ,100);
	                endwait = F_LIST2;
			break;
			
		}

                gname = suck_getnext();
                if (gname!=NULL) {
                	dmsg("suck: idle {%s} \n",gname);
			ncpy(cur_group,gname,BFSZ-1);
                
                /*feed is put in queue when item arrives for 'autofeed' */
                	if (isafeed) {suck_dofeed(gname); break;}
                	if (isamail) {suck_domail(gname); break;}
		}
                if (gname==NULL) {
                        if (isopen) chan_close(t);
                        isopen = FALSE;
                        if (sched_expire) state = F_EXPIRE;
                        else if (sched_update) {state = F_LIST; inlist=TRUE;}
                        break;
                }
                state = F_UPDATE;       
                goto again;
            case F_WAIT:
                break;
            case F_OPEN:
                if (strlen(cur_feeder)==0) {state = F_IDLE; break; }
                if (isopen) {
                        if (strcmp(cur_open,cur_feeder)==0) {
                                state = endwait;
                                goto again;
                        } else {
                                suck_write_s(t,"quit\015\012");         
                                chan_close(t); 
                                isopen = FALSE;
                        }
                }
                t = chan_open(cur_feeder);
		diskfull = FALSE;
                ncpy(cur_open,cur_feeder,BFSZ-1);
                imsg("suck: Open channel %d to {%s} \n",t,cur_feeder);
                if (t>=0) {
                        isopen = TRUE;
                        chan_settimeout(t,tout); /* INCREASE TO 5 MINUTES */
                        chan_setstate(t,C_CHECK);
                        state = F_WAIT;
                        failwait = F_IDLE;
                        setchan = C_CHECK;
                        /* endwait should be set by caller */
                        if (strlen(cur_user)>0) {
                                open_end = endwait;
                                docheck(F_OPEN_AUTHUSER,C_WAIT,F_OPEN_AUTHUSER);
                        } else  if (feed_reader()) {
				if (!inactive) {
					open_end = endwait;
					endwait = F_OPEN_DOMODE;
				}
                        }
                } else {
                        state=F_IDLE;
                        lprintf(lastsuck,BFSZ-1,"last error: NNTP open of {%s} failed, check dnews.log\015\012",cur_feeder);
                }
                break;
            case F_OPEN_DOMODE: /* Swallow the WELCOME output.*/
		imsg("suck: %s Connected: %.80s\n",cur_open,nocrlf(nntp_status));
		suck_write_s(t,"mode reader\015\012"); 
                chan_setstate(t,C_CHECK);
                state = F_WAIT;
                failwait = open_end;
                endwait = open_end;
                setchan = C_WAIT;
		break;
            case F_OPEN2: /* Swallow the mode reader command output.!!! not used now!!!*/
                chan_setstate(t,C_CHECK);
                state = F_WAIT;
                failwait = open_end;
                endwait = open_end;
                setchan = C_WAIT;
                break;
            case F_OPEN_AUTHUSER:
                if (nntp_bad) emsg("suck: Auth open failed %s \n",nocrlf(nntp_status));
		if (config_true(C_SUCK_NOAUTH1)) {
			state = F_OPEN_MODE;
			goto again;
		}
                lprintf(bf,1000-1,"authinfo user %s\015\012",cur_user); suck_write_s(t,bf);
                docheck(F_OPEN_AUTHPASS,C_WAIT,F_OPEN_AUTHPASS);
                break;
            case F_OPEN_AUTHPASS:
                if (nntp_err!=381) emsg("suck: Auth user failed %s \n",nocrlf(nntp_status));
                lprintf(bf,1000-1,"authinfo pass %s\015\012",cur_pass); suck_write_s(t,bf);
                docheck(F_OPEN_MODE,C_WAIT,F_OPEN_MODE);
                break;
            case F_OPEN_AUTHUSER2:
		imsg("Logging in again after mode reader command\n");
                lprintf(bf,1000-1,"authinfo user %s\015\012",cur_user); suck_write_s(t,bf);
                docheck(F_OPEN_AUTHPASS2,C_WAIT,F_OPEN_AUTHPASS2);
                break;
            case F_OPEN_AUTHPASS2:
                if (nntp_err!=381) emsg("suck: Auth user failed2 %s \n",nocrlf(nntp_status));
                lprintf(bf,1000-1,"authinfo pass %s\015\012",cur_pass); suck_write_s(t,bf);
                docheck(F_OPEN_END,C_WAIT,F_OPEN_END);
                break;
            case F_OPEN_END:
                if (nntp_err!=281) emsg("suck: Auth pass failed %s \n",nocrlf(nntp_status));
                state = open_end;
                chan_setstate(t,C_WAIT);
                break;
            case F_OPEN_MODE:
                if (nntp_err!=281) emsg("suck: Auth pass failed %s \n",nocrlf(nntp_status));
                if (feed_reader()) {
                        lprintf(bf,1000-1,"mode reader\015\012"); suck_write_s(t,bf);
                        if ((strlen(cur_user)>0) && (!config_true(C_SUCK_NOAUTH2))) {
				docheck(F_OPEN_AUTHUSER2,C_WAIT,F_OPEN_AUTHUSER2);
			} else {
	                        docheck(open_end,C_WAIT,open_end);
			}
                } else {
                        state = open_end;
                        chan_setstate(t,C_WAIT);
                }
                break;
            case F_LIST:
                sched_update = FALSE;
                state = F_OPEN; /*Open the channel*/
                ncpy(cur_feeder,suck_name[get_n],100);
                ncpy(cur_user,suck_user[get_n] ,100);
                ncpy(cur_pass,suck_pass[get_n] ,100);
		if (strlen(match_site)>0) {
			if (strcmp(cur_feeder,match_site)!=0) {
				strcpy(cur_user,"");
				strcpy(cur_pass,"");
			}
			ncpy(cur_feeder,match_site,100);
		}

                endwait = F_LIST2;
                break;
            case F_LIST2:
                endwait = F_LIST3; state = F_WAIT;
                failwait = F_IDLE;
                setchan = C_GETLIST;

                chan_setstate(t,C_CHECK);
                if (donames) {
                        wmsg("suck: Sending command: list newsgroups \n");
                        suck_write_s(t,"list newsgroups\015\012");
                } else {
			imsg("suck: list sent\n");
                        suck_write_s(t,"list\015\012");
                }
                break;
            case F_LIST3:
                endwait = state+1;
                break;
            case  F_DOLIST:
                /* Generate list of new groups to process */
                if (donames) {
	                state=F_IDLE;
                	do_names();
		} else {
			state = F_GENERATE;
			generate_start();
                	/* do_generate();*/
		}
                break;
            case F_GENERATE:
	    	if (intrim) do_trim();
		else generate_chunk();
                break;
            case F_UPDATE:
                state = F_OPEN; 			/* Open the channel*/
		if (gname!=NULL) suck_feeder(gname); 	/* Find name of feeder and user/password */
                endwait = F_FORGROUP;
                goto again;
	    case F_FORGROUP:
		if (gname!=NULL) ncpy(cur_group,gname,BFSZ-1);
		if (config_true(C_SUCK_STREAM)) goto stream_groups;
		header_next = F_FORGROUP2;
		goto listgroup;
            case F_FORGROUP2:
                xstat_start = xnow;
                stat_n = 0;
		stat_size = 0;
                if (gname==NULL) break;
                dmsg("suck: forgroup {%s} \n",gname);
                if (!group_wanted(gname)) {
                        imsg("Skipping group which is not in 'me' feed {%s}\n",gname);
                        state = F_IDLE;
                        break;
                }
                lprintf(bf,1000-1,"group %s\015\012",gname);
                suck_write_s(t,bf);
                chan_setstate(t,C_CHECK);
                endwait = F_GET; state = F_WAIT;
                failwait = F_IDLE;
                setchan = C_WAIT;
                break;
            case F_GET: /*   211 num from to group selected*/
		nextloop = 0;
                item_i = atoi(get_word(nntp_status,2));
                item_end = atoi(get_word(nntp_status,3));
		xgroup_end = item_end;		
                if (gname==NULL) break;
                gg = group_find(gname);
                if (gg<0) {
                        emsg("suck: skipping non existent group %s\n",gname);
                        state = F_GROUPNEXT;
                        break;
                }
		oldp = 100.0 - atof(config_get(C_SUCK_PERCENT));
		if (oldp<100.0) {
			i = (int) (item_end - ((oldp/100.0)*group_getrange(gg)));	
			imsg("suck_percent, only sucking up to %d of %d (%d items in group).\n",
				i,item_end,group_getrange(gg));
			if (i<item_end) item_end = i;			
		}

               	group_setcatchup(gg,pop_feed,item_end);
                fend = group_feedend(gg,pop_feed);       
		oldlimit = atoi(config_get(C_SUCK_MAXOLD));
		if (oldlimit == 0) oldlimit = 2000;
		if (oldlimit>0) {
		 	if ((item_end - fend) > oldlimit) {
				fend = item_end - oldlimit;
				imsg("Limiting number of old items to fetch to %d (suck_maxold)\n",oldlimit);
			}
		}
		dmsg("suck: suck group %d %d, fend %d\n",item_i,item_end,fend);
		if (!config_true(C_SUCK_NOFIX)) {
			if (item_end<fend) {
				group_setfeedend(gg,pop_feed,item_end);
				emsg("suck: Adjusting upto counter for {%s} from %d to %d\n",gname,fend,item_end);
	                        group_setcatchup(gg,pop_feed,item_end);
			}
		}
	        if (item_i<=fend) item_i = fend+1;
                if (item_i>item_end) {
                        state = F_GROUPNEXT;
                        group_usecatchup(gg,pop_feed);
                        imsg("suck: Skipping group %s (no new items) \n",gname);
                        break;
                }
                wmsg("suck: Fetching group %s %d new from %d to %d\n",gname,item_end-item_i+1,item_i,item_end);
                sucklog("suck: Fetching group %s %d new from %d to %d\n",gname,item_end-item_i+1,item_i,item_end);

                just_started = TRUE;
                lprintf(bf,1000-1,"stat %d\015\012",item_i);
		isheader = hdr_matchgroup(gname);
		if (isheader) imsg("suck: Only fetching headers for (%s)\n",gname);
		if (isheader) {
			hdr_setgrp(gg);
		} else 	hdr_setgrp(-1);
		instat = item_i;
		stat_loop = 0;
                suck_write_s(t,bf);
                chan_setstate(t,C_GETITEM_NEXT);
                state = F_WAIT; /* This part waits getitem to change state*/
                break;
            case F_GROUPNEXT:
		nextloop = 0;
		stat_loop = 0;
                if (gname!=NULL) {
                	lprintf(lastsuck,BFSZ-1,"      Group %s    completed, got %d items\015\012",gname,stat_n);
                	if (stat_n>0) imsg("suck: Articles fetched %s %d \n",gname,stat_n);
                	sucklog("suck: Articles fetched %s %d \n",gname,stat_n);
		}
                elapsed = xnow-xstat_start;
                if (elapsed==0) elapsed = 1;            
                if (stat_n>0) imsg("suck: Articles / Second %g \n",(double) stat_n/elapsed);
		if (stat_n>0) {
			stats_feed(cur_feeder,xnow-xstat_start,0,stat_n,0,stat_size,0.0,0.0);
		}
                /* group_setcached(gg,TRUE); */
                if (gg>0) group_setupdate(gg,FALSE);
                
                gtotal += stat_n;
                if (gtotal>890) {
			hist_flush();
                	group_write_changed();
                	gtotal=0;
                } /*Should call every 'n' articles as well */ 

		if (config_true(C_SUCK_CLOSE)) {
			if (stat_n>1) {
	                        if (isopen) chan_close(t);
	                        isopen = FALSE;
			}
		}
                state = F_IDLE;
                goto again;
            case F_EXPIRE:
		/* Close channel if it's open !! */
		if (isopen) {
		   isopen = FALSE;
		   chan_close(t);
		   ncpy(cur_group,"",BFSZ-1);
		   if (isamail) if (gname!=NULL) suck_trylater(gname);
		}
		sched_expire = FALSE; isexpire = FALSE;
                expire_do(); /* Doing background expire processing */
                break;
            case F_FEED:
                state++;
                break;
            case F_FEED_OPEN:
		if (gname==NULL) break;
                if (isopen) { 
                        suck_write_s(t,"quit\015\012");         
                        chan_close(t); 
                        isopen = FALSE;
                }
                str_clearbig(xbody); 
                str_clearbig(xhead); 
                t = chan_open(feed_toname(gname)); isopen = TRUE;
                ncpy(cur_open,gname,BFSZ-1);

                suck_feeder_name(gname);

                if (t>=0) {
                        chan_settimeout(t,tout); /* INCREASE TO 5 MINUTES */
                        open_end = F_FEED_NEXT;
                        docheck(F_FEED_AUTHUSER,C_WAIT,F_IDLE);
                } else {
                        isopen = FALSE;
                        outmsg("suck: Could not connect to %s \n",gname);
                        state=F_IDLE;
                }
                break;
            case F_FEED_AUTHUSER:
                if (feed_posting()) {
                        lprintf(bf,1000-1,"mode reader\015\012"); suck_write_s(t,bf);
			/* Chrisp changed in 5.5g7, so that it continues regardless of response to mode reader */
                        docheck(F_FEED_AUTHUSER2,C_WAIT,F_FEED_AUTHUSER2);
			break;
                } 
	    case F_FEED_AUTHUSER2:
                if (nntp_bad) outmsg("suck: Auth open failed %s \n",nocrlf(nntp_status));
                if (strlen(cur_user)==0) goto feed_mode;
                lprintf(bf,1000-1,"authinfo user %s\015\012",cur_user); suck_write_s(t,bf);
                docheck(F_FEED_AUTHPASS,C_WAIT,F_FEED_AUTHPASS);
                break;
            case F_FEED_AUTHPASS:
                if (nntp_err!=381) outmsg("suck: Auth user failed %s \n",nocrlf(nntp_status));
                lprintf(bf,1000-1,"authinfo pass %s\015\012",cur_pass); suck_write_s(t,bf);
                docheck(F_FEED_MODE,C_WAIT,F_FEED_MODE);
                break;
            case F_FEED_MODE:
                if (nntp_err!=281) outmsg("suck: Auth pass failed %s \n",nocrlf(nntp_status));
feed_mode:
		if (diskfull) {
			imsg("Closing connection as their disk is full\n");
	                suck_write_s(t,"quit\015\012");         
	                chan_close(t); 
	                isopen = FALSE;
	                state = F_IDLE;
			diskfull = FALSE;
			break;
		}
                if (feed_posting()) {
                        lprintf(bf,1000-1,"mode reader\015\012"); suck_write_s(t,bf);
                        docheck(open_end,C_WAIT,open_end);
                } else {
                        state = open_end;
                        chan_setstate(t,C_WAIT);
                }
                break;
            case F_FEED_NEXT: /* Change back to dmsg when fixed.*/
                /* ignore status, if it failed then it is already a reader */
next_one:       if (!feed_next(&gg,&gnum)) { state = F_FEED_CLOSE; break; }
            	ispostonly =  config_true(C_POSTONLY);
		if (gg== -2) ispostonly = TRUE;
		imsg("suck: feed_out item %d %d \n",gg,gnum);
                if (ispostonly) 

                        xstr = post_getlocal(gnum);
                else    xstr = item_head(gg,gnum);
                if (xstr==NULL) {
                        outmsg("suck: Item doesn't exist %d:%d \n",gg,gnum);
			
                        goto next_one;
                }
                str_copy(xhead,xstr);
		if (config_true(C_PATH_REMOVE)) if (feed_posting()) remove_header(xhead,"Path:");
		if (feed_posting()) remove_header(xhead,"NNTP-Posting-Host:");
		if (feed_posting()) remove_header(xhead,"X-Trace:");
		if (feed_posting()) remove_header(xhead,"Xref:");
		if (strlen(config_get(C_HDR_REMOVE))>0) {
		    char buf[BFSZ];
		    char buf2[BFSZ];
		    char *cp;
		    ncpy(buf,config_get(C_HDR_REMOVE),BFSZ-10);
		    cp = strtok(buf," \t:,");
		    while( cp != NULL ) {
			ncpy(buf2,cp,BFSZ-10);
			strcat(buf2,":");
			remove_header(xhead,buf2);
			imsg("suck: removed header (%s)\n",buf2);
			cp = strtok(NULL," \t:,");
		    }
		}
                if (!ispostonly) {
                        xstr = item_body(gg,gnum);
                }
                if (xstr==NULL) {
                        outmsg("suck: Item body doesn't exist %d:%d (may need to delete *.feed,*.send* and fix newsfeeds.conf)\n",gg,gnum);
                        goto next_one;
                }
                str_copy(xbody,xstr);
                if (feed_posting()) {
                        suck_write_s(t,"post\015\012");
                } else {
                        lprintf(bf,1000-1,"ihave %s\015\012",get_ahead(xhead,"Message-ID:"));
                        suck_write_s(t,bf);
                }
                chan_setstate(t,C_CHECK);  
                state = F_WAIT;
                failwait = F_FEED_SEND;
                endwait = F_FEED_SEND;
                setchan = C_WAIT; /* Channel mode after gets result*/
                break;
            case F_FEED_SEND:
                if (feed_posting()) ok = 340; else ok=335; 
		if (nntp_err==435) {
			feed_stats_dup();
			outmsg("suck: Message rejected, duplicate? check your EXCLUDE setting {%s} %.100s\n",nocrlf(nntp_status),get_ahead(xhead,"Message-ID:"));
                        if (!config_true(C_CONFIRM_SUCCESS)) tell_user(xhead,xbody,nntp_status);
	                state = F_FEED_NEXT;            
			break;		
		}
		if (nntp_err==437) {
			feed_stats(FALSE);
			outmsg("suck: Message rejected, upstream site didn't want it {%s} %.100s\n",nocrlf(nntp_status),get_ahead(xhead,"Message-ID:"));
                        if (!config_true(C_CONFIRM_SUCCESS)) tell_user(xhead,xbody,nntp_status);
	                state = F_FEED_NEXT;            
			break;		
		}
                if (nntp_err!=ok) {
			feed_stats(FALSE);
                        outmsg("suck: feedee not accepting posts {%s} (%s)\n",nocrlf(nntp_status),cur_open);
                        feed_reque(FALSE); /* Wasn't the articles fault */
                        if (!config_true(C_CONFIRM_SUCCESS)) if (feed_firsttime()) tell_user(xhead,xbody,nntp_status);
                        state = F_FEED_CLOSE;
                        break;
                }
                chan_write_str(t,xhead);
                if (!ispostonly) {
                        suck_write_s(t,"\015\012");
                        chan_write_str(t,xbody);
                }
                suck_write_s(t,".\015\012");

                chan_setstate(t,C_CHECK);  
                state = F_WAIT;
                failwait = F_FEED_SEND2;
                endwait = F_FEED_SEND2;
                setchan = C_WAIT; /* Channel mode after gets result*/
                break;
            case F_FEED_SEND2:
                ncpy(rep_id,get_ahead(xhead,"Message-ID:"),100);
                ncpy(rep_from,get_ahead(xhead,"From:"),100);
                ncpy(rep_subj,get_ahead(xhead,"Subject:"),100);
                if (feed_posting()) ok = 240; else ok=235; 
		if (config_true(C_RETRY_441)) {
			if (nntp_err==441) {emsg("Because retry_441 true is set will retry (%s)\n",nocrlf(nntp_status)); goto retryit;}
		} else {
			if (nntp_err==441) {emsg("Because retry_441 true is NOT set will not retry this post (%s)\n",nocrlf(nntp_status)); }
		}
                if (nntp_err!=ok && nntp_err==436) {
retryit:                feed_reque(TRUE); /* That article failed, add to count */
                        outmsg("reque: {%s} {%s} {%s} remote err: %s\n"
                                ,rep_id, rep_from, rep_subj ,nocrlf(nntp_status));
                        if (!config_true(C_CONFIRM_SUCCESS)) if (feed_firsttime()) tell_user(xhead,xbody,nntp_status);
			feed_stats(FALSE);
                } else {
                        if (nntp_err==ok) {
				feed_stats(TRUE);
                                outmsg("ok: {%s} {%s} {%s} %s\n"
                                ,rep_id, rep_from, rep_subj,cur_open );
                        } else {
				feed_stats(FALSE);
                                outmsg("reject: {%s} {%s} {%s} remote err: %s\n"
                                ,rep_id, rep_from, rep_subj,nocrlf(nntp_status));
                        }
                        if (nntp_err==ok) strcat(nntp_status," Success!!!");
                        if (nntp_err==ok || !config_true(C_CONFIRM_SUCCESS)) tell_user(xhead,xbody,nntp_status);
			feed_delete();
                        if (ispostonly) 
                                if (nntp_err==ok) feed_delete(); 
			
                }
                str_clearbig(xbody); 
                state = F_FEED_NEXT;            
                break;
            case F_FEED_CLOSE:
                imsg("suck: Feed Close \n");
                feed_end(); /* Flush any unsent articles back into queue etc */
                suck_write_s(t,"quit\015\012");         
                chan_close(t); 
                isopen = FALSE;
                state = F_IDLE;
                break;
            case F_MAIL_OPEN:
                if (isopen) { 
                        suck_write_s(t,"quit\015\012"); chan_close(t); isopen = FALSE;
                }
		if (gname==NULL) break;
                lprintf(bf,1000-1,"%s:25",gname);
                done_helo = FALSE;
                t = chan_open(feed_toname(bf)); isopen = TRUE;
                if (t>=0) {
                        chan_settimeout(t,tout); /* 5 minutes should be ok */
                        docheck(F_MAIL_NEXT,C_WAIT,F_IDLE);                     
                } else {
                        isopen = FALSE; state=F_IDLE;
                        outmsg("suck: MAIL: Could not connect to %s \n",gname);
                }
                break;
            case F_MAIL_NEXT:
mnext_one:      if (!feed_next(&gg,&gnum)) { state = F_MAIL_CLOSE; break; }
                imsg("suck: Feed Mailing local item %d:%d\n",gg,gnum);
                xstr = post_getlocal(gnum);
                if (xstr==NULL) { outmsg("suck: Mail item gone? %d:%d \n",gg,gnum); goto mnext_one; }
                str_copy(xhead,xstr);

                if (!done_helo) {
                        lprintf(bf,1000-1,"HELO %s\015\012",config_get(C_MYNAME)); suck_write_s(t,bf);
                        docheck(F_MAIL_FROM,C_WAIT,F_MAIL_NEXT);                        
                        done_helo = TRUE;
                        break;
                }
            case F_MAIL_FROM:
		toi = 0;
		if (strlen(config_get(C_MAIL_BOUNCE))>0) {
	                lprintf(bf,1000-1,"MAIL FROM:%s\015\012",email_fix(config_get(C_MAIL_BOUNCE))); suck_write_s(t,bf);
		} else {
	                lprintf(bf,1000-1,"MAIL FROM:%s\015\012",email_fix(get_ahead(xhead,"From:"))); suck_write_s(t,bf);
		}
                docheck(F_MAIL_RCPT,C_WAIT,F_MAIL_NEXT);
                break;
            case F_MAIL_RCPT:
		ncpy(bf2,email_fixn(toi,get_ahead(xhead,"To:")),100);
		if (strlen(bf2)==0) goto mail_data;
		imsg("Sending email to %s\n",bf2);
                lprintf(bf,1000-1,"RCPT TO:%s\015\012",bf2); suck_write_s(t,bf);
		toi++;
                docheck(F_MAIL_RCPT,C_WAIT,F_MAIL_NEXT);
                break;
            case F_MAIL_DATA:
mail_data:      lprintf(bf,1000-1,"DATA\015\012"); suck_write_s(t,bf);
                docheck(F_MAIL_SEND,C_WAIT,F_MAIL_NEXT);
                break;
            case F_MAIL_SEND:
                chan_write_str(t,xhead);
                suck_write_s(t,".\015\012");
                docheck(F_MAIL_SEND2,C_WAIT,F_MAIL_NEXT);
                break;
            case F_MAIL_SEND2:
                outmsg("Email sent ok to {%s} \n",get_ahead(xhead,"To:"));
		feed_stats(TRUE);
                tell_user(xhead,xbody,nntp_status);
                feed_delete(); /* If local feed, delete items */        
                state = F_MAIL_NEXT;            
                break;
            case F_MAIL_CLOSE:
                imsg("suck: mail: Closing mail connection\n");
                feed_end(); /* Flush any unsent articles back into queue etc */
                suck_write_s(t,"QUIT\015\012");         
                chan_close(t); 
                isopen = FALSE;
                state = F_IDLE;
                break;
	    case F_HEADER_EXPIRE:
listgroup:
		if (gname==NULL) {
			state = header_next;
			break;
		}
		if (!hdr_matchgroup(gname)) {
			state = header_next;
			goto again;
		}
		if (config_true(C_HEADER_NO_EXPIRE)) {
			state = header_next;
			goto again;
		}
		liststart = TRUE;
		imsg("Expiring items in header group %s\n",gname);
		if (config_true(C_HEADER_NO_LISTGROUP)) {
	                lprintf(bf,1000-1,"GROUP %s\015\012",gname); suck_write_s(t,bf);
	                lprintf(bf,1000-1,"XHDR Date 1-\015\012"); suck_write_s(t,bf);
			liststart_g = TRUE;
		} else {
	                lprintf(bf,1000-1,"LISTGROUP %s\015\012",gname); suck_write_s(t,bf);
		}
                chan_setstate(t,C_LISTGROUP);
                state = F_WAIT;
		break;
	    case F_STREAM_GROUP:
stream_groups:    
		header_next = F_STREAM_GROUP2;
		goto listgroup;
	    case F_STREAM_GROUP2:
		if (gname!=NULL) ncpy(cur_group,gname,BFSZ-1);
		xstat_start = xnow;
                stat_n = 0;
		stat_size = 0;
                if (gname==NULL) break;
                if (!group_wanted(gname)) {
                        imsg("Skipping group which is not in 'me' feed {%s}\n",gname);
                        state = F_IDLE;
                        break;
                }
                lprintf(bf,1000-1,"group %s\015\012",gname);
                suck_write_s(t,bf);
                chan_setstate(t,C_STREAM_GROUP);
                state = F_WAIT;
                failwait = F_IDLE;
                setchan = C_WAIT;
                break;
            case F_STREAM_NEXT:
                if (gname!=NULL) {
                	lprintf(lastsuck,BFSZ-1,"      Group %s    completed, got %d items\015\012",gname,stat_n);
                	wmsg("suck_stream: Articles fetched %s %d \n",gname,stat_n);
		}
                elapsed = xnow-xstat_start;
                if (elapsed==0) elapsed = 1;
                if (stat_n>0) imsg("suck: Articles / Second %g \n",(double) stat_n/elapsed);
		if (stat_n>0) {
			stats_feed(cur_feeder,xnow-xstat_start,0,stat_n,0,stat_size,0.0,0.0);
		}
                if (gg>0) group_setupdate(gg,FALSE);
                gtotal += stat_n;
                if (gtotal>890) {
                       	hist_flush();
			group_write_changed(); 
                	gtotal=0;
                } /*Should call every 'n' articles as well */ 
                state = F_IDLE;
                goto again;
        }
	vtrace();
        /*dmsg("lib: suck returns\n");
	*/
}
char *email_fixn(int upto, char *list)
{
	static char bf[BFSZ];
	char *s;
	int i = 0;
	ncpy(bf,list,BFSZ-1);
	s = strtok(bf,",\t \r\n");
	for (;s!=NULL;) {
		if (strchr(s,'@')!=NULL) {
			if (i==upto) return email_fix(s);
			i++;
		}
		s = strtok(NULL,",\t \r\n");
	}
	return "";
}
/*
	listgroup test.chat
	211 Article list follows
	4
	5
	.
*/
static int stopnow;
//#define MAXLIST 30000
int suck_listgroup(char *bf, int sz)
{
	static int *listid;
	static int nlist;
	char *s;
	int n = atoi(bf);
	s = strchr(bf,'\r'); if (s!=NULL) *s = 0;
	s = strchr(bf,'\n'); if (s!=NULL) *s = 0;
	if (liststart_g) {
		liststart_g = FALSE;
		if (n>299) {
			emsg("suck: listgroup bad response {%s} {%s} {%s}\n",gname,nocrlf(bf),cur_open);
			state = header_next;
			liststart_g = liststart = FALSE;
			return TRUE;
		}
		nlist = 0;
		return TRUE;
	}
	if (liststart) {
		if (n>299) {
			emsg("suck: listgroup bad response {%s} {%s} {%s}\n",gname,nocrlf(bf),cur_open);
			state = header_next;
			return TRUE;
		}
		liststart = FALSE;
		if (listid==NULL) listid = mymalloc(sizeof(int)*max_items());
		nlist = 0;
		return TRUE;
	}
	/*imsg("listgroup got (%s) ",bf);*/
	if (strcmp(bf,".")==0) {
		imsg("got list of items upstream, %d\n",nlist);
		if (gname!=NULL) db_trimgroup(gname,listid,nlist);
		state = header_next;
		return TRUE;
	}
	if (nlist>=(max_items()-5)) {
	    int trim=6000;
	    int i;
	    for (i=0; i<nlist-trim;i++) {
		listid[i] = listid[i+trim];
	    }
	    nlist -= trim;	    
	    if (nlist<0) nlist = 0;
	}
	if (nlist>=(max_items()-5)) return TRUE;
	listid[nlist++] = n;
	return TRUE;
}
int suck_stream_group(char *bf, int sz)
{
	int oldp;
	int fend;
	int i;
	char *gn;
        nntp_err = atoi(bf);
        ncpy(nntp_status,bf,200);
	dmsg("suck: stream_group -- %s\n",nocrlf(nntp_status));
        if (nntp_err >399) {
		gn = gname;
		if (gn==NULL) gn = "unknown";
                emsg("suck: Group command failed use matchgroups {%s} {%s} {%s}, \n",gn,nocrlf(nntp_status),cur_open);
		lprintf(lastsuck2,BFSZ-1,"info: (%s) {%s} Missing group? use matchfeed {%s}\015\012",lib_hour(),cur_open,nocrlf(nntp_status));
                state = F_STREAM_NEXT;
		suck_do();
           	return TRUE;
	}
        item_i = atoi(get_word(nntp_status,2));
        item_end = atoi(get_word(nntp_status,3));
        if (gname==NULL) return TRUE;
        gg = group_find(gname); 
        if (gg<0) { emsg("suck: skipping non existent group %s\n",gname); state = F_STREAM_NEXT; return TRUE; }
        fend = group_feedend(gg,pop_feed);
 	group_setcatchup(gg,pop_feed,item_end);
	oldlimit = atoi(config_get(C_SUCK_MAXOLD));
	if (oldlimit>0) {
	 	if ((item_end - fend) > oldlimit) {
			fend = item_end - oldlimit;
			imsg("Limiting number of old items to fetch to %d\n",oldlimit);
		}
	}

	if (!config_true(C_SUCK_NOFIX)) {
	 if (item_end<fend) {
		group_setfeedend(gg,pop_feed,item_end);
		imsg("Adjusting stream upto counter for {%s} from %d to %d\n",gname,fend,item_end);
                group_setcatchup(gg,pop_feed,item_end);
	 }
	} 

	dmsg("suck: Stream from %d to %d fend %d\n",item_i,item_end,fend);
	oldp = (int) (100.0 - atof(config_get(C_SUCK_PERCENT)));
	if (oldp<100.0) {
		i = (int) (item_end - ((oldp/100.0)*group_getrange(gg)));	
		imsg("suck_percent, only sucking up to %d of %d (%d items in group).\n",
			i,item_end,group_getrange(gg));
		if (i<item_end) item_end = i;			
	}
        

        if (item_i<=fend) item_i = fend+1;
        if (item_i>item_end) {
		state = F_STREAM_NEXT;
		if (group_feedend(gg,pop_feed) < item_i) group_setfeedendif(gg,pop_feed,item_end);
                imsg("suck: Skipping group %s (no new items) \n",gname);
                sucklog("suck: Skipping group %s (no new items) \n",gname);
		suck_do();
             	return TRUE;
        }

	wmsg("suck: Fetching group %s %d new from %d to %d\n",gname,item_end-item_i+1,item_i,item_end);
	sucklog("suck: Fetching group %s %d new from %d to %d\n",gname,item_end-item_i+1,item_i,item_end);
 	lprintf(bf,BFSZ-1,"stat %d\015\012",item_i);
        suck_write_s(t,bf);
	isheader = hdr_matchgroup(gname);
	if (isheader) imsg("suck: Only fetching headers for (%s)\n",gname);

	
	chan_setstate(t,C_STREAM_NEXT);
	infindfirst = TRUE;
	infind_loop = 0;
	return TRUE;	
} 
void suck_stream_findfirst(char *bf);
void suck_stream_findfirst(char *in)
{
	int c = atoi(in);
	char bf2[BFSZ];
	int was;
	char bf[BFSZ];
	int wanted = 0;
	char *s;
	infind_loop++;

	if (infind_loop>5000) goto skip;	
	if (item_i>item_end) goto skip;
	if (c<300) {
		wanted = 0;
		s = strchr(in,' ');if (s!=NULL) wanted = atoi(s+1);
	 	if (wanted!=0) lprintf(bf,BFSZ-1,"stat %d\015\012",wanted);
	 	else lprintf(bf,BFSZ-1,"stat\015\012");
	        suck_write_s(t,bf);
		infindfirst = FALSE;
		suck_stream_batch(TRUE);
		return;
	}
	if (c==420 || c==423  || c==430) {
		imsg("suck: stream gap hit, searching %s:%d/%d (%s)\n",gname,item_i+1,item_end,nocrlf(in));
		lprintf(bf,BFSZ-1,"stat %d\015\012",++item_i);
	        suck_write_s(t,bf);
		return;
	}
skip:;
	infindfirst = FALSE;
	state = F_STREAM_NEXT;
	was = group_feedend(gg,pop_feed);
	if (group_feedend(gg,pop_feed) < item_i) group_setfeedendif(gg,pop_feed,item_end);
	ncpy(bf2,in,60);
        imsg("suck: Skipping group %s, was %d now %d %d, item_i %d, status %s \n",gname,was,group_feedend(gg,pop_feed),item_end,item_i,nocrlf(bf2));
	suck_do();
	return;
}

static int nntp_item;
static int sbatch,batchi,nmissing;
static int sget[100]; /* List of message ID's we really want */
static int nsget;
void suck_stream_batch(int first)
{
	int i;
	char bf[BFSZ];
	nmissing = 0;
	sbatch = 10;
	nsget = 0;

	if ((stat_n>0) |  stopnow) if (!first || stopnow) if ((stat_n>suck_batchn) || (stat_size>suck_batchsize) || stopnow) {
		if ((config_true(C_SUCK_REQUEUE) && (gname!=NULL))  && !stopnow ) {
      			suck_queue_feed(gname,1,pop_feed);
               		if (doskip && item_i+10 < group_feedend(gg,pop_feed)) {
			 	wmsg("Reque: skipping gaps in the rest of {%s}  %d %d\n",gname,item_i,group_feedend(gg,pop_feed));
			 	sucklog("Reque: skipping gaps in the rest of {%s}  %d %d\n",gname,item_i,group_feedend(gg,pop_feed));
				group_usecatchup(gg,pop_feed);
			} else {
               			group_setcatchup(gg,pop_feed,group_feedend(gg,pop_feed));
			 	imsg("Requeing the rest of {%s}  %d items, %g bytes\n",gname,stat_n,stat_size);
			 	sucklog("Requeing the rest of {%s}  %d items, %g bytes\n",gname,stat_n,stat_size);
		 	}
		} else {
		 	imsg("That's enough from one group %d items, %g bytes\n",stat_n,stat_size);
		 	sucklog("That's enough from one group %s %d items, %g bytes\n",gname,stat_n,stat_size);
		}
		stopnow = FALSE;
		state = F_STREAM_NEXT;
		suck_do(); 
		return;
	}


	if ((item_end - item_i + 1 )<10) sbatch = item_end - item_i + 1;
	if (first) sbatch--;

	dmsg("suck: stream_batch %d \n",sbatch);

	if ((sbatch<=0) && !first) { /* Finished update */
		chan_setstate(t,C_WAIT);
		state = F_STREAM_NEXT;
		suck_do();
           	return ;
	}

	batchi = 0;
	for (i=0; i<sbatch; i++) {
		lprintf(bf,BFSZ-1,"next\015\012");
	        suck_write_s(t,bf);
	}
	if (first) sbatch++;
	chan_setstate(t,C_STREAM_NEXT);
}
int suck_stream_next(char *bf, int sz)
{
	char *s;
        nntp_err = atoi(bf);
        ncpy(nntp_status,bf,200);
	dmsg("suck: stream_next -- %s\n",nocrlf(nntp_status));

	if (infindfirst) {
		suck_stream_findfirst(nntp_status);
		return TRUE;
	}
	
	batchi++;	
	dmsg("suck: batch %d of %d \n",batchi,sbatch);
	if (nmissing>4 || (nmissing >= sbatch) || ((nntp_err>300) && ((nmissing+1) >= sbatch))) {
	 	if (nntp_err!=421) imsg("End of group probably %s \n",nocrlf(nntp_status));
		item_i = item_end+1;
		goto ret;
	}
	if (nntp_err>300) {
		dmsg("suck: no article {%s}\n",nocrlf(nntp_status));
		nmissing++;
	} else {
		s = strchr(bf,' ');
		if (s!=NULL) {
			nntp_item = atoi(s+1);
			item_i = nntp_item+1;
		}
	    	if (!hist_gotit(get_word(nntp_status,2))) {
			sget[nsget++] = nntp_item;
			dmsg("suck: stream ask for item %d (%s)\n",nntp_item,get_word(nntp_status,2));
		} else {
			imsg("suck: cross post duplicate item %d\n",nntp_item);
		        if (nntp_item>group_feedend(gg,pop_feed)) group_setfeedendif(gg,pop_feed,nntp_item);
			if (doskip && nntp_item+10 < group_feedend(gg,pop_feed)) {
	                      group_usecatchup(gg,pop_feed);
			      item_i = group_feedend(gg,pop_feed);
			      stopnow = TRUE;
			      imsg("stopnow set to skip group\n");
			}
		}
	}
ret:;
	if (batchi>=sbatch) {
		suck_stream_get();
		return TRUE;
	}
        return TRUE;
}
static int sgeti;
void suck_stream_get(void)
{
	int i;
	char bf[BFSZ];
	sgeti = 0;
	dmsg("suck: stream_get %d \n",nsget);
        if (!lib_checkspace()) {
                state = F_STREAM_NEXT;
                return;
        }
	if (nsget==0) { /* Finished */
		suck_stream_batch(FALSE);
		return;
	}
	for (i=0; i<nsget; i++) {
		lprintf(bf,BFSZ-1,"%s %d\015\012",isheader ? "head" : "article",sget[i]);
	        suck_write_s(t,bf);
		dmsg("suck: stream asking for item %d\n",sget[i]);
	}
	chan_setstate(t,C_STREAM_ITEM);
}
int suck_stream_item(char *bf, int sz)
{
	char *s;
        nntp_err = atoi(bf);
        ncpy(nntp_status,bf,200);
	dmsg("suck: stream_item -- %s\n",nocrlf(nntp_status));
	
	sgeti++; 
        if ((nntp_err != 220) && (nntp_err!=221) ) {
		if ((nntp_err==423) || (nntp_err==430)) {
			imsg("Found gap from faulty news server\n"); 
		} else {
			emsg("suck: Item not available {%s} {%s}\n",nocrlf(nntp_status),cur_open);
		}
		if (sgeti>=nsget) {
		 	suck_stream_batch(FALSE);
		}		
	} else {
		dmsg("suck: stream getting item %s",nocrlf(nntp_status));
		s = strchr(bf,' ');
		if (s!=NULL) {
			nntp_item = atoi(s+1);
			item_i = nntp_item + 1;
	                suck_setfeedend(nntp_status);
		}
		group_xref_clear();
		if (gg>=0) group_xref_here(gg,nntp_item);
		chan_setstate(t,C_GETITEM);
		big_init(t);
	}
	return TRUE;
}

static int gap_skip;
int suck_getitem_status(char *bf, int sz) /* Expecting item head and body */
{
	char *s;
        int nntp_err;
        nntp_err = atoi(bf);
        ncpy(nntp_status,bf,200);
	dmsg("suck: item_status -- %s\n",nocrlf(nntp_status));
	if (nntp_err==480) imsg("suck: permission denied reading group %s\n",group_name(gg));
	if (just_started) nextloop = 0;
	if (nntp_err==223) {suck_getitem_next(bf,sz); return TRUE;}
	if (just_started) gap_skip = FALSE;
	s = strchr(bf,' ');
	if (s!=NULL) nntp_item = atoi(s+1);
	group_xref_clear();
	if (gg>=0) group_xref_here(gg,nntp_item);
        if ((nntp_err<100) || (nntp_err>299)) {
                if (((nntp_err==423) || (nntp_err==430)) && just_started) {  /* Might have hit a gap , so ask for next */
                        dmsg("suck: status!=220, but ok here %s\n",nocrlf(nntp_status));
                        just_started = FALSE;
                        suck_write_s(t,"next\015\012");
                        return TRUE;
                }
		if ((nntp_err==423) || (nntp_err==430)) {
			if (nextloop>2000) {
				emsg("Missing item loop, could not suck from faulty upstream server\n");
			} else {
				nextloop++;
				imsg("Missing item on upstream site - continuing %s\n",nocrlf(nntp_status));
	                        suck_write_s(t,"next\015\012");
	                        return TRUE;			
			}
		}
                dmsg("suck: status!=220, doing next group %s\n",nocrlf(nntp_status));
                state = F_GROUPNEXT; 
		return FALSE;
        }
        just_started = FALSE;

        if (!lib_checkspace()) {
                state = F_GROUPNEXT;
                return FALSE;
        }

        lprintf(bf,BFSZ-1,"next\015\012");     suck_write_s(t,bf);
        chan_setstate(t,C_GETITEM);
	xinitem_clear();
	big_init(t);

        return TRUE;
}
int suck_getitem_next(char *in, int sz)
{
        char bf[BFSZ];
        int nntp_err;
	int upto;

	if (config_true(C_SUCK_STREAM)) return suck_stream_item(in,sz); 


        nntp_err = atoi(in);
	if (nntp_err==480) imsg("suck: permission denied reading group %s\n",group_name(gg));
        ncpy(nntp_status,in,200);
	dmsg("suck: next -- %s\n",nocrlf(nntp_status));
        upto = atoi(get_word(nntp_status,1));
/*imsg("getitem_next %d %s\n",instat,nntp_status);*/

	if ((stat_n>suck_batchn) || (stat_size>suck_batchsize) || stopnow) {
		if ((config_true(C_SUCK_REQUEUE) && (gname!=NULL))  && !stopnow ) {
      			suck_queue_feed(gname,1,pop_feed);
		 	imsg("Requeing the rest of {%s}  %d items, %g bytes %d %d\n",gname,stat_n,stat_size,upto,group_feedend(gg,pop_feed));
		 	sucklog("Requeing the rest of {%s}  %d items, %g bytes %d %d\n",gname,stat_n,stat_size,upto,group_feedend(gg,pop_feed));
	               	group_setcatchup(gg,pop_feed,group_feedend(gg,pop_feed));
		} else {
		 	imsg("Thats enough from one group %d items, %g bytes\n",stat_n,stat_size);
		 	sucklog("Thats enough from one group %d items, %g bytes\n",stat_n,stat_size);
		}
		stopnow = FALSE;
		state = F_GROUPNEXT; 
		return TRUE;
	}
        if (nntp_err!=223) {
/*imsg("suck: DEBUG MESSAGE item=%d %d, instat=%d loop=%d",
	item_i,item_end,instat,stat_loop); */
                if (((nntp_err==423) || (nntp_err==430)) 
			&& (instat>=0) ) {  /* Might have hit a gap , so ask for next */
		  if ((stat_loop<3000) && (instat<=xgroup_end)) {
			if (item_i>item_end) goto skip;
                        just_started = FALSE;
			lprintf(bf,BFSZ-1,"stat %d\015\012",++instat);
			dmsg("suck: status!=220, but ok here %s\n",bf);
                        suck_write_s(t,bf);
			stat_loop++;
                        return TRUE;
		  } else {
			emsg("Big gap in group, catching up %s\n",group_name(gg));
			  goto skip;
		  }
                }
		if ((nntp_err==423) || (nntp_err==430)) {
			imsg("Found gap from faulty news server\n"); 
			if (nextloop>2000) {
				emsg("Missing item loop, could not suck from faulty upstream server\n");
			} else {
				nextloop++;
		                lprintf(bf,BFSZ-1,"next\015\012");     
				suck_write_s(t,bf);
		                chan_setstate(t,C_GETITEM_NEXT);
		                return TRUE;
			}			
		}

                if (nntp_err==420) { /* Nothing in this group, set fend*/
skip:                       group_usecatchup(gg,pop_feed);
                }
                dmsg("suck: status!=223, doing next group %s \n",in);
		if (nntp_err==480) imsg("suck: permission denied, doing next group %s\n",in);

                state = F_GROUPNEXT; return FALSE;
        }
	instat = -1;
	stat_loop = 0;

        if (hist_gotit(get_word(nntp_status,2))) {
		ncpy(bf,get_wordnolf(nntp_status,2),BFSZ-1);
                inmsg("suck: Already got %s\n",bf);
                lprintf(bf,BFSZ-1,"next\015\012");     suck_write_s(t,bf);
                suck_setfeedend(nntp_status);
                chan_setstate(t,C_GETITEM_NEXT);
                return TRUE;
        } else {
                dmsg("suck: we want it {%s} \n",get_word(nntp_status,2) );
                lprintf(bf,BFSZ-1,"%s\015\012",isheader ? "head" : "article");  suck_write_s(t,bf);
                chan_setstate(t,C_GETITEM_STATUS);
                return TRUE;
        }
}
void suck_setfeedend(char *s)
{
        int i;
        i = atoi(get_word(s,1));
        if (i>group_feedend(gg,pop_feed))  group_setfeedendif(gg,pop_feed,i);
}
void xinitem_clear(void)
{
	if (t>=0) big_free(t);
}
int suck_getitem(char *s, int sz)
{
        static int xinitem_failed;
	String *get;
	char *x;
	if (t<0) return FALSE;
  
        if (strcmp(s,".\015\012")==0) {
		dmsg("suck: Got end of item\n");
		if (isheader) {
			x = hdr_link(cur_feeder,group_name(gg),group_here_get());
			big_add(t,x,strlen(x));
		}

                chan_setstate(t,C_GETITEM_NEXT);        /* wait status code */
                suck_setfeedend(nntp_status);
                /* group_setcached(gg,TRUE);*/
                stat_n++;
		stat_size += big_len(t);
                msg_from(cur_feeder);
                if (xinitem_failed) {
                        msg_reject("Item far too big, ignoring it\n"); 
                } else  {
			get = big_get(t);
			if (isheader) item_setheader(TRUE,gname);
			if (ishdr())  {
				if (ishdr_old()) group_use_xref(TRUE);
				hdr_setupstream(nntp_item);
			}

			if (!config_true(C_SUCK_CHECK_AGE)) set_ignoreage(TRUE);
			if (get!=NULL) item_suck(get);
			set_ignoreage(FALSE);
			if (isheader) item_setheader(FALSE,"");
			if (ishdr_old())  	group_use_xref(FALSE);

                }
                big_free(t);
                xinitem_failed = FALSE;
		if (config_true(C_SUCK_STREAM)) {
			if (sgeti>=nsget) {
			 	suck_stream_batch(FALSE);
			}		
		}
        } else {
                if (big_len(t) >= MAX_STRLEN) xinitem_failed = TRUE;
                else big_add(t,s,sz);
        }
        return TRUE;
}
int suck_check(char *bf, int sz)
{
	char tmp[BFSZ];
	strcpy(tmp,"nocurrentgroup");
	if (gname!=NULL) ncpy(tmp,gname,100);
        ncpy(nntp_status,bf,200);
	dmsg("suck: check -- %s\n",nocrlf(nntp_status));
	if (nntp_status[3]=='-') {
		imsg("Ignoring comment from mail server %s\n",nocrlf(nntp_status));
		return TRUE;
	}
        state = endwait;
        nntp_bad = FALSE;
        nntp_err = atoi(bf);

        if (nntp_err >399) {
                if (nntp_err==400) if (endwait==F_OPEN_DOMODE) { /* Swallow 400 error as it just means we can't send news */
			imsg("Ignoring disk full error as we want to suck\n");
			diskfull = TRUE;
			goto isok;
		}
                nntp_bad = TRUE;
                state = failwait;
		if (state==F_GET) {
			emsg("suck: Group command failed use matchgroups  {%s} {%s}, \n",nocrlf(nntp_status),cur_open);
			sucklog("suck: Group command failed use matchgroups  {%s} {%s}, \n",nocrlf(nntp_status),cur_open);
			lprintf(lastsuck2,BFSZ-1,"info: (%s) {%s} Missing group? use matchfeed {%s}\015\012",lib_hour(),cur_open,nocrlf(nntp_status));
		}
                if (nntp_err!=423) wmsg("suck: BAD status code returned {%s} {%s} {%s}\n",nocrlf(nntp_status),tmp,cur_open);
                if ((nntp_err==423) || (nntp_err==430)) item_i++;
        } else {
isok:           dmsg("suck: check set cstate to [%d] %d \n",t,setchan);
                dmsg("suck: check set fstate {%s} \n",f_text[state]);  
                chan_setstate(t,setchan);
                if (endwait==F_OPEN2) { /* Swallow mode output */
                        chan_setstate(t,C_CHECK);
                        state = F_WAIT;
                        failwait = open_end;
                        endwait = open_end;
                } 
        }
        return TRUE;
}
static int fixmod;
int trim_fix_mod(int x)
{
	fixmod = x;
	return TRUE;
}
int trim_file(int chan, char *fname)
{
        char grp[200],mod[200];
	char *s;
	char bf[BFSZ];
	int i,r;
	int upto = 0;
	static char *trim_ok;
	FILE *f;
	int nmodchange=0;
	int g,nadded=0;
	int ndone = 0;
	int nmod=0;
	int wasmod;
	int ngroups = group_n();
	
	if (strlen(fname)==0) {
		emsg("No file specified for matchgroups\n");
		return FALSE;
	}
	imsg("matchgroups processing\n");
	if (trim_ok==NULL) trim_ok = mymalloc(ngroups);
	for (i=0; i<ngroups; i++) trim_ok[i] = FALSE;
	f = fopen(file_config(fname),"r");
	if (f==NULL) {
		lprintf(bf,BFSZ-1,"Cannot find (%s)\r\n",file_config(fname));
		suck_write_s(chan,bf);
		r = FALSE;
		goto ret;
	}
	
	for (;!feof(f);){
		if (fgets(bf,BFSZ-10,f)==NULL) break;
		s = strtok(bf," \n\t"); if (s==NULL) continue;
	        ncpy(grp,s,150); 
		strcpy(mod,"");
	        s = strtok(NULL," \n\t"); if (s==NULL) goto cont;
	        s = strtok(NULL," \n\t"); if (s==NULL) goto cont;
	        s = strtok(NULL," \n\t"); if (s==NULL) goto cont;
	        ncpy(mod,s,199);
		if (*mod!='y') if (*mod!='m') *mod = 'y';
cont:
		g = group_find(grp);  
		if (fixmod) if (g<0) continue;
		if (fixmod) if (group_isdeleted(g)) continue;
	        if (strlen(grp)==0) continue;
	        if (!group_wanted(grp)) continue;
	        if (g>=0) {
			wasmod = group_ismod(g);
			group_undelete(g);
			if (strlen(mod)>0) group_setmod(g,mod);
			if (g<ngroups) trim_ok[g] = TRUE;
			if (wasmod != group_ismod(g)) nmodchange++;
			if (group_ismod(g)) nmod++;
		} 
		if (g<0) {
			g = group_new(grp,"matchgroups",mod,"");
			nadded++;
		}
	}
	
	if (!fixmod) for (i=0; i<ngroups; i++) {
	  if (strcmp(group_name(i),"control")==0) continue;
	  if (strcmp(group_name(i),"control.cancel")==0) continue;
	  if (!trim_ok[i]) {
                wmsg("rmgroup %s\n",group_name(i));
                group_setlife(i,0);
                group_setcached(i,FALSE);
                group_delete(i);
		ndone++;
 	  }
	}

	imsg("matchgroups completed, %d groups deleted %d added %d modfix\n",ndone,nadded,nmodchange);
	lprintf(bf,BFSZ-1,"matchgroups completed, %d groups deleted, %d added, %d modfixed %d moderated\n",ndone,nadded,nmodchange,nmod);
	suck_write_s(chan,bf);
	r = TRUE;
ret:
	myfree(trim_ok);
	trim_ok = NULL;
	return r;
}
int suck_getlist(char *s, int sz)
{
        if (newlist==NULL) newlist = str_new();
        if (strcmp(s,".\015\012")==0) {
                imsg("suck: End of list found, total size %d bytes\n",str_len(newlist));
                /* state = endwait;*/
		state = F_DOLIST;
                chan_setstate(t,C_WAIT);
        } else {
		if (str_len(newlist)>6000000) {
			str_clear(newlist);
			emsg("suck_getlist, exceeded 6mb, cleared, upstream site has bad active file\n");
		}		
                str_adds(newlist,s,sz);
        }
        return sz;
}
/* 
void do_generate(void)
{

        char *grps[max_groups]; 
        int ngrps=0;
        char *s;
        int i;

        wmsg("suck: Generate begins...\n");
        s = strtok(str_s(newlist),EOL);
        for (;s!=NULL;) {
                if (ngrps>=max_groups) {
                        emsg("Too many groups\n");
                        goto cont;
                }
                grps[ngrps++] = mystrdup(s);
                if (grps[ngrps-1]==NULL) emsg("Generate mystrdup failed\n");
                s = strtok(NULL,EOL);
        } 
cont:
        dmsg("suck: Generate lines start\n");
        for (i=0; i<ngrps; i++) gen_line(grps[i]);
        dmsg("suck: Generate lines done\n");
        for (i=0; i<ngrps; i++) myfree(grps[i]);
        dmsg("suck: Generate free done\n");
        group_write();
        wmsg("suck: Generate ends\n");
        str_copys(newlist,"",0); 
}
*/

static int gen_upto,gen_len;
void generate_start(void)
{
        /* For each line in newlist, if cached, add to uplist and */ 
        if (newlist==NULL) newlist = str_new();
	gen_upto = 0;
	gen_len = str_len(newlist);
}
void generate_chunk(void)
{
	char *s;
	char *s2;
	char bf[BFSZ];
	int i;

        if (newlist==NULL) newlist = str_new();
	if (str_len(newlist)==0) goto done;
	for (i=0; i<400; i++) {
		s = str_s(newlist) + gen_upto;
		s2 = strstr(s,EOL);
		if (s2==NULL) {
done:			state = F_IDLE;
			gen_upto = 0;	
			str_free(newlist); newlist = NULL;
		        group_write();
			return;
		}
		gen_upto = gen_upto + s2-s+strlen(EOL); 
		*s2 = 0;
		ncpy(bf,s,150);
		gen_line(bf);
	}
}
void do_trim(void)
{
	char *s;
	char *s2;
	char bf[BFSZ];
	char grp[BFSZ],mod[BFSZ];
	int i;
	int upto = 0;
	static char *trim_ok;
	int g;
	int ndone = 0;
	int nadded = 0;
	int nun=0;
	int ngroups = group_n();
	char mgroups[BFSZ];	

	strcpy(mgroups,"*");
	if (suck_groups[trim_n]!=NULL) ncpy(mgroups,suck_groups[trim_n],BFSZ-10);
	if (strlen(match_groups)>0) ncpy(mgroups,match_groups,BFSZ-10);
	imsg("matchsite processing %s %s\n",match_site,match_groups);

	if (trim_ok==NULL) trim_ok = mymalloc(group_n()+10);
	for (i=0; i<ngroups; i++) trim_ok[i] = FALSE;

	for (i=0; i<max_groups; i++) {
		s = str_s(newlist) + upto;
		s2 = strstr(s,EOL);
		if (s2==NULL) {
			state = F_IDLE;
			upto = 0;	
			str_free(newlist); newlist = NULL;
			goto trimit;
		}
		upto = upto + s2-s+2; 
		*s2 = 0;

		ncpy(bf,s,150);
		s = strtok(bf," \n\t"); if (s==NULL) continue;
	        ncpy(grp,s,150); 
	        s = strtok(NULL," \n\t"); if (s==NULL) continue;
	        s = strtok(NULL," \n\t"); if (s==NULL) continue;
	        s = strtok(NULL," \n\t"); if (s==NULL) continue;
	        ncpy(mod,s,199);
		if (*mod!='y') if (*mod!='m') *mod = 'y';
	        g = group_find(grp);  
		if (strlen(grp)==0) continue;
	        if (!group_wanted(grp)) continue;
		if ((g>=0) && (g<ngroups)) {
			trim_ok[g] = TRUE;
			if (!group_match(mgroups,group_namelc(g))) continue;
			if (!trim_only) {
				if (group_isdeleted(g)) nun ++;
				group_undelete(g);
				group_setmod(g,mod);
			}
		} 
		if (!trim_only) if (g<0) {
			if (!group_match(mgroups,grp)) continue;
			nadded++;
			g = group_new(grp,"matchgroups",mod,"");
		}
	}
trimit:;
	
	for (i=0; i<ngroups; i++) {
	  if (!trim_ok[i]) {
	 	if (group_match(mgroups,group_namelc(i))) {
    	                wmsg("rmgroup %s \n",group_name(i));
                        group_setlife(i,0);
                        group_setcached(i,FALSE);
                        group_delete(i);
			ndone++;
 		} else dmsg("suck: Leaving group %s as it doesn't match suck groups\n",group_name(i));
	  }
	}

	imsg("matchgroups completed, deleted %d, added %d, undeleted %d, %s %s\n",ndone,nadded,nun,match_site,match_groups);
	intrim = FALSE;
	if (trim_ok!=NULL) myfree(trim_ok);
	trim_ok = NULL;
	strcpy(match_groups,"");
	strcpy(match_site,"");
}


void do_names(void)
{
        char *s;
        int i;
        /*RHP array>32k so replaced with malloc version*/
        char **grps; /*char *grps[max_groups];   ANOTHER NAUGTY ONE */
        int ngrps=0;

	if (newlist==NULL) {
		emsg("suck: Didn't successfully get list of names 'list newsgroups' failed\n");
		return;
	}
	if (str_len(newlist)==0) {
		emsg("suck: Didn't successfully get list of names 'list newsgroups' returned no names on upstream site\n");
		return;
	}
        grps = mymalloc(max_groups * sizeof (char *));
        if (grps==NULL) {emsg("suck: Memory  allocation failure\n"); return;}/*RHP is return ok?*/
        s = strtok(str_s(newlist),EOL);
        for (;s!=NULL;) {
                if (ngrps>=max_groups) {
                        emsg("Too many groups\n");
                        goto cont;
                }
                grps[ngrps++] = mystrdup(s,SD_SUCK);
                if (grps[ngrps-1]==NULL) emsg("Generate mystrdup failed\n");
                s = strtok(NULL,EOL);
        } 
cont:
        wmsg("suck: do_names, got %d names \n",ngrps);
        for (i=0; i<ngrps; i++) group_names_add(grps[i]);
        dmsg("suck: groups_write\n");
        group_names_write();
        str_copys(newlist,"",0); 
        dmsg("suck: done do_names\n");
	if (grps!=NULL) myfree(grps); /*RHP bettter free it again*/
}
/* comp.sys.atari.announce 0000000049 0000000050 m */
void gen_line(char *rec)
{
        char grp[200],mod[200];
        int g,i;
	double oldp;
	int start,end,fend,lstart;
        char *s;

        s = strtok(rec," \n\t"); if (s==NULL) return;
        ncpy(grp,s,150); 
        s = strtok(NULL," \n\t"); if (s==NULL) return;
        end = atoi(s);
        s = strtok(NULL," \n\t"); if (s==NULL) return;
        start = atoi(s);
        s = strtok(NULL," \n\t"); if (s==NULL) return;
        ncpy(mod,s,199);
        dmsg("group: Gen Line {%s} \n",grp);
        g = group_find(grp);  
	if (config_true(C_GETGROUPS_UNDELETE)) {
	 if (g>=0) if (group_isdeleted(g)) {
		group_setmod(g,mod);
	 }
	}
        if (g>=0) if (!group_isdeleted(g)) {
		if (!config_true(C_ACTIVE_STATIC)) {
			if (!config_true(C_FIXMOD_DISABLE)) group_setmod(g,mod);
		}
	}
        if (g<0) {
                dmsg("group: Gen Line NEW GROUP {%s} \n",grp);
                if (strlen(grp)==0) return;
                if (!group_wanted(grp)) return;

		if (!inactive) {
			if (strcmp(suck_groups[get_n],"*")!=0) {
			   if (!group_match(suck_groups[get_n],grp)) return;
			}
			if (!suck_infile(get_n,grp)) return;
			if (config_true(C_ACTIVE_STATIC)) return;
		}
                imsg("suck: Adding new group {%s} \n",grp);
                g = group_new(grp,"slave",mod,"");
                if (g<0) return;
		if (config_true(C_REPLICATE)) {
                	group_setstart(g,start);
                	group_setend(g,start-1);
		} else {
			lstart = start;
			if (lstart>99999999) lstart = 10;
                	group_setstart(g,lstart);
                	group_setend(g,lstart-1);
			if (lstart<3) {
	                	group_setstart(g,lstart+2);
	                	group_setend(g,lstart-1+2);
			}
		}
                group_setrange(g,0);
                group_setfeedend(g,0,0);
                group_setfeedend(g,1,0);
                group_setfeedend(g,2,0);
                group_setfeedend(g,3,0);
                group_setfeedend(g,get_n,start-1);
        }
	if (inactive) return;
	if (get_n==0) {
           if (group_iscachedreal(g)) if (!group_isdeleted(g)) {
                fend = group_feedend(g,0);
		oldp = 100.0 - atof(config_get(C_SUCK_PERCENT));
		if (oldp<100.0) {
			i = (int) (end - ((oldp/100.0)*group_getrange(g)));	
			imsg("suck_percent, only sucking up to %d of %d (%d items in group)\n",
				i,end,group_getrange(g));
			end = i;
		}
		dmsg("suck: Group is cached %d {%s} start %d, end %d, fend %d\n",g,grp,start,end,fend);
		if (!config_true(C_SUCK_NOFIX)) {
		 if (fend > end) {
		 	imsg("Adjusting gen upto for {%s} from %d to %d \n",grp,fend,end);
			fend = end;
                        group_setcatchup(g,0,end);
			group_setfeedend(g,get_n,fend);
		 }
		}
                if (fend < end) {
                        dmsg("suck: and new items exist %d \n",end-fend+1);
			if (!suck_infile(0,group_name(g))) goto nextone;
			if (strcmp(suck_groups[0],"*")!=0) {
                	   if (!group_match(suck_groups[0],group_namelc(g))) {
			   	goto nextone;
			   }
			}
                        suck_queue(grp,1);
                        group_setcatchup(g,0,end);
nextone:;
                }
          }
	}
}

void gen_addline(char *rec)  /* this routine isn't used now*/
{
        char grp[200],mod[200];
        int g;
        int start,end,fend;

        ncpy(grp,get_word(rec,0),150);
        end = atoi(get_word(rec,1));
        start = atoi(get_word(rec,2));
        ncpy(mod,get_word(rec,3),100);
        if (strlen(grp)==0) return;
        if (!group_wanted(grp)) return;
        imsg("suck: Adding new group {%s} \n",grp);
        g = group_new(grp,"slave",mod,"");
        if (g<0) return;
        group_setstart(g,start);
        group_setend(g,start-1);
        group_setrange(g,0);
        group_setfeedend(g,0,start-1);
        group_setfeedend(g,1,0);
        group_setfeedend(g,2,0);
        group_setfeedend(g,3,0);
        if (group_iscachedreal(g))  {
                fend = group_feedend(g,0);
                dmsg("suck: Group is cached %d {%s} start %d, end %d, fend %d\n",g,grp,start,end,fend);
                if (fend < end) {
                        dmsg("suck: and new items exist %d \n",end-fend);
                        suck_queue(grp,1);
                        group_setcatchup(g,0,end);
                }
        }
}
void gen_free(void)
{
        return;
}
int suck_queue(char *grp, int pri)
{
	int g;
	dmsg("suck: suck_queue %s %d\n",grp,pri);
	if (is_slave()) return TRUE;
	if ((pri==10) || (pri==11)) {
		return suck_queue_feed(grp,pri,0);
	}
	g =  group_find(grp);
	if (g<0) {
		emsg("suck_queue (%s) group doesn't exist\n",grp);
		return TRUE;
	}
	if (!pull_suck(0,g,FALSE)) {
		return suck_queue_feed(grp,pri,0);
	} 
	return TRUE;
}
int suck_queue_anyonex(char *grp, int pri, char *file, int line)
{
	int s = suck_which(grp);
	int g;
	if (is_slave()) return TRUE;
	if (strcmpnc(grp,cur_group)==0) {
		imsg("Not requeing group as we are still sucking it %s %s:%d\n",grp,file,line);
		return TRUE;
	}

	if (s<0) {
		imsg("Cannot find site to suck (%s) from. (ignore if local group or ihave feed)\n",grp);
		return TRUE;
	}
	dmsg("suck: suck_queue_anyone %s %d %s:%d\n",grp,pri,file,line);
	if (config_true(C_SUCK_LATER)) {
		imsg("Not sucking group %s right now because suck_later is set\n",grp);
		return TRUE;			
	}
	g = group_find(grp);
	if (g<0) {
		imsg("suck_que_anyone(%s) group doesn't exist\n",grp);
		return TRUE;
	}
	if (!pull_suck(s,g,FALSE)) {
		return suck_queue_feed(grp,pri,s);
	} 
	return TRUE;
}
int suck_queue_feed_pri(char *grp, int pri,  int feed)
{
        int i;
	if (is_slave()) return TRUE;
        if ((nque+10)>=max_groups) {
		emsg("fatal: Too many groups in update queue, panic");
		return FALSE;
        }
	if (config_true(C_SUCK_DISABLE)) if ((pri==0) || (pri==1)) {
		dmsg("suck: suck_disable'd so not fetching %s\n",grp);
		return TRUE;
	}
        for (i=0; i<nque; i++) {
		if ((strcmp(que[i],grp)==0) && (feed==que_feed[i])) goto removeit;
        }
	goto insertit;
removeit:;
        mystrfree(que[i],SD_SUCK);
        for (; i<nque; i++) {
                que[i] = que[i+1];
                que_pri[i] = que_pri[i+1];
                que_feed[i] = que_feed[i+1];
        }
	nque--;
insertit:;
        for (i=nque; i>0; i--) {
                que[i] = que[i-1];
                que_pri[i] = que_pri[i-1];
                que_feed[i] = que_feed[i-1];
        }
	que_pri[0] = pri;
	que_feed[0] = feed;
	sucklog("Insert %s to queue entry %d\n",grp,0);
	que[0] = mystrdup(grp,SD_SUCK);    
	nque++;
	return TRUE;
}
int suck_queue_feed(char *grp, int pri,  int feed)
{
        int i;
        if ((nque+10)>=max_groups) {
		emsg("fatal: Too many groups in update queue, panic");
		return FALSE;
        }
	if (config_true(C_SUCK_DISABLE)) if ((pri==0) || (pri==1)) {
		dmsg("suck: suck_disable'd so not fetching %s\n",grp);
		return TRUE;
	}
        
	if (nque>1000) if (feed_nocheck) goto isok;
	if (nque>20000) if (feed_nocheck) {
		imsg("suck: not queuein (%s) g more than 20K groups\n",grp);
	}
	for (i=0; i<nque; i++) {
		if ((strcmp(que[i],grp)==0) && (feed==que_feed[i])) return TRUE; /* Already in list */
        }
isok:
	que_pri[nque] = pri;
	que_feed[nque] = feed;
	sucklog("Adding %s to queue entry %d\n",grp,nque);
	que[nque++] = mystrdup(grp,SD_SUCK);    

	if (que[nque-1]==NULL) emsg("fatal: Memory allocation failed in suck_que\n");
	return TRUE;
}
void suck_update_clear(void)
{
 	int i;
	char *s;
	stopnow = TRUE; /* Stop when this item has arrived */
	for (i=0; i<20000; i++) {
		s = suck_getnext();
		if (s==NULL) return;
	}
	for (;cmdq_pop()!=NULL;) {

	}

}
int suck_ismail(void)
{
	int i;
	 for (i=0; i<nque; i++) {
		if (que_pri[i]==11) return TRUE;
       	 }	
	return FALSE;
}
char *suck_getnext(void)  /*Pop item off bottom of queue.*/
{
        static char tmp[200];
        int i;
        int pop;
        isafeed = FALSE;
        isamail = FALSE;
        if (nque==0) return NULL;
/*      if (nque>0) {
                dmsg("Queue of things to suck\n");
                for (i=0; i<nque; i++) {
                        dmsg("que[%d] = %s \n",i,que[i]);
                }
        }
*/
	if (!lib_checkspace()) {
	 for (i=0; i<nque; i++) {
		if (que_pri[i]==11) goto this_one; /* Do email if out of disk space */
       	 }	
	}
        for (i=0; i<nque; i++) {
                if (que_pri[i]==10) {
			if (suck_dontretry(que[i])) {
				imsg("Skipping {%s} until retry time is up\n",que[i]); 
				continue;
			}
                	goto this_one; /* Do feeds first */
		}
        }
        for (i=0; i<nque; i++) {
                if (que_pri[i]==0) goto this_one; /* Then fetch new groups  */
        }
        for (i=0; i<nque; i++) {
                if (que_pri[i]==10) goto this_one; 
                if (que_pri[i]==11) goto this_one;
        }
        i = 0;
this_one:
        pop = i;
        
        ncpy(tmp,que[pop],100);
	pop_feed = que_feed[pop];
        if (que_pri[pop]==10) isafeed = TRUE;
        if (que_pri[pop]==11) isamail = TRUE;

        mystrfree(que[pop],SD_SUCK);
        for (i=pop; i<nque; i++) {
                que[i] = que[i+1];
                que_pri[i] = que_pri[i+1];
                que_feed[i] = que_feed[i+1];
        }
        
        nque--;
        dmsg("suck: Pop group {%.80s} \n",tmp);
        return tmp;
}
int suck_quelen(void)
{
 	return nque;
}
char *suck_showque(int i)
{
	static char bf[BFSZ];

	if (que_pri[i] == 0) {
             lprintf(bf,BFSZ-1,"  Group %s    pending \015\012",que[i]);
	}
	if (que_pri[i] >= 10) {
             lprintf(bf,BFSZ-1,"  Feed %s    pending \015\012",que[i]);
	}
	if (que_pri[i] == 1) {
	     lprintf(bf,BFSZ-1,"  Group %s    pending (%s)\015\012",que[i],suck_name[que_feed[i]]);
	}
	return bf;
}
char *suck_isbusy(void)
{
	if ((state==F_IDLE) || (state==F_EXPIRE)) 
		return "181 DNEWS is not busy\r\n";
	else
		return "481 DNEWS is still busy\r\n";
}
int is_expring(void)
{
	if (is_expiring()) return TRUE;
	if (state==F_EXPIRE) return TRUE;
	return FALSE;
}
void suck_queue_show(String *body)
{
        char bf[BFSZ];
        int i;
        int g,np;
        char *type;

        if (nque==0 && state==F_IDLE) {
                str_addsn(body,"Nothing is being sucked at present.\015\012");
                str_addsn(body,lastsuck);
                str_addsn(body,lastsuck2);
                return;
        }
        lprintf(bf,BFSZ-1,"Contents of %s group fetching queue, %d entries:\015\012",DNEWS_NAME,nque);
        str_adds(body,bf,strlen(bf));
        str_addsn(body,lastsuck);
        str_addsn(body,lastsuck2);

        if (!lib_checkspace()) {
                lprintf(bf,BFSZ-1," *** Feeds PAUSED due to disk space shortage *** \015\012");
                str_adds(body,bf,strlen(bf));
        }
        if (state==F_EXPIRE) {
                lprintf(bf,BFSZ-1,"    Background process running job %s\015\012",expire_getmode());
                str_adds(body,bf,strlen(bf));
        }
        if (inlist) {
	        if (newlist==NULL) newlist = str_new();
                lprintf(bf,BFSZ-1,"    Getting list of news groups (%d bytes)\015\012",str_len(newlist));
                str_adds(body,bf,strlen(bf));
        }
        if (state==F_GENERATE) {
                lprintf(bf,BFSZ-1,"    Adding new groups from feeder %d/%d\015\012",gen_upto,gen_len);
                str_adds(body,bf,strlen(bf));
        }
        if (gname!=NULL) {
                type = "";
                if (isafeed) type = "(Feed)";
                if (isamail) type = "(SMTP)";
                g = group_find(gname);
                if (g>=0) {
                        lprintf(bf,BFSZ-1,"    Group %s    fetching, got %d items %s\015\012",
                                gname,stat_n,type);
                        str_adds(body,bf,strlen(bf));
                } else {
                        lprintf(bf,BFSZ-1,"    Feed %s    %s\015\012",gname,type);
                        str_adds(body,bf,strlen(bf));
                }
        }               
	for (i=0; i<cmdq_len(); i++) {
		lprintf(bf,BFSZ-1,"    cmd: %s    pending\015\012",		    cmdq_name(i));
		str_adds(body,bf,strlen(bf));
	}
	for (i=0,np=0; i<nque && np<6; i++) {
                if (que_pri[i]==0) {
                        lprintf(bf,BFSZ-1,"    Group %s    pending \015\012",que[i]);
                        str_adds(body,bf,strlen(bf));
                        np++;
                }
        }
        for (i=0,np=0; (i<nque) && (np<6) ; i++) {
                if ((que_pri[i]==10) || (que_pri[i]==11)) {
                        np++;
                        lprintf(bf,BFSZ-1,"    Feed %s    pending\015\012",que[i]);
                        str_adds(body,bf,strlen(bf));
                }
        }
        for (i=0,np=0; (i<nque) && (np<6) ; i++) {
                if (que_pri[i]==1) {
                        np++;
                        lprintf(bf,BFSZ-1,"    Group %s    pending (%s)\015\012",que[i],suck_name[que_feed[i]]);
                        str_adds(body,bf,strlen(bf));
                } 
        }
        strcpy(bf,"End of queue.\015\012\015\012");
        str_adds(body,bf,strlen(bf));
}
void suck_queue_hide(String *body, int gwant)
{
        char bf[BFSZ];
        int i;
	char gn[BFSZ];
	char *xname;
        int g,np;
        char *type;

	if (pull_true(0)) {
		str_addsn(body,pull_status());
		return;
	}
	ncpy(gn,group_name(gwant),100);
        if (nque==0 && state==F_IDLE) {
                str_addsn(body,"Nothing is being sucked at present.\015\012");
                if (strstr(lastsuck,gn)!=NULL) str_addsn(body,lastsuck);
                if (strstr(lastsuck2,gn)!=NULL) str_addsn(body,lastsuck2);
                return;
        }
        lprintf(bf,BFSZ-1,"Contents of %s group fetching queue, %d entries:\015\012",DNEWS_NAME,nque);
        str_adds(body,bf,strlen(bf));
        if (strstr(lastsuck,gn)!=NULL) str_addsn(body,lastsuck);
        if (strstr(lastsuck2,gn)!=NULL) str_addsn(body,lastsuck2);

        if (!lib_checkspace()) {
                lprintf(bf,BFSZ-1," *** Feeds PAUSED due to disk space shortage *** \015\012");
                str_adds(body,bf,strlen(bf));
        }
        if (state==F_EXPIRE) {
                lprintf(bf,BFSZ-1,"    Background process running job %s\015\012",expire_getmode());
                str_adds(body,bf,strlen(bf));
        }
        if (inlist) {
	        if (newlist==NULL) newlist = str_new();
                lprintf(bf,BFSZ-1,"    Getting list of news groups (%d bytes)\015\012",str_len(newlist));
                str_adds(body,bf,strlen(bf));
        }
        if (state==F_GENERATE) {
                lprintf(bf,BFSZ-1,"    Adding new groups from feeder %d/%d\015\012",gen_upto,gen_len);
                str_adds(body,bf,strlen(bf));
        }
        if (gname!=NULL) {
                type = "";
                if (isafeed) type = "(Feed)";
                if (isamail) type = "(SMTP)";
                g = group_find(gname);
                if (g>=0) {
			if (g==gwant) xname = gname; else xname="other.users.request";
                        lprintf(bf,BFSZ-1,"    Group %s    fetching, got %d items %s\015\012",
                                xname,stat_n,type);
                } else {
                        lprintf(bf,BFSZ-1,"    Feed %s    %s\015\012",gname,type);
                        str_adds(body,bf,strlen(bf));
                        if (strstr(bf,gname)!=NULL) str_adds(body,bf,strlen(bf));
                }
        }               
	for (i=0; i<cmdq_len(); i++) {
		lprintf(bf,BFSZ-1,"    cmd: %s    pending\015\012",		    cmdq_name(i));
		str_adds(body,bf,strlen(bf));
	}
	for (i=0,np=0; i<nque && np<6; i++) {
                if (que_pri[i]==0) {
			if (strcmpnc(que[i],gn)==0) 
	                        lprintf(bf,BFSZ-1,"    Group %s    pending \015\012",que[i]);
			else 
	                        lprintf(bf,BFSZ-1,"    Group other.users.request    pending \015\012");
                        str_adds(body,bf,strlen(bf));
                        np++;
                }
        }
        for (i=0,np=0; (i<nque) && (np<6) ; i++) {
                if ((que_pri[i]==10) || (que_pri[i]==11)) {
                        np++;
			lprintf(bf,BFSZ-1,"    Feed %s    pending\015\012",que[i]);
                        str_adds(body,bf,strlen(bf));
                }
        }
        for (i=0,np=0; (i<nque) && (np<6) ; i++) {
                if (que_pri[i]==1) {
                        np++;
			if (strcmpnc(que[i],gn)==0) 
	                        lprintf(bf,BFSZ-1,"    Group %s    pending (%s)\015\012",que[i],suck_name[que_feed[i]]);
			else
				lprintf(bf,BFSZ-1,"    Group other.users.request    pending (%s)\015\012",suck_name[que_feed[i]]);
                        str_adds(body,bf,strlen(bf));
                } 
        }
        strcpy(bf,"End of queue.\015\012\015\012");
        str_adds(body,bf,strlen(bf));
}
/* Check if we should be doing anything */
/* SHOULD DO EXPIRE */
/* SHOULD DO AGE */
void send_site(char *sitetime);
void suck_sched(void)
{
        static int last_hour,last_min,last_wday=-2;
	char bf[BFSZ];

	static int expire_hour;
	time_t now;
        static int loaded;
        struct tm *t; 
	struct tm atm;
	static int last_mon= -1;
        char send_at[BFSZ];

	strcpy(send_at,config_get(C_SEND_AT));
	if (strlen(send_at)==0) strcpy(send_at,"20 *");
        now = time(NULL);
        t = safe_localtime(&now,&atm);


	if (last_mon>=0) if (t->tm_mon!=last_mon) {
 		if (!(is_readonly()||get_expire_mode())) group_monthly(); /* brian */
	}
	last_mon = t->tm_mon;

	if (last_wday>=0) if (t->tm_wday != last_wday) {
		/* Work out key age once a day at least */
		lib_tz_reset(); /* Check timezone once a day */ 
		keylib_uncache();
		nt_getkey(bf,0);
		
		if (!config_true(C_LOG_NODAILY)) {
		 imsg("Rotating log file as wday changed\n");
		 slave_flush(TRUE);
/*		 sleep(1);*/
		 emsg_rotate();
		 slave_flush(FALSE);
		}
		group_clearxsort();
		if (!is_readonly) rej_rotate();
	}
        last_wday = t->tm_wday;
	
	if (is_readonly()||get_expire_mode()) return; /* brian */



        if ((t->tm_hour!=last_hour) || (t->tm_min!=last_min)) {
	        if (!loaded) suck_load(); loaded = TRUE;
		if (conf_isat(config_get(C_GUP_AT))) gup_send();

		if (conf_isat(send_at)) feed_doit();
		send_site(config_get(C_SEND_AT_SITE1));
		send_site(config_get(C_SEND_AT_SITE2));
		send_site(config_get(C_SEND_AT_SITE3));
		send_site(config_get(C_SEND_AT_SITE4));
		send_site(config_get(C_SEND_AT_SITE5));
		send_site(config_get(C_SEND_AT_SITE6));
		send_site(config_get(C_SEND_AT_SITE7));
		send_site(config_get(C_SEND_AT_SITE8));
		send_site(config_get(C_SEND_AT_SITE9));
		if (conf_isat(config_get(C_GROUPS_AT))) suck_update_groups_nq(0);
		if (conf_isat(config_get(C_GROUPS_AT2))) suck_update_groups_nq(1);
		if (conf_isat(config_get(C_GROUPS_AT3))) suck_update_groups_nq(2);
		if (conf_isat(config_get(C_GROUPS_AT4))) suck_update_groups_nq(3);
		if (conf_isat(config_get(C_GROUPS_AT5))) suck_update_groups_nq(4);
		if (conf_isat(config_get(C_GROUPS_AT6))) suck_update_groups_nq(5);
		if (conf_isat(config_get(C_GROUPS_AT7))) suck_update_groups_nq(6);
		if (conf_isat(config_get(C_GROUPS_AT8))) suck_update_groups_nq(7);
		if (conf_isat(config_get(C_GROUPS_AT9))) suck_update_groups_nq(8);
		if (conf_isat(config_get(C_GROUPS_AT10))) suck_update_groups_nq(9);
		if (conf_isat(config_get(C_GROUPS_AT11))) suck_update_groups_nq(10);
		if (conf_isat(config_get(C_GROUPS_AT12))) suck_update_groups_nq(11);
		if (conf_isat(config_get(C_GROUPS_AT13))) suck_update_groups_nq(12);
		if (conf_isat(config_get(C_GROUPS_AT14))) suck_update_groups_nq(13);
		if (conf_isat(config_get(C_GROUPS_AT15))) suck_update_groups_nq(14);
		if (conf_isat(config_get(C_GROUPS_AT16))) suck_update_groups_nq(15);
		if (conf_isat(config_get(C_GROUPS_AT17))) suck_update_groups_nq(16);
		if (conf_isat(config_get(C_GROUPS_AT18))) suck_update_groups_nq(17);
		if (conf_isat(config_get(C_GROUPS_AT19))) suck_update_groups_nq(18);
		if (conf_isat(config_get(C_GROUPS_AT20))) suck_update_groups_nq(19);
		if (conf_isat(config_get(C_GROUPS_AT21))) suck_update_groups_nq(20);
		if (conf_isat(config_get(C_GROUPS_AT22))) suck_update_groups_nq(21);
		if (conf_isat(config_get(C_GROUPS_AT23))) suck_update_groups_nq(22);
		if (conf_isat(config_get(C_GROUPS_AT24))) suck_update_groups_nq(23);
		if (conf_isat(config_get(C_GROUPS_AT25))) suck_update_groups_nq(24);
		if (conf_isat(config_get(C_GROUPS_AT26))) suck_update_groups_nq(25);
		if (conf_isat(config_get(C_GROUPS_AT27))) suck_update_groups_nq(26);
		if (conf_isat(config_get(C_GROUPS_AT28))) suck_update_groups_nq(27);
		if (conf_isat(config_get(C_GROUPS_AT29))) suck_update_groups_nq(28);
                if (conf_isat("50 11")) group_age();
                if (conf_isat(config_get(C_UPDATE_AT))) {
                  if (strlen(config_get(C_NNTP_SUCK1))>0) {
                        imsg("suck: Scheduling UPDATE hour=%d min=%d \n",t->tm_hour,t->tm_min);
                        suck_update_cached();
                  }
                }
                if (strlen(config_get(C_USER_CACHE_CLEAR))>0) if (conf_isat(config_get(C_USER_CACHE_CLEAR))) 
			user_cache_free();
                if (strlen(config_get(C_NNTP_SUCK2))>0) if (conf_isat(config_get(C_UPDATE_AT2))) suck_update_n(1); 
                if (strlen(config_get(C_NNTP_SUCK3))>0) if (conf_isat(config_get(C_UPDATE_AT3))) suck_update_n(2); 
                if (strlen(config_get(C_NNTP_SUCK4))>0) if (conf_isat(config_get(C_UPDATE_AT4))) suck_update_n(3); 
                if (strlen(config_get(C_NNTP_SUCK5))>0) if (conf_isat(config_get(C_UPDATE_AT5))) suck_update_n(4); 
                if (strlen(config_get(C_NNTP_SUCK6))>0) if (conf_isat(config_get(C_UPDATE_AT6))) suck_update_n(5); 
                if (strlen(config_get(C_NNTP_SUCK7))>0) if (conf_isat(config_get(C_UPDATE_AT7))) suck_update_n(6); 
                if (strlen(config_get(C_NNTP_SUCK8))>0) if (conf_isat(config_get(C_UPDATE_AT8))) suck_update_n(7); 
                if (strlen(config_get(C_NNTP_SUCK9))>0) if (conf_isat(config_get(C_UPDATE_AT9))) suck_update_n(8); 
                if (strlen(config_get(C_NNTP_SUCK10))>0) if (conf_isat(config_get(C_UPDATE_AT10))) suck_update_n(9); 
                if (strlen(config_get(C_NNTP_SUCK11))>0) if (conf_isat(config_get(C_UPDATE_AT11))) suck_update_n(10); 
                if (strlen(config_get(C_NNTP_SUCK12))>0) if (conf_isat(config_get(C_UPDATE_AT12))) suck_update_n(11); 
                if (strlen(config_get(C_NNTP_SUCK13))>0) if (conf_isat(config_get(C_UPDATE_AT13))) suck_update_n(12); 
                if (strlen(config_get(C_NNTP_SUCK14))>0) if (conf_isat(config_get(C_UPDATE_AT14))) suck_update_n(13); 
                if (strlen(config_get(C_NNTP_SUCK15))>0) if (conf_isat(config_get(C_UPDATE_AT15))) suck_update_n(14); 
                if (strlen(config_get(C_NNTP_SUCK16))>0) if (conf_isat(config_get(C_UPDATE_AT16))) suck_update_n(15); 
                if (strlen(config_get(C_NNTP_SUCK17))>0) if (conf_isat(config_get(C_UPDATE_AT17))) suck_update_n(16); 
                if (strlen(config_get(C_NNTP_SUCK18))>0) if (conf_isat(config_get(C_UPDATE_AT18))) suck_update_n(17); 
                if (strlen(config_get(C_NNTP_SUCK19))>0) if (conf_isat(config_get(C_UPDATE_AT19))) suck_update_n(18); 
                if (strlen(config_get(C_NNTP_SUCK20))>0) if (conf_isat(config_get(C_UPDATE_AT20))) suck_update_n(19); 
                if (strlen(config_get(C_NNTP_SUCK21))>0) if (conf_isat(config_get(C_UPDATE_AT21))) suck_update_n(20); 
                if (strlen(config_get(C_NNTP_SUCK22))>0) if (conf_isat(config_get(C_UPDATE_AT22))) suck_update_n(21); 
                if (strlen(config_get(C_NNTP_SUCK23))>0) if (conf_isat(config_get(C_UPDATE_AT23))) suck_update_n(22); 
                if (strlen(config_get(C_NNTP_SUCK24))>0) if (conf_isat(config_get(C_UPDATE_AT24))) suck_update_n(23); 
                if (strlen(config_get(C_NNTP_SUCK25))>0) if (conf_isat(config_get(C_UPDATE_AT25))) suck_update_n(24); 
                if (strlen(config_get(C_NNTP_SUCK26))>0) if (conf_isat(config_get(C_UPDATE_AT26))) suck_update_n(25); 
                if (strlen(config_get(C_NNTP_SUCK27))>0) if (conf_isat(config_get(C_UPDATE_AT27))) suck_update_n(26); 
                if (strlen(config_get(C_NNTP_SUCK28))>0) if (conf_isat(config_get(C_UPDATE_AT28))) suck_update_n(27); 
                if (strlen(config_get(C_NNTP_SUCK29))>0) if (conf_isat(config_get(C_UPDATE_AT29))) suck_update_n(28); 
                if (strlen(config_get(C_NNTP_SUCK30))>0) if (conf_isat(config_get(C_UPDATE_AT30))) suck_update_n(29); 

                if (strlen(config_get(C_NNTP_SUCK1))>0) {
                  suck_isat(config_get(C_UPDATE_GROUPS1)); 
                  suck_isat(config_get(C_UPDATE_GROUPS2)); 
                  suck_isat(config_get(C_UPDATE_GROUPS3)); 
                }
		if (config_true(C_EXPIRE_MINUTES)) {
		        if (conf_isat(config_get(C_EXPIRE_AT))) {
 		            if (!suck_spawned_expire()) {	/* brian */
 			        sched_expire = TRUE; isexpire = TRUE;
		                imsg("suck: Scheduling EXPIRE hour=%d min=%d \n",t->tm_hour,t->tm_min);
			    } else {
 				imsg("suck: expire not scheduled because spawned expire already running hour=%d min=%d \n",t->tm_hour,t->tm_min); 
 			    }
		        }
		}
                last_hour = t->tm_hour; last_min = t->tm_min;
        }

	if (!config_true(C_EXPIRE_MINUTES)) {
          if ((t->tm_hour!=expire_hour)) {
	        if (conf_isat_hour(config_get(C_EXPIRE_AT))) {
	        	if (!suck_spawned_expire()) {	/* brian */
		            sched_expire = TRUE; isexpire = TRUE;
	                    imsg("suck: Scheduling EXPIRE hour=%d min=%d \n",t->tm_hour,t->tm_min);
			    group_write_backup();
			}
			else {
			    imsg("suck: expire not scheduled because spawned expire already running hour=%d min=%d \n",t->tm_hour,t->tm_min); 
			}
	        }
		expire_hour = t->tm_hour;
	  }
	}
}
void send_site(char *sitetime)
{
	char bf[BFSZ];
	char *s;
	strcpy(bf,sitetime);
	if (strlen(bf)==0) return;
	s = strchr(bf,' ');
	if (s==NULL) {emsg("Expecting \"site.name minutes hours \" %s\n",sitetime); return;}
	*s = 0; s++;
	if (conf_isat(s)) {
		imsg("Sending any que'd items due to sent_at_siteN rule (%s)\n",sitetime);
		feed_doit_name(bf);
	}
}

String *suck_status(void)
{
        static String *xbody;

        if (xbody==NULL) xbody = str_new();
        str_copysn(xbody,"");
        suck_queue_show(xbody);
        return xbody;
}
#define TEMP_FILE file_work("confirm.tmp")
void tell_user(String *xhead, String *xbody, char *status)
{
        char from[201];
        char subj[BFSZ];
        char bf[BFSZ];
        char me[BFSZ];
	char xstatus[BFSZ];
	char path[BFSZ];
	char *p[100];
	int npath;
        String *xnew;
        char *s;
	int postok = FALSE;
        
	ncpy(xstatus,status,100);
        ncpy(bf,config_get(C_CONFIRM),BFSZ-1); strlwr(bf);
        if (strlen(bf)==0) return;
        if (strlen(bf)<3) {emsg("Confirm setting must be at least 3 characters long {%s}\n",bf); return;}
	ncpy(path,get_ahead(xhead,"Path:"),200);
	npath = token_split(path,p,"!, \t\015\012",20);
	if (npath>3) {
		emsg("Confirm not sent as path suggests the post is not local %d\n",npath);
		return;
	}
        ncpy(from,get_ahead(xhead,"Reply-To:"),200);
        if (strlen(from)==0) ncpy(from,get_ahead(xhead,"From:"),200);
        strlwr(from);
        fix_address(from);
        strcpy(subj,"");

        lprintf(me,BFSZ-1,"%s@%s",DNEWS_NAME,config_get(C_MYNAME));    strlwr(me);
       	if (strlen(config_get(C_MAIL_FROM))>0) {
          strcpy(me,config_get(C_MAIL_FROM)); 
	}
	strlwr(from); strlwr(me);
        if (strstr(from,me)!=NULL) {imsg("Don't confirm our own messages\n"); return; }
	
	if ((atoi(status)>199) && (atoi(status)<400)) {
		postok = TRUE;
	}
	if (status[0]=='+') postok = TRUE;
	if (strstr(status,"moderator")!=NULL) postok = TRUE;

        wmsg("suck: Confirmation %s post to {%s} \n",postok ? "Successful" : "Failed" ,from);
        dmsg("suck: tell_user match {%s} {%s}  \n",from,bf);
        if (group_match(bf,from)) {
                fix_address(from);
                remove(TEMP_FILE);
                s = strchr(status,'\015'); if (s!=NULL) *s = 0;
                s = strchr(status,'\012'); if (s!=NULL) *s = 0;
                lprintf(subj,BFSZ-1,"News Posting %s {%s}",postok ? "Succeeded" : "Failed", status);
                xnew = str_new();
                lprintf(bf,BFSZ-1,"From: %s\015\012",me); str_copysn(xnew,bf);
                lprintf(bf,BFSZ-1,"Subject: %s\015\012",subj); str_addsn(xnew,bf);
                lprintf(bf,BFSZ-1,"To: %s\015\012",from); str_addsn(xnew,bf);
                lprintf(bf,BFSZ-1,"Date: %s\015\012",lib_email_date()); str_addsn(xnew,bf);
                str_addsn(xnew,"\015\012");
		if (postok) {
			if (lib_exists(file_work("upstream.ok"))) {
				str_addfile(xnew,file_work("upstream.ok"));
				lprintf(bf,BFSZ-1,"    %s\r\n",xstatus); str_addsn(xnew,bf);
			} else {
				lprintf(bf,BFSZ-1,"Dear user, your recent usenet post has been successfully sent upstream\r\n"); str_addsn(xnew,bf);
				lprintf(bf,BFSZ-1,"The status returned from the upstream site was:\r\n"); str_addsn(xnew,bf);
				lprintf(bf,BFSZ-1,"    %s\r\n",xstatus); str_addsn(xnew,bf);
				lprintf(bf,BFSZ-1,"A copy of the headers of the posted message follows\r\n\r\n"); str_addsn(xnew,bf);
			}
		} else {
			if (lib_exists(file_work("upstream.bad"))) {
				str_addfile(xnew,file_work("upstream.bad"));
				lprintf(bf,BFSZ-1,"    %s\r\n",xstatus); str_addsn(xnew,bf);
			} else {
				lprintf(bf,BFSZ-1,"Dear user, your recent usenet post was NOT sent upstream\r\n"); str_addsn(xnew,bf);
				lprintf(bf,BFSZ-1,"The status returned from the upstream site was:\r\n"); str_addsn(xnew,bf);
				lprintf(bf,BFSZ-1,"    %s\r\n",xstatus); str_addsn(xnew,bf);
				lprintf(bf,BFSZ-1,"A copy of the headers of the posted message follows\r\n\r\n"); str_addsn(xnew,bf);
				lprintf(bf,BFSZ-1,"Please contact your local news administrator for further advice\r\n\r\n"); str_addsn(xnew,bf);
			}
		}
		str_add(xnew,xhead);
                str_addsn(xnew,"\015\012");
                str_write(xnew,TEMP_FILE);
		if (xbody!=NULL) if (config_true(C_CONFIRM_BODY)) {
			str_append(xbody,TEMP_FILE);	
		}
                lib_mail_file2(from,subj,TEMP_FILE);
                str_free(xnew);
        } else {
                wmsg("suck: Confirmation not sent, wild didn't match {%s} \n",bf);
        }
}
int feed_reader(void)
{
        char bf[BFSZ];
        ncpy(bf,config_get(C_FEED_READER),BFSZ-1);
        strlwr(bf);
        if (strcmp(bf,"true")==0) return TRUE;
        if (strcmp(bf,"false")==0) return FALSE;
        return TRUE;
/*      return feed_byposting(); */
}

int suck_bygroup(void)
{
        int i,nc;
	int max=atoi(config_get(C_SUCK_MAXGROUPS));
        int list_min=atoi(config_get(C_LIST_MIN));
	if (pull_true(0)) list_min = 100000; /* Force fetching by group for pull mechanism*/
        if (list_min==0) list_min = 3000;
        nc=0;

       	for (i=0; i<group_n(); i++) if (group_iscachedreal(i)) nc++;
	if (!config_true(C_CACHEDALL)) 
	  if (max>0) if (nc>max) {
		emsg("suck: SUCK_MAXGROUPS exceeded %d>%d, uncaching groups\n",nc,max);
		group_uncache_most();
		suck_update_clear();

	  }
       	
	if (nc>list_min) return FALSE;
        if (strlen(config_get(C_NNTP_SUCK1))==0) return TRUE;

        for (i=0; i<group_n(); i++) if (!group_isdeleted(i)) if (group_iscachedreal(i)) {
		if (suck_infile(0,group_name(i)))
                  if (group_match(suck_groups[0],group_namelc(i))) {
			if (!pull_suck(0,i,FALSE)) {
		                suck_queue_feed(group_name(i),1,0);
			}
	                group_setcatchup(i,0,group_feedend(i,0));
		}
        }
        return TRUE;
}
int suck_infile(int n, char *gname)
{
	FILE *f;
	char bf[BFSZ];
	char fname[BFSZ];
	char *w;
	int lastm,m;
	char xgroup[BFSZ];
	char *s;

	if (n<0) return TRUE;
	if (suck_file[n]==NULL) return TRUE;	
	if (strlen(suck_file[n])==0) return TRUE;

	strcpy(fname,file_work(suck_file[n]));
	f = fopen(fname,"r");
	if (f==NULL) {
		filemsg("Unable to read (%s) \n",fname);
		return TRUE;
	}

	m = FALSE;
	ncpy(xgroup,gname,BFSZ-1);
	strlwr(xgroup);
        lastm = FALSE;
	for (;!feof(f);) {
		if (fgets(bf,BFSZ-10,f)==NULL) break;
		strlwr(bf);
		s = strchr(bf,' '); if (s!=NULL) *s = 0;
		s = strchr(bf,'\n'); if (s!=NULL) *s = 0;
		if (bf[0]==0) continue;
                m = TRUE;
		w = bf;
                if (*w=='!') {
                        w++;
                        m = FALSE;
                }
                if (match_wild(w,xgroup)) {
			lastm = m;
		} 
	}
	fclosep(f);
	return lastm;
}

/* feeder site groups,groups user password */
void suck_load(void)
{
        nsuck = 0;      

        suck_groups[nsuck] = "*";  /* Dont free, will break like this*/
        suck_user[nsuck] = "";
        suck_pass[nsuck] = "";
        suck_name[nsuck] = "";
	suck_file[0] = mystrdup(config_get(C_SUCK_FILE1),SD_SUCK);
	suck_file[1] = mystrdup(config_get(C_SUCK_FILE2),SD_SUCK);
	suck_file[2] = mystrdup(config_get(C_SUCK_FILE3),SD_SUCK);
	suck_file[3] = mystrdup(config_get(C_SUCK_FILE4),SD_SUCK);

        suck_addfeed(config_get(C_NNTP_SUCK1)); 
        suck_addfeed(config_get(C_NNTP_SUCK2)); 
        suck_addfeed(config_get(C_NNTP_SUCK3)); 
        suck_addfeed(config_get(C_NNTP_SUCK4)); 
        suck_addfeed(config_get(C_NNTP_SUCK5)); 
        suck_addfeed(config_get(C_NNTP_SUCK6)); 
        suck_addfeed(config_get(C_NNTP_SUCK7)); 
        suck_addfeed(config_get(C_NNTP_SUCK8)); 
        suck_addfeed(config_get(C_NNTP_SUCK9)); 
        suck_addfeed(config_get(C_NNTP_SUCK10)); 
        suck_addfeed(config_get(C_NNTP_SUCK11)); 
        suck_addfeed(config_get(C_NNTP_SUCK12)); 
        suck_addfeed(config_get(C_NNTP_SUCK13)); 
        suck_addfeed(config_get(C_NNTP_SUCK14)); 
        suck_addfeed(config_get(C_NNTP_SUCK15)); 
        suck_addfeed(config_get(C_NNTP_SUCK16)); 
        suck_addfeed(config_get(C_NNTP_SUCK17)); 
        suck_addfeed(config_get(C_NNTP_SUCK18)); 
        suck_addfeed(config_get(C_NNTP_SUCK19)); 
        suck_addfeed(config_get(C_NNTP_SUCK20)); 
        suck_addfeed(config_get(C_NNTP_SUCK21)); 
        suck_addfeed(config_get(C_NNTP_SUCK22)); 
        suck_addfeed(config_get(C_NNTP_SUCK23)); 
        suck_addfeed(config_get(C_NNTP_SUCK24)); 
        suck_addfeed(config_get(C_NNTP_SUCK25)); 
        suck_addfeed(config_get(C_NNTP_SUCK26)); 
        suck_addfeed(config_get(C_NNTP_SUCK27)); 
        suck_addfeed(config_get(C_NNTP_SUCK28)); 
        suck_addfeed(config_get(C_NNTP_SUCK29)); 
        suck_addfeed(config_get(C_NNTP_SUCK30)); 
	
}
void suck_addfeed(char *s)
{
        char bf[BFSZ];
        ncpy(bf,s,BFSZ-1);
        s = strtok(bf," \n\015\012\t"); 

        suck_groups[nsuck] = "";  /* Dont free, will break like this*/
        suck_user[nsuck] = "";
        suck_pass[nsuck] = "";
        suck_name[nsuck] = "";

        if (s!=NULL) {
                suck_groups[nsuck] = "*";  /* Dont free, will break like this*/
                suck_user[nsuck] = "";
                suck_pass[nsuck] = "";
                if (nsuck==0) {
                  suck_user[nsuck] = mystrdup(config_get(C_NNTP_USER),SD_SUCK);
                  suck_pass[nsuck] = mystrdup(config_get(C_NNTP_PASS),SD_SUCK);
                }
                suck_name[nsuck] = mystrdup(s,SD_SUCK);
                s = strtok(NULL," \n\015\012\t"); if (s==NULL) goto done;
                suck_groups[nsuck] = mystrdup(s,SD_SUCK);
                s = strtok(NULL," \n\015\012\t"); if (s==NULL) goto done;
                suck_user[nsuck] = mystrdup(s,SD_SUCK);
                s = strtok(NULL," \n\015\012\t"); if (s==NULL) goto done;
                suck_pass[nsuck] = mystrdup(s,SD_SUCK);
done:        	;   
        }
	nsuck++;
}
char * suck_getgroups(int n)
{
	if (n>=nsuck) return "";
	if (n<0) return "";
 	return suck_groups[n];
}

void suck_feeder(char *gname)
{
        int i;
        static int loaded;
        if (!loaded) suck_load(); loaded = TRUE;
	i = pop_feed;
	ncpy(cur_feeder,suck_name[i],100);
        ncpy(cur_user, suck_user[i],100);
        ncpy(cur_pass, suck_pass[i],100);
        imsg("suck: group %s from %s (%s ****) \n",gname,suck_name[i],suck_user[i]); 
}


int suck_which(char *gname)
{
        int i;
        static int loaded;
        if (!loaded) suck_load(); loaded = TRUE;
        for (i=nsuck-1; i>=0 ; i--) {
                if (group_match(suck_groups[i],gname)) return i;
        }
	return -1;
}
void suck_info(int i, char *host, char *user, char *pass)
{
	strcpy(host,suck_name[i]);
	strcpy(user,suck_user[i]);
	strcpy(pass,suck_pass[i]);
}

void suck_feeder_name(char *gname)
{
        static int loaded;
        int i;
        if (!loaded) suck_load(); loaded = TRUE;
        for (i=nsuck-1; i>=0 ; i--) {
                if (strcmp(suck_name[i],gname)==0) {
                        ncpy(cur_feeder,suck_name[i],100);
                        ncpy(cur_user, suck_user[i],100);
                        ncpy(cur_pass, suck_pass[i],100);
                        imsg("suck: upstream name %s (%s ****) \n",suck_name[i],suck_user[i]); 
                        return;
                }
        }                               
        imsg("suck: No user-pass for %s (probably fine, ignore)\n",gname); 
        ncpy(cur_feeder,gname,BFSZ-1);
        strcpy(cur_user,"");
        strcpy(cur_pass,"");
}
void suck_isat(char *gtime)
{
        int i;
        char *s;
        char bf[BFSZ],t[BFSZ],grps[BFSZ];
        ncpy(bf,gtime,BFSZ-1);
        s = strtok(bf," \n\t\015\012"); if (s==NULL) return;
        ncpy(grps,s,BFSZ-1);
        s = strtok(NULL," \n\t\015\012"); if (s==NULL) {emsg("suck: update_groups didn't have time minutes \n"); return;}
        ncpy(t,s,BFSZ-1);
        s = strtok(NULL," \n\t\015\012"); if (s==NULL) {emsg("suck: update_groups didn't have time hours \n"); return;}
        strcat(t," ");
        strcat(t,s);

        if (!conf_isat(t)) return;

        for (i=0; i<group_n(); i++) if (group_iscachedreal(i)) {
                if (group_match(grps,group_name(i))) {
                        imsg("suck: update_groupsN  {%s}\n",group_name(i)); 
                        suck_queue(group_name(i),1);
                        group_setcatchup(i,0,group_feedend(i,0));
                }
        }
        return;
}
char *suck_getsuckn(int n)
{
        char *s;
        static char bf[BFSZ];
        if (n==1) ncpy(bf,config_get(C_NNTP_SUCK1),BFSZ-1);
        if (n==2) ncpy(bf,config_get(C_NNTP_SUCK2),BFSZ-1);
        if (n==3) ncpy(bf,config_get(C_NNTP_SUCK3),BFSZ-1);
        if (n==4) ncpy(bf,config_get(C_NNTP_SUCK4),BFSZ-1);
        if (n==5) ncpy(bf,config_get(C_NNTP_SUCK5),BFSZ-1);
        if (n==6) ncpy(bf,config_get(C_NNTP_SUCK6),BFSZ-1);
        if (n==7) ncpy(bf,config_get(C_NNTP_SUCK7),BFSZ-1);
        if (n==8) ncpy(bf,config_get(C_NNTP_SUCK8),BFSZ-1);
        if (n==9) ncpy(bf,config_get(C_NNTP_SUCK9),BFSZ-1);
        if (n==10) ncpy(bf,config_get(C_NNTP_SUCK10),BFSZ-1);
        if (n==11) ncpy(bf,config_get(C_NNTP_SUCK11),BFSZ-1);
        if (n==12) ncpy(bf,config_get(C_NNTP_SUCK12),BFSZ-1);
        if (n==13) ncpy(bf,config_get(C_NNTP_SUCK13),BFSZ-1);
        if (n==14) ncpy(bf,config_get(C_NNTP_SUCK14),BFSZ-1);
        if (n==15) ncpy(bf,config_get(C_NNTP_SUCK15),BFSZ-1);
        if (n==16) ncpy(bf,config_get(C_NNTP_SUCK16),BFSZ-1);
        if (n==17) ncpy(bf,config_get(C_NNTP_SUCK17),BFSZ-1);
        if (n==18) ncpy(bf,config_get(C_NNTP_SUCK18),BFSZ-1);
        if (n==19) ncpy(bf,config_get(C_NNTP_SUCK19),BFSZ-1);
        if (n==20) ncpy(bf,config_get(C_NNTP_SUCK20),BFSZ-1);
        s = strtok(bf," \n\t");
        if (s==NULL) return "NNTP_SUCKn_NOT_DEFINED_PROPERLY";
        if (strlen(s)==0) return "NNTP_SUCKn_NOT_DEFINED_PROPERLY";
        return s;
}

char *email_fix(char *s)
{
	static char bf[BFSZ];
	static char bf2[BFSZ];
	char *ss=s;
	ncpy(bf,s,BFSZ-1);

	s = strtok(bf," \t\n");
	for (;s!=NULL;) {
	 	if (*s=='<') goto found;
		if (strchr(s,'@')!=NULL) goto found;
		s = strtok(NULL," \t\n");
	}
	return "<>";
found:
	if (*s=='<') {
		ncpy(bf2,s,BFSZ-1);
		return bf2;
	}
       	lprintf(bf2,BFSZ-1,"<%s>",s);
	return bf2;
}

static char *retry_name[10];
static int retry_after[10];
int suck_retryfind(char *s)
{
	int i;
 	for (i=0; i<10; i++) {
		if (retry_name[i]==NULL) continue;
		if (strcmp(s,retry_name[i])==0) return i;
	}
	return 9;
}
int suck_dontretry(char *s)
{
	int t = lib_gmtime();
	int r = suck_retryfind(s);
	if (t<retry_after[r]) return TRUE;
	return FALSE;	
}
void suck_trylater(char *s)
{
	int r;
 	imsg("Won't retry sending feed to {%s} for 10 minutes\n",s);
 	r = suck_retryfind(s);
	if (retry_name[r]==NULL) retry_name[r] = mystrdup(s,SD_SUCK);
	if (strcmp(retry_name[r],s)!=0) retry_name[r] = mystrdup(s,SD_SUCK);
	retry_after[r] = lib_gmtime() + 10*60;
}

void remove_header(String *xhead,char *hname)
{
	char *s,*s2;
	int len;
	if (xhead==NULL) return;
	s = str_s(xhead);
	if (s==NULL) return;
	for (;s!=NULL;) {
		if (strncmp(s,hname,strlen(hname))==0) goto removeit;
		s = strstr(s,"\015\012");
		if (s!=NULL) s += 2;
	}
	return;
removeit:
	s2 = strstr(s,"\015\012"); 	if (s2==NULL) return;
	s2 += 2;
	len = str_len(xhead) - (s2 - str_s(xhead));
	memmove(s,s2,len); /* copy the null as well */
	len = str_len(xhead) - (s2-s);
	str_setlen(xhead,len);	
}
#ifndef BIGBF
#define BIGBF 10000
#endif
void suck_post_file(char *fname);

void suck_posts(void)
{
	static Sary *ls;
	char bf[BFSZ];
	char tmp[BFSZ];
	static int last_write;
	int i;
	if (ls==NULL) ls = s_alloc();

	if (config_true(C_DISKIO_MIN)) return;
	s_clear(ls);
	s_wild(ls,file_work(""),"post*.rdr");

	if (s_len(ls)>0)  imsg("Found %d readonly posts\n",s_len(ls));

	for (i=0; i<s_len(ls); i++) {
		lprintf(tmp,BFSZ-1,"post%d.do",i);
		strcpy(bf,file_work(tmp));
		if (rename(file_work(s_line(ls,i)),bf)!=0) {
			filemsg("Rename of (%s) to (%s) failed\n",file_work(s_line(ls,i)),bf);	
		}
	}
	s_clear(ls);
	s_wild(ls,file_work(""),"post*.do");
	if (s_len(ls)==0) return;
	imsg("Found %d do posts \n",s_len(ls));
	for (i=0; i<s_len(ls); i++) {
		suck_post_file(file_work(s_line(ls,i)));
	}
	imsg("Finished do posts \n");
	last_write = lib_gmtime();
	db_flush2();
	hist_flush();
	group_write_changed_later();
}
void suck_post_item(String *xitem);
void suck_post_file(char *xfname)
{
	char *s;
	char bf[5000];
	String *xitem;
	FILE *f;
	char fname[BFSZ];
	ncpy(fname,xfname,BFSZ-1);
	xitem = str_new();
#ifdef WIN32
	f = fopen(fname,"rb");
#else
	f = fopen(fname,"r");
#endif
	if (f==NULL) {
		filemsg("Could not read file %s\n",fname);
		return;
	}
	for (;!feof(f);) {
		if (fgets(bf,4000,f)==NULL) break;
		s = strchr(bf,'\r'); if (s!=NULL) *s = 0;
		s = strchr(bf,'\n'); if (s!=NULL) *s = 0;
		if (strcmp(bf,".")==0) {
			suck_post_item(xitem);
			str_clear(xitem);
		} else {
			str_addsn(xitem,bf);
			str_addsn(xitem,"\r\n");
		}
	}
	fclosep(f);
	remove(fname);
	imsg("Removed %s\n",fname);
	str_free(xitem);
}
char *post_geterr(void);
void post_setmodgrp(int x);

int suck_post_mod(String *anitem)
{
        int ismod;
        char bf[BFSZ];
        int ngpoint;
        char *gpoint[1000];
        int g,gi;
	char grps[BFSZ];
	char opath[BFSZ];
	char msgid[BFSZ];
	int modgrp;

        ncpy(grps,get_ahead(anitem,"Newsgroups:"),BFSZ-1);
        strlwr(grps);
        ngpoint = token_split(grps,gpoint,", \t\n",200); /*MAX_GRPS*/ 
        ismod = FALSE;
        for (g=0; g<ngpoint; g++) {
                gi = group_find(gpoint[g]); 
		if (gi<0) continue;
                if (group_ismod(gi)) {
                        ismod = TRUE;
                        modgrp = gi; /* Remeber which group was moderated */
                }
        }
        if (!ismod) return FALSE;
	post_setmodgrp(modgrp);
	imsgf("Processing moderated post from readonly server\n");
	feed_setpost(TRUE);
        ncpy(opath,get_ahead(anitem,"Path:"),BFSZ-100);
        ncpy(msgid,get_ahead(anitem,"Message-ID:"),BFSZ-100);
	if (!feed_me(msgid,"",gpoint,NULL,opath,str_len(anitem),anitem)) {
		imsg("Moderated me feed failed %s\n",post_geterr());
		msg_reject(post_geterr());
		feed_setpost(FALSE);	
		return TRUE;
	} 
	feed_setpost(FALSE);
        if (post_moderated(anitem)) {
		msg_ok("Post moderated succeeded");
                feed_doit_if();
		return TRUE;
        } else {
		lprintf(bf,BFSZ-1,"441 %s\015\012",post_geterr());
                msg_reject("Post moderated failed: %s",post_geterr());
                return TRUE;
        }
}
void suck_post_item(String *xitem)
{
	char *fn;
	FILE *f;

	if (suck_post_mod(xitem)) return;

	feed_setpost(TRUE);

	if (!item_suck(xitem)) {
		if (item_tryagain()) {
			fn = "post.rdr";
		} else {
			fn = "post.er";
			stats_feed("readers",1,0,0,1,str_len(xitem),0.0,0.0);
		}
		f = fopen(file_work(fn),"ab");
		if (f!=NULL) {
			fwrite(str_s(xitem),1,str_len(xitem),f);
			fprintf(f,".\r\n");
			fclosep(f);
		}
	} else {
		stats_feed("readers",1,0,1,0,str_len(xitem),0.0,0.0);
	}
	feed_setpost(FALSE);
}

static char uucp_file[BFSZ];
static FILE *uuin;
void uucp_open(void);
void uucp_suckx(void);
void uucp_suck(void)
{
	static int cnt;

	if (!config_true(C_SUCK_UUCP_FASTER)) { 
		if (state==F_EXPIRE) if (cnt<5) { cnt++; return;}
	}
	cnt = 0;
	
	
	stat_start(X_UUCP);
	uucp_suckx();
	stat_end(X_UUCP);
}


void uucp_suckx(void)
{
 	static int lastt;
	static String *article;
	static int tryagain;
	static int hlen;
	char *s;
	int k;
	static int sk_start;
	static int sk_nok;
	static int sk_ndup;
	static int uline;
	static int next_hlen;
	static int sk_reject; 
	static double sk_size;
	char *last=NULL;
	char bf[BIGBF];
	int fiddle;
	int nfail;
	static int isplain;
	static int dorename;
	static int doneinit;
	int delay = 0;
	if (lastt==lib_gmtime()) return;
	lastt = lib_gmtime();
again:
	nfail = 0;
	if (config_true(C_SUCK_UUCP_FASTER)) {
		delay = 2;
	}

/*	dmsg("suck: uucp: about to process something\n");*/
	if (is_readonly()) return;
        if (!lib_checkspace()) {
		dmsg("suck: uucp paused as disk space is full\n");
		return;
	}

	if (config_true(C_UUCP_SIMPLE)) isplain = TRUE;
	if (uuin==NULL) { uucp_open(); hlen = 0; tryagain = FALSE; dorename = FALSE; uline = 0;} 


/*	dmsg("suck: uucp: found some files %p\n",uuin);*/
	if (!doneinit) {
		sk_start = lib_gmtime();
		doneinit = TRUE;
	}


	if (uuin==NULL) return;
	if (article==NULL) article = str_new();
	for (k=0;!feof(uuin);k++) {
	  if (k % 1000 == 99) if ((lib_gmtime()-delay)>lastt) break;
	  uline++;
	  if (next_hlen>0) hlen = next_hlen;
	  next_hlen = 0;
	  if (fgets(bf,BIGBF-10,uuin)!=NULL) {		
		fiddle = 0;
		s = strchr(bf,'\015'); if (s!=NULL) {
			if (s[1]=='\n') {
				*s = 0;
				fiddle = 1;
			}
		}
		s = strchr(bf,'\012'); if (s!=NULL) *s = 0;
		if (isplain) {
			if (strncmp(bf,"#! rnews",8)==0) {
				hlen = 0;
				if (str_len(article)>0) goto isearly;
				else continue;
			}
			goto normal;
		}
		if (hlen<0) imsg("uucp: hlen = %d, thats odd: %s\n",hlen,bf);
		if ((hlen<200) && (hlen>0)) if (strncmp(bf,"#! rnews",8)==0) {
			if (hlen<10) {
				imsg("uucp: rnews header %d bytes early, correcting on the fly\n",hlen);
				hlen = 0;
				next_hlen = atoi(bf+9);
				goto isearly;
			}
			dorename = TRUE;
			imsg("Looks like the rnews header is early, attempting to read article, line=%d, left=%d\n",uline,hlen);
			dmsg("suck: hlen=%d, line %d",hlen,uline);
			sk_reject++;
			str_clearbig(article);
			hlen = 0;
		}
		if (hlen<=0) {
			if (strncmp(bf,"#! rnews",8)!=0) {
				imsg("Invalid header record, %s, searching %d \n",bf,++nfail);
				dorename = TRUE;
				if (nfail>4000) goto fail;
				continue;
			}
			hlen = atoi(bf+9);
			if (hlen==0) {
				isplain = TRUE; 
				if (str_len(article)>0) goto isearly;
				continue;
			}
		} else {
			hlen -= (strlen(bf) + 1);
			if (config_true(C_UUCP_BINARY)) {
				hlen  -= fiddle;
			}
normal:			if (strcmp(bf,".")==0) strcpy(bf,"..");
			/* if (strlen(msgid)==0) {
			 	if (strncmp(bf,"Message-ID:",11)==0) {
					strcpy(msgid,bf+12);
				}
			} */
			s = bf + strlen(bf);
			strcpy(s,"\015\012");
			str_addsn(article,bf);
			if (str_len(article)>max_strbf()) {
				str_clear(article);
				emsg("Article exceeded item_max in suck\n");
			}
			if (isplain) continue;
		}
isearly:;	if (hlen<=0) {
			if (hlen<0) {
				imsg("End of item in middle of line, %d extra bytes\n",-hlen);
				dorename = TRUE;
				goto fail2;
			}
	                msg_from("suck_uucp");
			item_setuucp(TRUE);
			if (!item_suck(article)) {
				if (item_was_mod()) {
				 if (config_true(C_UUCP_SEND_MOD)) {
					if (suck_post_moderated(article)) goto wasok;
					else imsg("Moderated post from bag file failed\n");
				 } else {
					imsg("Bag file contained un-approved moderator post, set uucp_send_mod true to send these to moderator instead of rejecting\n");
				 }
				}

				item_setuucp(FALSE);
				if (item_iscrosspost()) sk_ndup++; else sk_reject++;
				if (item_tryagain()) tryagain = TRUE;
			} else {
wasok:				nfail = 0;
				sk_nok++;
				sk_size += str_len(article);
			}
			item_setuucp(FALSE);
fail2:			str_clearbig(article);
			/*strcpy(msgid,"");*/

			if (config_true(C_CANCEL_RELEASE)) {
				mutex_release(M_MAIN);
				imsg("Mutex release in sucking module\n");
				mutex_wait(M_MAIN);
			}
		}
		if (tryagain) goto fail;
	  } else {
	  	if (uuin!=NULL) if (!feof(uuin)) goto fail;
	  }
	}
	if (feof(uuin)) {
		fclosep(uuin);
                msg_from("suck_uucp");
		if (hlen>0) imsg("End of bag file while expecting %d more bytes\n",hlen);
		if (article!=NULL) {
			if (isplain) if (!item_suck(article)) {
			if (item_was_mod()) {
			 if (config_true(C_UUCP_SEND_MOD)) {
				if (suck_post_moderated(article)) goto wasok2;
				else imsg("Moderated post from bag file failed\n");
			 } else {
				imsg("Bag file contained un-approved moderator post, set uucp_send_mod true to send these to moderator instead of rejecting\n");
			 }
			}


			if (item_iscrosspost()) sk_ndup++; else sk_reject++;
		} else {
wasok2:			sk_nok++;
			sk_size += str_len(article);
		}
		str_clearbig(article);
		}
		uuin = NULL;
		if (dorename) goto fail;
		if (remove(uucp_file)!=0) filemsg("Unable to delete {%s} \n",uucp_file);
		stats_feed("suck_uucp",lib_gmtime()-sk_start,sk_reject,sk_nok,sk_ndup,sk_size,0.0,0.0);
		sk_nok = sk_reject = 0;
		sk_size = 0; sk_ndup = 0;
		sk_start = lib_gmtime();
		goto again;
	}
	return;
fail:
	if (uuin!=NULL) {
		fclosep(uuin);
	}
	ncpy(bf,uucp_file,BFSZ-1);
	for (s=bf;*s!=0;s++) {
		if (*s==']') last = s;
		if (*s=='/') last = s;
		if (*s=='\\') last = s;
	}
	if (last!=NULL) strcpy(last+1,"uucp.bug");
	remove(bf);
	rename(uucp_file,bf);
	emsg("suck_uucp: Failed so renaming to %s, nok = %d, nfail=%d\n",bf,sk_nok,sk_reject);
	uuin = NULL;

	stats_feed("suck_uucp",lib_gmtime()-sk_start,sk_reject,sk_nok,sk_ndup,sk_size,0.0,0.0);
	sk_nok = sk_reject = 0;
	sk_ndup = 0;
	sk_size = 0;
	sk_start = lib_gmtime();
	
	return;
}
char uucp_spool[BFSZ];
char uucp_dir[BFSZ];
char uucp_wild[BFSZ];

void uucp_open(void)
{
	static Sary *ls;
	static int upto;
	char *s;
 	static int lastt;

	
	if (ls==NULL) {ls = s_alloc(); upto = 0;}
	if (upto<s_len(ls)) goto next_one;
	if (lastt+60 >lib_gmtime()) return; /* Only check every 60 seconds */

	lastt = lib_gmtime();
	ncpy(uucp_spool,config_get(C_SUCK_UUCP),BFSZ-1);
	if (strlen(uucp_spool)==0) return;
	s = strchr(uucp_spool,'*');
#ifdef Mac
	 if (s==NULL) strcat(uucp_spool,":*.bag");
#else
	 if (s==NULL) strcat(uucp_spool,"/*.bag");
#endif
 	lib_dir_split(uucp_spool,uucp_dir,uucp_wild);
	dmsg("suck: Scanning (%s) (%s) \n",uucp_dir, uucp_wild);
				
	s_wild(ls,uucp_dir,uucp_wild);
	imsg("suck: suck_uucp: found %d files (%s) (%s) \n",s_len(ls),uucp_dir,uucp_wild);
	if (s_len(ls)==0) {dmsg("suck: uucp no files found (%s) (%s)\n",uucp_dir,uucp_wild); return;}
again:;
	if (upto>=s_len(ls)) {
		s_free(ls);
		ls = NULL;
		upto = 0;
		return;
	}
next_one:
	s = s_line(ls,upto++); 
	if (strstr(s,"uucp.bug")!=NULL) goto again;
#ifdef VMS
        lprintf(uucp_file,BFSZ-1,"%s%s",uucp_dir,s);
#else
#ifdef Mac
        lprintf(uucp_file,BFSZ-1,"%s:%s",uucp_dir,s);
#else
        lprintf(uucp_file,BFSZ-1,"%s/%s",uucp_dir,s);
#endif
#endif
	imsg("suck: uucp processing %s\n",uucp_file);
	if (config_true(C_UUCP_BINARY)) {
		uuin = myfopen(uucp_file,"rb");
	} else {
		uuin = myfopen(uucp_file,"r");
	}
	if (uuin==NULL) filemsg("Could not read {%s} \n",uucp_file);
}

static int	sp_nok,sp_reject;
static int	sp_size ;
static int	sp_start;
static int dnews_format = FALSE;
static int isimport;
void convert_dnewsfmt(void)
{
	dnews_format = TRUE;
}
static int msfmt;
void convert_msfmt(void)
{
	msfmt = TRUE;
}
void suck_convert_file(char *fname)
{
	FILE *f;
	char bf[BIGBF];
	char *s;
	char *last;
	int g;
	int itemi;
	static String *article;
	int tryagain = FALSE;
		
	if (dnews_format) {db_convert_file(fname); return;}

	if (article==NULL) article = str_new();
	str_clearbig(article);
	f = fopen(fname,"r");
	if (f==NULL) {filemsg("Cannot read %s",fname); return;}
	for (;!feof(f);) {
		if (fgets(bf,BIGBF-10,f)==NULL) break;
		s = strchr(bf,'\015'); if (s!=NULL) *s = 0;
		s = strchr(bf,'\012'); if (s!=NULL) *s = 0;
		if (msfmt) if (strcmp(bf,".")==0) break;
		if (bf[0]=='.') {
			memmove(bf+1,bf,strlen(bf)+1);
			bf[0] = '.';
		}
		strcat(bf,"\015\012");
		str_addsn(article,bf);
		if (str_len(article)>conf_item_max()) {
		 	emsg("Article too big to convert {%s}\n",fname);
			sp_reject++;
			fclosep(f);
			goto finish;
		}
	}
	fclosep(f);
	if (msfmt) goto skiprep;
	if (!config_true(C_CONVERT_ADD)) {
		item_set_xreplic(TRUE);
		last = fname;
		for (s=fname;*s!=0;s++) {
			if (*s=='/') last = s;
			if (*s=='\\') last = s;
			if (*s==']') last = s;		
		}
		if (!isdigit(*(last+1))) emsg("Import, can't find item number for {%s} \n",fname);
		itemi = atoi(last+1);
		g = group_find_file(fname);
		group_xref_clear();
		if (g>=0) {
			imsg("Importing (%s) -> %s:%d\n",fname,group_name(g),itemi);
			group_xref_here(g,itemi);
		} else imsg("Couldn't figure out group name (%s), ok if xref is in file, rename to ...news...\n",fname);
		
	}
skiprep:;
	if (!config_true(C_CONVERT_FEED)) {
		item_stopfeeds(TRUE);
	}
	item_setimport(TRUE);
	if (!item_suck(article)) {
		item_setimport(FALSE);
		sp_reject++;
		if (item_tryagain()) tryagain = TRUE;
		if (item_iscrosspost()) {
			dmsg("suck: Ignoring cross post\n");
			if (!isimport) remove(fname);
		} else imsg("Convert failed {%s} {%s}\n",fname,msg_reason());
	} else {
		item_setimport(FALSE);
		sp_nok++;
		if (!isimport) remove(fname);
		sp_size += str_len(article);
	}
	item_stopfeeds(FALSE);
	item_set_xreplic(FALSE);

finish:;
	str_clearbig(article);
}
char *spl_next(void);
void suck_convert(void)
{
	int time_start=lib_gmtime();
	char *i_file=spl_next();
	if (sp_start==0) sp_start = time_start;	
	if (i_file==NULL) {
		dnews_format = FALSE;
		return;
	}

	for (;(i_file!=NULL) && (lib_gmtime()==time_start); ) {
		suck_convert_file(i_file);
		i_file = spl_next();
	}		 	
	if (i_file!=NULL) suck_convert_file(i_file);

	stats_feed("spool_convert",lib_gmtime()-sp_start,sp_reject,sp_nok,0,sp_size,0.0,0.0);
	sp_nok = sp_reject = 0;
	sp_size = 0;
	sp_start = lib_gmtime();
}
static Sary *xstack;
static int nstack=0;
char *spl_next(void)
{
	int i;
	Sary *xdir;
	static char bf[BFSZ],bf2[BFSZ];
	char f_dir[BFSZ];
	char f_name[BFSZ];
	
	if (xstack==NULL) return NULL;
next_one:
	if (nstack==0) return NULL; 
	ncpy(bf,s_line(xstack,--nstack),BFSZ-1);
 	lib_dir_split(bf,f_dir,f_name);
	if (strstr(f_name,".nws")!=NULL) return bf;
	if (strstr(f_name,".NWS")!=NULL) return bf;
	if (strcmp(f_name,".")==0) goto next_one;
	if (strcmp(f_name,"..")==0) goto next_one;
  	if (!dnews_format) if (strchr(f_name,'.')!=NULL) goto next_one;
	if (strlen(f_name)==0) goto next_one;
	if (strncmp(f_name,"db_",3)==0) goto next_one;
	if (strncmp(f_name,"DB_",3)==0) goto next_one;
	if (!isdigit(*f_name)) goto add_dir;
	return bf;
add_dir:
	imsg("convert: Scanning %s\n",bf);
	xdir = s_alloc();
	if (msfmt) 	s_wild(xdir,bf,"*.*");
	else		s_wild(xdir,bf,"*");
	for (i=0; i<s_len(xdir); i++) {
		strcpy(bf2,bf);
		strcat(bf2,"/");
		strcat(bf2,s_line(xdir,i));
	 	s_set(xstack,nstack++,bf2);
	}
	s_free(xdir);
	goto next_one;
}
void suck_set_import(int t)
{
	isimport = t;
}
void suck_start_convert(char *path)
{
	imsg("Starting conversion of {%s} tree\n",path);
	if (xstack==NULL) xstack = s_alloc();
	s_set(xstack,nstack++,path);
}
#define SECOND_AT (10*3600)
static int isinuse;
int second_check(char *ex);
void suck_isused(void)
{
	isinuse = TRUE;
}
void suck_second(void)
{				      
#ifdef WIN32xxx
	static int att;
	static int wasok = FALSE;
	char ex[BFSZ];
	static int done;
	if (!done) { srand(lib_gmtime()); att = rand() % (SECOND_AT);}
	done = TRUE;

 	if ((lib_gmtime() % (SECOND_AT)) != att) return;
	if (!second_check(ex)) {
		if (isinuse) {
			emsg("Second info setting set\n");
			/*if (wasok) ncmd_tell(9);
			else ncmd_tell(2);*/
		 	/* nntp_xxsetup(4);*/ /* Limit to 4 users */
		}
	}
	wasok = TRUE;
#endif
}			  
#define DNEWS_KEYFILE file_kfile()
int second_check(char *ex)
{
#ifndef KEYLIB
        FILE *f;
	static char cur_key[BFSZ];
        int users,flags;
	static int wasok = FALSE;
        
        if (strlen(cur_key)==0) {
                f = fopen(DNEWS_KEYFILE,"r"); 
                if (f==NULL) {
	                f = fopen(DNEWS_KEYFILE,"w"); 
                	if (f!=NULL) {
				fclosep(f);
				remove(DNEWS_KEYFILE);
				return FALSE;
			} else {
				filemsg("Unable to create file\n");
				wasok = TRUE;
				return TRUE;
			}
                }
                fgets(cur_key,BFSZ-1,f);
                fclosep(f);
	}
        if (!db_rcheck(cur_key,host_details(),&users,&flags,ex)) {
		emsg("License key is not valid {%s}\n",ex);
		return FALSE;
        }       
	wasok = TRUE;
#endif
	return TRUE;
}


#define MAXQ 100
static char *cmdq_str[MAXQ];
static int nq;
int cmdq_add(char *s)
{
	if (nq>=MAXQ) {emsg("Cannot add to cmd que %s\n",s); return nq;}
	cmdq_str[nq++] = mystrdup(s,SD_SUCK);
	return nq;
}
int suck_update_groups_nq(int f)
{
	char bf[BFSZ];
	lprintf(bf,BFSZ-1,"getgroups %d",f);
	cmdq_add(bf);
	return TRUE;
}
int cmdq_do(void)
{
    char cmd[BFSZ],p1[BFSZ];
    char *in;
    char p2[BFSZ];
    
    if (!suck_spawned_expire()) {
    	in = cmdq_pop();
	ncpy(match_site,"",BFSZ-1);
	ncpy(match_groups,"",BFSZ-1);
	if (in==NULL) return FALSE;
	ncpy(cmd,get_word(in,0),99);
	ncpy(p1,get_word(in,1),99);
	ncpy(p2,get_word(in,2),99);
        strlwr(cmd);
	imsgf("cmdq: Processing next background command (%s)\n",in); 
	if (strcmp(cmd,"trimgroups")==0) {
		suck_trim_groups_n(atoi(p1));
	} else if (strcmp(cmd,"matchsite")==0) {
		suck_matchsite(p1,p2);
	} else if (strcmp(cmd,"getactive")==0) {
                suck_getactive(p1);
	} else if (strcmp(cmd,"getgroups")==0) {
                suck_update_groups_n(atoi(p1));
	} else if (strcmp(cmd,"update")==0) {
		if (atoi(p1)==0) suck_update_cached();
		else suck_update_n(atoi(p1));
	} else if (strcmp(cmd,"rebuild_index")==0) {
		cmd_pause(TRUE,"Rebuilding index");
		db_set_rebuild(TRUE); /* Tell it to delete the current index */
                suck_addlost();
	} else if (strcmp(cmd,"resort_index")==0) {
		suck_resort();
	} else if (strcmp(cmd,"expire")==0) {
		suck_expire();
	} else if (strcmp(cmd,"expire_groups")==0) {
		suck_expire_groups(p1);
	} else emsg("Unknown que'd command (%s) \n",cmd);
	return TRUE;
    }
    return FALSE;
}
int cmdq_len(void)
{
	return nq;
}
char * cmdq_name(int i)
{
	if (cmdq_str[i]!=NULL) return cmdq_str[i];
	return "thats most strange";
}
char *cmdq_pop(void)
{
	int i;
	static char bf[BFSZ];
	if (nq<=0) return NULL;
	strcpy(bf,cmdq_str[0]);
	mystrfree(cmdq_str[0],SD_SUCK);
	nq--;
	for (i=0; i<nq; i++) {
		cmdq_str[i] = cmdq_str[i+1];
	}
	return bf;
}
	   
#define FILE_GUP file_work("gupsend.tmp")
void gup_send(void)
{
	char to[BFSZ];
	FILE *f;
	int i;
	ncpy(to,config_get(C_GUP_TO),BFSZ-1);
	f = fopen(FILE_GUP,"w");
	if (f==NULL) {filemsg("Unable to write to %s\n",FILE_GUP); return;};
	fprintf(f,"site %s\n",config_get(C_GUP_SITE));
	fprintf(f,"delete *\n");
	for (i=0; i<group_n(); i++) {
		if (group_isdeleted(i)) continue;
		if (group_iscached(i)) {
			if (strlen(config_get(C_GUP_MASK))==0) goto isok;
			if (group_match(config_get(C_GUP_MASK),group_name(i))) {
isok:				fprintf(f,"include %s\n",group_name(i));
			}
		}
	}
	if (strlen(config_get(C_GUP_ADD))!=0) fprintf(f,"include %s\n",config_get(C_GUP_ADD));
	fprintf(f,"quit\n\n");
	fclosep(f);
	imsg("Generated gup file %s, sending to %s\n",FILE_GUP,to);

	lib_mail_msg(to,"Group Update (gup) generated by dnews",FILE_GUP);
}
int suck_post_moderated(String *anitem)
{
	int r;

	feed_setpost(FALSE);
#ifdef SPAWN_FILTER
	if (strlen(config_get(C_FILTER_CMD))>0) {
		r = filter_isok(str_s(anitem),str_len(anitem));
		if (r>=400) return FALSE;
	}
#endif
        if (post_moderated(anitem)) {
                imsg("UUCP: Post to moderator succeeded\n");
                feed_doit_if();
		return TRUE;
	}
        imsg("UUCP: Post to moderator failed\n");
	return FALSE;
}

int is_spawn_expire(void)
{
	if (!config_true(C_SPAWN_EXPIRE)) return FALSE;
	if (!db_ispile()) {
		emsg("FATAL spawn_expire true, but piles not being used, see http://netwinsite.com/dnews/piles.htm\n");
		return FALSE;
	}
	return TRUE;
}
void suck_write_s(int t,char *bf)
{
	dmsg("suck: -->  %s\n",nocrlf(bf));	
	chan_write_s(t,bf);
}
