/* Copyright (C) 1995 NetWin World, New Zealand, All rights reserved. */
#ifdef WIN32
#include <windows.h>
#include <winbase.h>
#endif
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <errno.h>
#ifdef isunix
#include <unistd.h>
#include <sys/wait.h>
#endif
#include "emsg.h"
#include "str.h"
#include "group.h"
#include "lib.h"
#include "item.h"
#include "dnews.h"
#include "chan.h"
#include "hist.h"
#include "expire.h"
#include "nntp.h"
#include "suck.h"
#include "post.h"
#include "conf.h"
#include "feed.h"
#include "dfilter.h"
#include "lib_spawn.h"
#include "mutex.h"
#include "thout.h"

void feed_init(int ff);
void live_close(int chan);
#undef mymalloc
#undef myfree
#define mymalloc(x) mymalloc2(x,M_FEED)
#define myfree(x) myfree2(x,M_FEED)

/*
site ajax
        exclude bad1,bad2 
        groups *,!*.sex.*  
        distrib !comp,rec,nz,world
        pending_max 10000000  delete .feed and .sending file if bigger than this
        maxsize 1000000
        maxcross 20
        ignorepath
        builtin         
        posting
        onlyposts
        path_max 3
	rules d:\dnews\me.rul

        groups /file/name
        bind 1.2.3.4
	delay 60   (Delay 60 seconds at least)
        

IF (Delay 60)
	Don't send to live channel as we get it
	Don't rename .feed file unless it's at least delay seconds old
	Rename it to .wait and don't rename to .send until delay seconds
	after that. 
		

Haven't implemented maxcross limit !!!
*/
#define MAX_REJECT 200
#define MAX_LIVECHAN 10
typedef struct {
	Thout *th;
        char *site, *exclude, *groups, *distrib; 
	char *to;
        char **reject;
        char **rejhead;
        int *isreject;
        int *isnot;
        int nreject;
        int max,  cross, nopath, builtin, posting,nsent,smtp,live_only,onlyposts,path_max;
        int last_time, nok, ndup, nlost, nrejects, nreque, nspool, start, chomps;
        int uucp,uucp_size,uucp_n;
        char *uucp_dir;
        char *uucp_name; /* Anything with a %d in it, e.g. item_%d.bag */
	int uucp_timer;
        int live;
	int xdelay;
        int live_chan[MAX_LIVECHAN];
        int live_nchan;
        int live_nocheck;
        int nrecent;
        int nretry,sleeping;
        int isstream;
        int isprog;
	int isdfeed;
	int issendthread;
        int isxreplic;
        int prog_single; 
        char *prog_name;
        char *prog_args;
        int islog;
        int pending_max;
        int deleted;
        int gtime;
        char *gfile;
        char *bind;
	char *rules;
	char *gmask;
	char *gblock;
	FProg *rules_code;
} Feed;
int feed_done(void) ;
void prog_close(int f);

#ifdef NETWARE
#define SHORT_NAMES TRUE
#endif

#ifdef SHORT_NAMES
#define DNEWS_NEWSFEEDS "feeds.con"
#else
#define DNEWS_NEWSFEEDS "newsfeeds.conf"
#endif

#define MAX_LINE 10000
#ifdef NETWARE
#define MAX_FEEDS 20
#else
#define MAX_FEEDS 200
#endif
int tout_free_pot(void);
static Feed feeds[MAX_FEEDS];
static int nfeeds;
static Feed cur;
void spam_store(String *xitem);
int spam_subject(String *xitem,int ng,char *reason);
void spam_rej_free(void);
void spam_rej_load(char *fname);
void feed_queue(int i);
int uucp_buff_flush(int f);
int uucp_buff_add(int f, char *bf, int len);
#define UBFSZ 200000
static char *ubuf[MAX_FEEDS]; /* 300k buffer for each uucp feed */
static int ubuf_len[MAX_FEEDS]; /* 300k buffer for each uucp feed */
static int send_total,send_missed;
void report_spammer(String *anitem, char *reason);
static int prog_pid;

/* brian - new scoring filter - start */
#define score_testing FALSE
#define FILE_SCORETEST file_work("test.txt")
#define FILE_SCOREWORDS file_work("filter.scr")
#define SCORE_TESTMAX 20000


#define score_hash(x) ((x[0] & 0x5f) + ((x[1] & 0x5f)<<3) + ((x[2] & 0x5f)<<5)) 
#define SCORE_HASH_REJ 15000
typedef struct SCOREREJ {
        char *s;
        short len;
        short hits;
	int score;
        struct SCOREREJ *next;
} Scorerej;
Scorerej *scorehashrej[SCORE_HASH_REJ];

int score_nfile;
/* brian - new scoring filter - end */

#define spam_rej_hash(x) ((x[0] & 0x5f) + ((x[1] & 0x5f)<<3) + ((x[2] & 0x5f)<<5)) 

void feed_clear(void);
void setfree(char **s);
void spam_rej_add(char *s);
void feed_add(char *rec);
void feed_defaults(void);
int spam_test(String *xitem,int ng,char *reason, char *msgid);
int feed_pool_flush(void);
static int npost;
static int post_feed;

void cmd_complete(int chan, int p1, int p2, int from, int to, int state);
int tout_shutdown(int ff);

char *live_ring_msgid(int chan, int r);
int live_ring_find(int chan, char *intsr, int len);
int live_send_spooled(int ff);
char *feed_to(Feed *f);
void live_mode_status(int chan,char *bf,int sz);
void live_ihave_status(int chan,char *bf,int sz);
void live_reque(int chan, int r);
void live_reject(int chan, int r);
void live_dup(int chan, int r);
char *linknamec(int chan);
void live_ok(int chan, int r);
void live_send(int chan, char *msgid, char *groupnum, String *xitem, int nretry);
void live_takethis_status(int chan, char *bf, int sz);
void live_start_send(int chan, int r);
void program_feed(int f,char *msgid,char *groupnum,String *xitem);
void feed_mask_init(int i);

int live_find_fchan(int ff, int chan);
static int feed_one_disable;

void feed_flushuucp(void);
void feed_flushuucp2(void);
void uucp_next(int f);
int uucp_nexti(int ff);
int uucp_len(String *str); 
void feed_uucp_write(int f, String *xitem);
void feed_pool_checkpoint(int f) ;
void feed_pool_fill(int f) ;
int feed_pool_next(int f) ;
void feed_pool_reque(int f,int r);
void feed_pool_ok(int f) ;
void live_trylater(int chan);

void live_feed(int ff,char *msgid,char *groupnum,String *xitem);
void live_idle(void) ;
void live_read(int chan, char *bf, int sz);
void live_send(int chan, char *msgid, char *groupnum, String *xitem, int nretry);
void live_takethis_status(int chan, char *bf, int sz);
int feed_filesz(int x);

int live_ring_next(int chan);
void ring_free(int chan, int r);
#ifdef OS2
#define NOPOPEN
#endif
#ifdef NOPOPEN
FILE *popen(char *prog,char *mode);
FILE *popen(char *prog,char *mode)
{
  emsg("Sorry popen is not implemented on OS2\n");
  return NULL;
}
void pclose(FILE *x);
void pclose(FILE *x)
{
 
}
#endif


/*      Responding-System: cbosgd.ATT.COM
      cbosgd:osg,cb,btl,bell,world,comp,sci,rec,talk,misc,news,soc,to,
            test
      ucbvax:world,comp,to.ucbvax:L:
      cbosg:world,comp,bell,btl,cb,osg,to.cbosg:F:/usr/spool/outnews
            /cbosg
      cbosgb:osg,to.cbosgb:F:/usr/spool/outnews/cbosgb
      sescent:world,comp,bell,btl,cb,to.sescent:F:/usr/spool/outnews
            /sescent
      npois:world,comp,bell,btl,ug,to.npois:F:/usr/spool/outnews/npois
      mhuxi:world,comp,bell,btl,ug,to.mhuxi:F:/usr/spool/outnews/mhuxi
*/
char *trim_wild(char *s)
{
	static char bf[BFSZ];
	char *out=bf;

	if (strcmp(s,"*")==0) return s;
	for (;(*s!=0) && (strlen(out)<300);s++) {
		if (strncmp(s,".*",2)==0) {s++; continue; }
		*out++ = *s;
	}
	*out++ = 0;
	return bf;

}
void feeds_write_sys(char *fname)
{
	FILE *f;
	int i;
	f = fopen(fname,"w");
	if (f==NULL) return;

	fprintf(f,"Responding-System: %s\n",config_get(C_MYNAME));
	fprintf(f,"%s:%s:F:None\n",config_get(C_MYNAME),trim_wild(feeds[0].groups));
        for (i=1; i<nfeeds; i++) {/*loop*/
		if (feeds[i].builtin || feeds[i].live  ) {
			fprintf(f,"%s:%s:F:None\n",feeds[i].site,trim_wild(feeds[i].groups));
		}
	}	
	fclosep(f);
}



void feed_read_gfile(int i);
void dfeed_idle(void);
void feed_idle(void)
{
        int m;
        int i;
        /* check for changes in gfile feed files */
        for (i=0; i<nfeeds; i++) {

          if (feeds[i].gfile!=NULL) {
                m = lib_file_mod(feeds[i].gfile);
                if (m!=feeds[i].gtime) {
                        feeds[i].gtime = m;
                        feed_read_gfile(i);
                }
          }
        }       
	dfeed_idle();
}
static int dfeed_pid;
void dfeed_idle(void)
{
	static int last;
	int i;
	char bf[BFSZ];
	int any=FALSE;
	static int firsttime = TRUE;
	static int done_prog;

	if (is_readonly()) return;
	if (lib_gmtime()-last<(60)) return;
        for (i=0; i<nfeeds; i++) {
		if (!feeds[i].isdfeed) continue;
		any = TRUE;
		feed_flushsite(feeds[i].site);
	}
	if (any) if (!spawn_isalive(dfeed_pid)) {
		strcpy(bf,file_log("dfeed.log"));
		if (firsttime) remove(file_log("dfeed.ded"));
		if (dfeed_pid!=0) if (firsttime) {
			emsg("Renaming dfeed.log to dfeed.ded, remove after examining the failure\n");
		}
		firsttime = FALSE;
		rename(bf,file_log("dfeed.ded"));
		dfeed_pid = lib_spawn_cmd(file_exe("dfeed"));
		imsg("Starting dfeed process, pid=%d\n",dfeed_pid);
	}				
	if (strlen(config_get(C_MULTICAST_LIST))>0) {
		int pid;
		char list[BFSZ];
		char cmd[BFSZ];
		char *s;
		if (lib_exists(file_log("dudp.status"))) {
			if (time(NULL) - lib_file_mod(file_log("dudp.status"))<120) {
				/*imsg("dudp: Seems to be running ok\n");*/
				return;
			} else imsg("dudp: Better restart it as the status file is old\n");
		}
		strcpy(list,config_get(C_MULTICAST_LIST));
		lprintf(cmd,BFSZ-1,"%s -p 1119 -t 127.0.0.1 -b -m ",file_exe("dudp"));
		if (strlen(config_get(C_DUDP_CMD))>0) lprintf(cmd,BFSZ-1,"%s -b -m ",config_get(C_DUDP_CMD));
		for (s=list;*s!=0;s++) {	
			if (*list==' ') *list = ',';
		}
		strcat(cmd,list);
		pid = lib_spawn_cmd(cmd);
#ifndef WIN32
		spawn_release(pid);		
#endif
	}
	if (prog_pid!=0) {
		if (!spawn_isalive(prog_pid)) {
			imsg("Oops %s seems to have died, restarting\n",config_get(C_SPAWN_PROG));
			spawn_release(prog_pid); /* this isn't really needed but probably won't jurt*/ 
			goto restart_prog;
		}
	}

	if (!done_prog) if (strlen(config_get(C_SPAWN_PROG))>0) {
		char cmd[BFSZ];

restart_prog:	done_prog = TRUE;
		lprintf(cmd,BFSZ-1,"%s",config_get(C_SPAWN_PROG));
		imsg("Starting spawn_prog (%s)\n",cmd);
		prog_pid = lib_spawn_cmd(cmd);
#ifndef WIN32
		spawn_release(prog_pid);		
#endif
	}
}

void feed_read_gfile(int i)
{
        FILE *f;
        char bf[BFSZ];
        int len;
        char *s;
        char *list;
        len = (int) (lib_length_file(feeds[i].gfile)*1.5+2000);
        imsgf("Reloading modified group file %s %d bytes\n",feeds[i].gfile,lib_length_file(feeds[i].gfile));
        list = mymalloc(len);
        f = myfopen(feeds[i].gfile,"r");
        if (f==NULL) {
                filemsg("Could not read (%s) \n",feeds[i].gfile);
                return;
        }
        strcpy(list,"");
        for (;!feof(f);) {
                if (fgets(bf,BFSZ-10,f)==NULL) break;
                s = strchr(bf,'\n'); if (s!=NULL) *s = 0;
                if (strlen(list)!=0) strcat(list,",");
		if ((int) (strlen(list) + strlen(bf))>=len-100) {
			emsg("Ooops, the files still group for feed %d, try again\n",i);
			feeds[i].gtime = 0;
			break;
		}
		strcat(list,bf);
        }
        fclosep(f);
        if (feeds[i].groups!=NULL) myfree(feeds[i].groups);
        feeds[i].groups = list;
	feed_mask_init(i);
        imsgf("Reloading complete\n");
}
int feed_thisposting(void)
{
        if (post_feed>=0) return feeds[post_feed].posting;
        return FALSE;
}
static Feed feeds_copy[MAX_FEEDS];
int feed_statssave(void)
{
        int i;
        for (i=1; i<nfeeds; i++) {
                feeds_copy[i] = feeds[i];
        }
        return TRUE;
}

int feed_statsset(void)
{
        int i;
        for (i=1; i<nfeeds; i++) {
                feeds[i].nok = feeds_copy[i].nok;
                feeds[i].nrejects = feeds_copy[i].nrejects;
                feeds[i].ndup = feeds_copy[i].ndup;
                feeds[i].nlost = feeds_copy[i].nlost;
                feeds[i].nreque = feeds_copy[i].nreque;
        }
        return TRUE;
}
int feed_statsclear(void)
{
        int i;
        for (i=1; i<nfeeds; i++) {
                feeds[i].chomps = 0;
                feeds[i].nok = 0;
                feeds[i].nrejects = 0;
                feeds[i].ndup = 0;
                feeds[i].nlost = 0;
                feeds[i].nreque = 0;
        }
        return TRUE;
}
void feed_reload(void)
{
        imsg("Reloading newsfeeds.conf file\n");
        feed_statssave();
        feed_flush2(TRUE);
        feed_flushuucp();
        feed_clear();
        feed_load();
        feed_statsset();
}
void feed_mask_reload(void)
{
	int i;
	imsg("Feed Mask Reload ....\n");
	for (i=0; i<nfeeds; i++) feed_mask_init(i);
	imsg("Feed Mask Reload Done\n");
}
void bit_set(char *mask, int x)
{
	unsigned char *m = (unsigned char *) mask;
	int r = x % 8;
	x = x / 8;
	mask[x] |= (1 << r);
}
void bit_set2(char *mask, int x, int v)
{
	unsigned char *m = (unsigned char *) mask;
	int r = x % 8;
	x = x / 8;
	if (v) mask[x] |= (1 << r);
	else mask[x] &= (0xff - (1 << r));
}
int bit_isset(char *mask, int x)
{
	unsigned char *m = (unsigned char *) mask;
	int r = x % 8;
	x = x / 8;
	return ((mask[x] & (1 << r))!=0);
}
void feed_addgroup(int g)
{
	int i;
	Feed *f;
	if (config_true(C_FEED_NOMASK)) return;

	for (i = 0; i<nfeeds; i++) {
		/*feed_mask_init(i);*/
		f = &feeds[i];
		if (f->groups==NULL) continue; 
		if (f->gmask==NULL) continue; /* only happens when groups not defined for me feed */ 
		if (group_match(f->groups,group_namelc(g))) {
			bit_set(f->gmask,g);
		} else {
			if (group_match_none()) {
				bit_set(f->gblock,g);
			}
		}
	}
}
void fxxeed_mask_init(int i)
{
	Feed *f=&feeds[i];
	char *gm,*gb;
	/* Init group mask for each feed */
	/* gmask, gblock */
	if (config_true(C_FEED_NOMASK)) return;
	if (f->groups==NULL) return;
	if (f->gmask!=NULL) myfree(f->gmask);
	if (f->gblock!=NULL) myfree(f->gblock);

	f->gmask = gm = mymalloc((max_groups+10)/8);
	memset(gm,0,(max_groups+10)/8);
	f->gblock = gb = mymalloc((max_groups+10)/8);
	memset(gb,0,(max_groups+10)/8);
	
	for (i=0; i<group_n(); i++) {
if ((i%3000)==3) imsgf("upto %d %d\n",i,strlen(f->groups));
                if (group_match(f->groups,group_namelc(i))) {
			bit_set(f->gmask,i);
		} else {
			if (group_match_none()) {
				bit_set(f->gblock,i);
			}
		}
	}
}


void feed_mask_init(int i)
{
	Feed *f=&feeds[i];
	char *gm,*gb;
	char *mylist,*mybf;
	char *s;


	/* Init group mask for each feed */
	/* gmask, gblock */
	if (config_true(C_FEED_NOMASK)) return;

	imsg("feed_mask_init(%d) called\n",i);
	if (f->groups==NULL) return;
	if (f->gmask!=NULL) myfree(f->gmask);
	if (f->gblock!=NULL) myfree(f->gblock);
	imsg("feed_mask_init(%d) called groups %d\n",i,group_n());

	f->gmask = gm = mymalloc((max_groups+10)/8);
	memset(gm,0,(max_groups+10)/8);
	f->gblock = gb = mymalloc((max_groups+10)/8);
	memset(gb,0,(max_groups+10)/8);
	

	/* Pull specific groups out of list, if not blocked add them */
	mybf = mymalloc(strlen(f->groups)+100);
	strcpy(mybf,f->groups);
	mylist = mymalloc(strlen(f->groups)+100);
	strcpy(mylist,"");

	s = strtok(mybf,",");
	for (;s!=NULL;) {
		if ((strchr(s,'*')==NULL) && (strchr(s,'!')==NULL) && (strchr(s,'@')==NULL)) {
			i = group_find(s);
			if (i>=0) {
				bit_set(f->gmask,i);		
			}
		} else {
			strcat(mylist,s);
			strcat(mylist,",");
		}
		s = strtok(NULL,",");
	}


	if (strlen(mylist)>0) for (i=0; i<group_n(); i++) {
                if (group_match(mylist,group_namelc(i))) {
			bit_set(f->gmask,i);
		} else {
			if (!bit_isset(f->gmask,i)) 
			  if (group_match_none()) {
				bit_set(f->gblock,i);
			}
		}
	}
	myfree(mylist);
	myfree(mybf);
}

/*
	if a token in 'groups' has no wild card, then remove
	it and add that group if it exists, then 
	process the rest.

*/

int feed_mask_test(int fi, int g, int *block)
{
	Feed *f=&feeds[fi];
	*block = FALSE;
	if (g<0 || g>=max_groups) return FALSE;
	if (bit_isset(f->gmask,g)) {
		return TRUE;
	}
	else if (bit_isset(f->gblock,g)) {
		*block = TRUE;
	}
	return FALSE;
}
static int isfeeder;
int feed_isfeeder(void)
{
	return isfeeder;
}
int feed_load(void)
{
        int i;
        FILE *f;
        char bf[10001];
        char *s;

        static int done;

/* Should shut down live channels and close files here*/
	score_free();
	score_load(FILE_SCOREWORDS);
        spam_rej_free();
        spam_rej_load(file_work("filter.dat"));
        spam_rej_load(file_work("filter.net"));
        npost = 0;
        feed_clear(); /* free memory used by structures */
        feed_defaults();
        f = xfopen(file_config(DNEWS_NEWSFEEDS),"r");
        if (f==NULL) {filemsg("feed: Can't open %s \n",file_config(DNEWS_NEWSFEEDS)); return FALSE;}
        for (;!feof(f);) {
                if (fgets(bf,10000,f)==NULL) break;
                s = strchr(bf,'\n');
                if (s!=NULL) *s = 0;
                feed_add(bf);
        }
        feed_new();
        if (nfeeds<=0) {
                emsg("%s file is empty, must contain at least\n",file_config(DNEWS_NEWSFEEDS));
                emsg("  site me\n");
                emsg("         groups *\n");
                dnews_ex("newsfeeds.conf file is empty");
        }
        if (is_readonly()) {
                if (nfeeds>1) nfeeds = 1;  
        }
	isfeeder = FALSE;
        for (i=0; i<nfeeds; i++) {/*loop*/
                if (feeds[i].site!=NULL) dmsg("feed: Site {%s} \n",feeds[i].site);
                if (feeds[i].exclude!=NULL) dmsg("feed: exclude {%s} \n",feeds[i].exclude);
                if (feeds[i].distrib!=NULL) dmsg("feed: distrib {%s} \n",feeds[i].distrib);
                if (feeds[i].groups!=NULL) dmsg("feed: groups {%s} \n",feeds[i].groups);
                dmsg("feed: maxsize %d maxcross %d ignorepath %d \n",
                        feeds[i].max,feeds[i].cross,feeds[i].nopath);
                feed_init(i);

		if (feeds[i].live || feeds[i].isdfeed) {
			isfeeder = TRUE;
		}

        }
        xfclose(f);
     
	for (i=0; i<nfeeds; i++) {
		if (feeds[i].rules!=NULL) {
			feeds[i].rules_code = dfilter_compile(feeds[i].rules);
		}
	}
	feed_mask_reload();
	
	return TRUE;
}
static int idle_lastt;

void feed_defaults(void)
{
        int k;
        idle_lastt = 0;
        cur.islog = FALSE;
        cur.max = 5000000;
        cur.cross = 100;
        cur.nopath = FALSE;
        cur.live_only = FALSE;
        cur.onlyposts = FALSE;
        cur.isxreplic = FALSE;
	cur.th = NULL;
        cur.live_nocheck = FALSE;
        cur.builtin = FALSE;
        cur.posting = FALSE;
        cur.smtp = FALSE;
        cur.site = cur.exclude = cur.groups = cur.distrib = NULL;
	cur.to = NULL;
        cur.live = FALSE;
        cur.live_nchan = 0;
        cur.nretry = 0;
        cur.sleeping = 0;
        cur.path_max = 0;
        cur.nrecent = 100;

        cur.isstream = FALSE;
        cur.uucp = FALSE;
        cur.uucp_size = 4000000;
        cur.pending_max = 10000000;
        cur.uucp_n = 500;
        cur.uucp_timer = 800;
        cur.uucp_dir = cur.uucp_name = NULL;
        cur.gmask = cur.gblock = cur.bind = cur.gfile = cur.rules = NULL;
	cur.rules_code = NULL;
        cur.gtime = 0;
        cur.isprog = FALSE;
        cur.xdelay = 0;
        cur.prog_single = FALSE;
        cur.prog_name = NULL;
        cur.prog_args = NULL;
        cur.deleted = 0;
	cur.issendthread = FALSE;
        cur.nsent = cur.nok = cur.ndup = cur.nlost = cur.nspool = 0;
        cur.nrejects = cur.nreque = 0;
        cur.start = lib_gmtime()-1;
        cur.chomps = 0;
        for (k=0; k<MAX_LIVECHAN; k++) cur.live_chan[k] = -1;

        cur.reject = NULL;
        cur.rejhead = NULL;
        cur.isreject = NULL;
        cur.isnot = NULL;
        cur.nreject = 0;
}
void live_shutdown(int ff);
void live_shutdown(int ff)
{
        Feed *f = &feeds[ff];   
        int i;
        int c;

        if (!f->live) return; 
        for (i=0; i<f->live_nchan; i++) {
                c = f->live_chan[i];
                if (c>=0)  {live_close(c); }
        }

}
void feed_clear(void)
{
        int i;
        int j;
        Feed *f;
        for (i=0; i<nfeeds; i++) {
		tout_shutdown(i);
                live_shutdown(i);
                setfree(&feeds[i].to);
                setfree(&feeds[i].site);
                setfree(&feeds[i].distrib);
                setfree(&feeds[i].groups);
                setfree(&feeds[i].exclude);
                setfree(&feeds[i].uucp_dir);
                setfree(&feeds[i].bind);
                setfree(&feeds[i].gmask);
                setfree(&feeds[i].gblock);
                setfree(&feeds[i].gfile);
                setfree(&feeds[i].rules);
		if (feeds[i].rules_code!=NULL) dfilter_free(feeds[i].rules_code);
		feeds[i].rules_code = NULL;

                setfree(&feeds[i].prog_name);
                setfree(&feeds[i].prog_args);
                if (feeds[i].reject!=NULL) {
                  f = &feeds[i];
                  for (j=0; j<f->nreject; j++) {
                        if (f->reject[j]!=NULL) mystrfree(f->reject[j],SD_FEED);                           
                        if (f->rejhead[j]!=NULL) mystrfree(f->rejhead[j],SD_FEED);                         
                  }
                  myfree(f->reject);
                  myfree(f->rejhead);
                  myfree(f->isreject);
                  myfree(f->isnot);
                  f->rejhead = NULL;
                  f->reject = NULL;
                  f->isreject = NULL;
                  f->isnot = NULL;
                  f->nreject = 0;
                }
        }
        nfeeds = 0;
}
void setfree(char **s)
{
        if (*s != NULL) mystrfree(*s,SD_FEED);
        *s = NULL;
}
void feed_new(void)
{
        if (cur.site==NULL) return;
        if (nfeeds+2>=MAX_FEEDS) {emsg("fatal: Can't handle that many feeds \n"); return;}
	if (nfeeds==0) if (strcmpnc(cur.site,"me")!=0) emsg("fatal: The first feed in newsfeeds.conf MUST be the ME feed, e.g. site me \n");
        feeds[nfeeds] = cur;
        nfeeds++;               
        feed_defaults();
}
char *f_get_var(char *s);
void feed_type(Feed *cur, char *type);
void feed_type(Feed *cur, char *type)
{
        cur->builtin = FALSE;
        cur->posting = FALSE;
        cur->smtp = FALSE;
        cur->live = FALSE;
	cur->isdfeed = FALSE;
	cur->issendthread = FALSE;
	cur->th = NULL;
        cur->uucp = FALSE;
        cur->isprog = FALSE;
        cur->prog_single = FALSE;
        strlwr(type);
        if (strcmp(type,"smtp")==0) cur->smtp = TRUE;
        else if (strcmp(type,"builtin")==0) cur->builtin = TRUE;
        else if (strcmp(type,"posting")==0) cur->posting = TRUE;
        else if (strcmp(type,"smtp")==0) cur->smtp = TRUE;
        else if (strcmp(type,"live")==0) cur->live = TRUE;
        else if (strcmp(type,"dfeed")==0) cur->isdfeed = TRUE;
        else if (strcmp(type,"uucp")==0) cur->uucp = TRUE;
        else if (strcmp(type,"program")==0) cur->isprog = TRUE;
        else if (strcmp(type,"single")==0) cur->prog_single = TRUE;
        else emsg("Unknown feed type {%s} \n",type);
        if (cur->live) cur->builtin = FALSE;
        if (cur->isdfeed) cur->builtin = FALSE;
        if (cur->posting) cur->builtin = TRUE;
        if (cur->prog_single) cur->isprog = TRUE;
}
void add_reject(Feed *f, char *head, char *match, int isreject);
char *feed_mystrdup(char *s)
{
	char *n = mymalloc(strlen(s)+1);
	strcpy(n,s);
	return n;
}
void feed_add(char *rec)
{
        int n;
        char *s;
        char *bf;
        char head[BFSZ];
        int l;
        static int group_cont;
        char urec[BFSZ];
	ncpy(urec,rec,BFSZ-100);
        strlwr(rec);
        s = get_word(rec,0);
        if (group_cont) { 
                s = get_word(rec,0);
                bf = mymalloc(strlen(cur.groups) + strlen(s) + 2);
                strcpy(bf,cur.groups); strcat(bf,s);
                myfree(cur.groups);
                cur.groups = bf;
contin:         l = strlen(cur.groups);
                if (l>=0) if (cur.groups[l-1]=='\\') {
                        group_cont = TRUE;
                        cur.groups[l-1] = 0;
                        return;
                } 
                group_cont = FALSE;
                return;
        }
        if (strlen(s)==0) return;
        if (*s=='#') return;

        for (n=0; strlen(s)>0; n++) {
                dmsg("feed: feed word {%s} \n",s);
                n++;
                if (strcmp(s,"site")==0) {feed_new(); cur.site = mystrdup(f_get_var(get_word(rec,n)),SD_FEED);}
                else if (strcmp(s,"exclude")==0) cur.exclude = strlwr(mystrdup(get_word(rec,n),SD_FEED));
                else if (strcmp(s,"distrib")==0) cur.distrib = mystrdup(get_word(rec,n),SD_FEED);
                else if (strcmp(s,"to")==0) cur.to = mystrdup(get_word(rec,n),SD_FEED);
                else if (strcmp(s,"log")==0) cur.islog = TRUE;
                else if (strcmp(s,"udp_ttl")==0) ;
                else if (strcmp(s,"compress")==0) ;
                else if (strcmp(s,"udp_speed")==0) ;
                else if (strcmp(s,"uucp_dir")==0) cur.uucp_dir = mystrdup(get_word(rec,n),SD_FEED);
                else if (strcmp(s,"bind")==0) cur.bind = mystrdup(get_word(rec,n),SD_FEED);
                else if (strcmp(s,"reject")==0) {ncpy(head,get_word(rec,n),BFSZ-1); n++; add_reject(&cur,head,get_wordq(rec,n),TRUE); return;}
                else if (strcmp(s,"accept")==0) {ncpy(head,get_word(rec,n),BFSZ-1); n++; add_reject(&cur,head,get_wordq(rec,n),FALSE); return;}
                else if (strcmp(s,"prog_name")==0) cur.prog_name = mystrdup(get_word(rec,n),SD_FEED);
                else if (strcmp(s,"rules")==0) cur.rules = mystrdup(get_word(rec,n),SD_FEED);
                else if (strcmp(s,"prog_args")==0) {cur.prog_args = mystrdup(get_wordq(rec,n),SD_FEED); break; }
                else if (strcmp(s,"groups")==0) {
                        cur.groups = feed_mystrdup(get_wordv(urec,n));
                        if ((cur.groups[1]==':')||(strncmp(cur.groups,"/",1)==0)||(strncmp(cur.groups,"\\",1)==0)) {
                                cur.gfile = mystrdup(cur.groups,SD_FEED);
                        }
                        goto contin;
                }
                else if (strcmp(s,"type")==0) feed_type(&cur,get_word(rec,n));
                else if (strcmp(s,"uucp_size")==0) cur.uucp_size = atoi(get_word(rec,n));
                else if (strcmp(s,"uucp_n")==0) cur.uucp_n = atoi(get_word(rec,n));
                else if (strcmp(s,"uucp_timer")==0) cur.uucp_timer = atoi(get_word(rec,n));
                else if (strcmp(s,"pending_max")==0) cur.pending_max = atoi(get_word(rec,n));
                else if (strcmp(s,"live_nchan")==0) cur.live_nchan = atoi(get_word(rec,n));
                else if (strcmp(s,"path_max")==0) cur.path_max = atoi(get_word(rec,n));
                else if (strcmp(s,"maxpath")==0) cur.path_max = atoi(get_word(rec,n));
                else if (strcmp(s,"maxsize")==0) cur.max = atoi(get_word(rec,n));
                else if (strcmp(s,"delay")==0) cur.xdelay = atoi(get_word(rec,n));
                else if (strcmp(s,"maxcross")==0) cur.cross = atoi(get_word(rec,n));
                else if (strcmp(s,"ignorepath")==0) {cur.nopath=TRUE; n--; }
                else if (strcmp(s,"live_only")==0) {cur.live_only=TRUE; n--; }
                else if (strcmp(s,"onlyposts")==0) {cur.onlyposts=TRUE; n--; }
                else if (strcmp(s,"xreplic")==0) {cur.isxreplic=TRUE; n--; }
                else if (strcmp(s,"sendthread")==0) {cur.issendthread=TRUE; n--; }
                else if (strcmp(s,"live_nocheck")==0) {cur.live_nocheck=TRUE; n--; }
                else if (strcmp(s,"builtin")==0) {cur.builtin=TRUE; n--; }
                else if (strcmp(s,"posting")==0) {cur.posting=TRUE; n--; npost++; }
                else if (strcmp(s,"smtp")==0) {cur.smtp=TRUE; n--; }
                else if (strcmp(s,"dolog")==0) {cur.islog=TRUE; n--; }
		else if (strcmp(s,"nchan")==0) { ;}
                else emsg("fatal: Error in newfeeds file %d{%s} make sure parameter lists have NO spaces\n",n,s);
                s = get_word(rec,n+1);
                dmsg("Next {%s} %d from {%s} \n",s,n+1,rec);
        }
        if (cur.live_nchan>=MAX_LIVECHAN) cur.live_nchan = MAX_LIVECHAN-1;
        if (cur.uucp_n==0) cur.uucp_n = 1;
}
int word_isnot(void);
void add_reject(Feed *f, char *head, char *match, int isreject)
{
        int n;
        if (strcmp(head,"hash")==0) {
                if (strlen(match)<3) {
                        emsg("hash rules must be more than 2 characters long (%s) \n",match);
                } else {
                        imsg("Adding spam hashed reject rule (%s)\n",match);
                        spam_rej_add(match);
                }
                return;
        }
        n = f->nreject++;
        if (n>=MAX_REJECT) {
                emsg("Too many accept/reject rules (%s) (%s) \n",head,match);
                f->nreject--;
                return;
        }
        if (f->reject==NULL) {
                f->reject = mymalloc(MAX_REJECT*sizeof(char *));
                memset(f->reject,0,MAX_REJECT*sizeof(char *));
                f->rejhead = mymalloc(MAX_REJECT*sizeof(char *));
                memset(f->rejhead,0,MAX_REJECT*sizeof(char *));
                f->isreject = mymalloc(MAX_REJECT*sizeof(int));
                memset(f->isreject,0,MAX_REJECT*sizeof(int));
                f->isnot = mymalloc(MAX_REJECT*sizeof(int));
                memset(f->isnot,0,MAX_REJECT*sizeof(int));
        }
        f->rejhead[n] = mystrdup(strlwr(head),SD_FEED);
        f->reject[n] = mystrdup(strlwr(str_decodelf(match)),SD_FEED);
        f->isreject[n] = isreject;
        f->isnot[n] = word_isnot();
}
char *f_get_var(char *s)
{
        if (strcmp(s,"$nntp_feeder")==0) return suck_getsuckn(1);
        if (strcmp(s,"$nntp_suck1")==0) return suck_getsuckn(1);
        if (strcmp(s,"$nntp_suck2")==0) return suck_getsuckn(2);
        if (strcmp(s,"$nntp_suck3")==0) return suck_getsuckn(3);
        if (strcmp(s,"$nntp_suck4")==0) return suck_getsuckn(4);
        if (strcmp(s,"$nntp_suck5")==0) return suck_getsuckn(5);
        if (strcmp(s,"$nntp_suck6")==0) return suck_getsuckn(6);
        if (strcmp(s,"$nntp_suck7")==0) return suck_getsuckn(7);
        if (strcmp(s,"$nntp_suck8")==0) return suck_getsuckn(8);
        if (strcmp(s,"$nntp_suck9")==0) return suck_getsuckn(9);
        if (*s=='$') emsg("feed: Unknown variable %s \n",s);
        return s;
}
/*      Return true if ME wants this group. */
int feed_me(char *msgid,char *distrib,char *groups[],int *groupid, char *opath, int size, String *xitem)
{
        int r;
        r = feed_test(0,msgid,distrib,groups,groupid,opath,size,xitem);
        return r;
}



/*      Feed the article to everyone else who might want it  */
static int post_incoming;
void feed_setpost(int t)
{
	inmsg_setpost(t);
        post_incoming = t;
}
int feed_ispost(void)
{
	return post_incoming;
}
int feed_link(char *msgid,char *distrib,char *groups[],int *groupid, char *opath, int size,char *groupnum, String *xitem)
{
        int f,r;

        if (xitem==NULL) return TRUE;
        if (str_s(xitem)==NULL) return TRUE;

        if (strstr(msgid,"<fake@netwin.com.nz>")!=NULL) {
                imsg("Not feeding fake message out to other sites\n");
                return TRUE;
        }
	/* if true block from feeds but store locally */
	if (feed_one_disable) {
		imsg("Bouncing local spam post from feeds\n");
		feed_one_disable = FALSE;
		return TRUE;
	}

        for (f=1; f<nfeeds; f++) {
                if (feeds[f].posting && !post_incoming && !config_true(C_POST_ROUTE)) {
                        dmsg("feedtst: Don't route to a posting feed\n"); 
                } else {
                        r = feed_test(f,msgid,distrib,groups,groupid,opath,size,xitem);
                        if (r) {
                                if (feeds[f].uucp) feed_uucp_write(f,xitem);
                                else if (feeds[f].live) {
                                        live_feed(f,msgid,groupnum,xitem);
                                } else if (feeds[f].isprog) {
                                        program_feed(f,msgid,groupnum,xitem);
                                } else if (feeds[f].issendthread) {
					if (!tout_feed(f,msgid,groupnum,xitem)) {
						feed_send(f,msgid,groupnum,0);	
					}	
				} else feed_send(f,msgid,groupnum,0) ;
                        } else dmsg("feedtst: To (%s) rule rejected %s %s \n",feeds[f].site,groupnum,msgid);
                } 
        }       
	tout_free_pot();

        return TRUE;    
}

void feed_exit(void)
{
        int f;
	
	feed_pool_flush();
        for (f=1; f<nfeeds; f++) {
                if (feeds[f].isprog) {
                        imsg("feed: Closing program feed [%d]\n",f);
                        prog_close(f);
                }
        }              

	if (prog_pid!=0) spawn_kill(prog_pid);
	prog_pid = 0;
}


int feed_wanted(char *s)
{
        if (nfeeds<=0) {
                emsg("newsfeeds.conf is faulty, it must contain a ME feed\n");
                return FALSE;
        }
        if (feeds[0].groups==NULL) {
                emsg("fatal: groups must be defined for ME feed\n");
                feeds[0].groups = feed_mystrdup("*");
        }
  return (group_match(feeds[0].groups,s));
}
int feed_posttest(char *groups[],int ng)
{
        int i;
        int f;

        post_feed = -1;
        for (f=0; f<nfeeds; f++) if (feeds[f].posting) break;
        if (f==nfeeds) return FALSE;
        
        for (f=0; f<nfeeds; f++) {
         if (feeds[f].posting) {
                for (i=0;(groups[i]!=NULL) && (i<ng);i++) {
                        post_feed = f;
                        if (feeds[f].groups!=NULL) if (group_match(feeds[f].groups,groups[i])) return TRUE;
                }
         }
        }
        post_feed = -1;
        return FALSE;
}
static int isspam;
int feed_wasspam(void)
{
        return isspam;
}
static int rulesonly;
void feed_skipspam(int x)
{
	rulesonly = x;
}
int feed_wild_test(int f,char *gname, int *blockit)
{
	*blockit = FALSE;
	if (group_match(feeds[f].groups,gname)) return TRUE;
        else if (group_match_none()) *blockit = TRUE;
	return FALSE;
}
static int is_skipfilter,is_skipfrom,is_skipdup;


int feed_test_real(int f, char *msgid,char *distrib,char *groups[], int *groupid, char *opath, int size,String *xitem);
int feed_test(int f, char *msgid,char *distrib,char *groups[], int *groupid, char *opath, int size,String *xitem)
{
	int r;
	stat_start(X_FEEDTST);
	r = feed_test_real(f,msgid,distrib,groups,groupid,opath,size,xitem);
	stat_end(X_FEEDTST);
	return r;
}
static int no_rules;
int feed_setnorules(int x)
{
	no_rules = x;
	return TRUE;
}
int feed_test_real(int f, char *msgid,char *distrib,char *groups[], int *groupid, char *opath, int size,String *xitem)
{
        char xpath[4000];
        char bigbf[11000];
        int npath;
        char *s2;
        int len;
        char *p[1000];
        char list[4000];
        int t;
	int blockit = FALSE;
        char bf[BFSZ];
        char *s;
        int i,r,ng;
        int gotone;
        int anyaccept;
        int isdefault = FALSE;
        int defreject = 0;
        char reason[BFSZ];
        char allowgroups[BFSZ];
        int me_cont = FALSE;
        cur = feeds[f];

	is_skipfilter = is_skipfrom = is_skipdup = FALSE;
        ncpy(allowgroups,config_get(C_SPAM_ALLOWGROUPS),BFSZ-1);
	vtrace(); 
	if (config_true(C_SKIP_DUP)) is_skipdup = TRUE;

	
	        for (i=0,ng=0;groups[i]!=NULL;i++) {
                ng++;
				if (groups[i]==1) {
					emsg("Sorry1, groups array, entry %d is corrupt\n",i);
					abort();
				}
        }

        isspam = FALSE;
        if (cur.onlyposts) {
                if (!post_incoming) {
                        msg_show("onlyposts is set, and it's not a local post");
                        dmsg("feedtst: [%d] onlyposts set and not a local post\n",f);
                        return FALSE;
                }
        }
        if (cur.smtp) {
                msg_show("Me feed might be missing");
                return FALSE;     /* Only for email posts */
        }
        if (!cur.nopath) {                      /* Check the path */
                ncpy(xpath,opath,3999);
                strlwr(xpath);
                npath = token_split(xpath,p," \t,;:!@",999);
                for (i=0; i<npath; i++) {
                        s = p[i];
			if (strlen(s)>100) {
                                msg_show("Very long site in path probably corrupt");
				return FALSE;
			}
                        if (strcmp(s,cur.site)==0) if (strcmp(s,"me")!=0) {
                                dmsg("feedtst: Rejected, it's in the path\n");
                                msg_show("Site already in path");
                                return FALSE;
                        }
                        /* not really a group but same match should work */
                        if (cur.exclude!=NULL) if (group_match(cur.exclude,s)) {
                                dmsg("feedtst: Rejected, it's in the excludepath\n");
                                msg_show("Site already in path");
                                return FALSE;
                        }
                }
                if (cur.path_max>0) if (npath>cur.path_max) {
                        lprintf(bf,BFSZ-1,"Path too long [%d] npath %d > path_max %d",f,npath,cur.path_max); 
                        msg_show(bf);
                        dmsg("feedtst: %s\n",bf);
                        return FALSE;
                }
        }

	vtrace(); 
        if (cur.groups==NULL) goto got_one;
        for (i=0,ng=0;groups[i]!=NULL;i++) {
                ng++;
				if (groups[i]==1) {
					emsg("Sorry2, groups array, entry %d is corrupt\n",i);
					abort();
				}
        }
        if (ng>cur.cross) {
                dmsg("feedtst: Rejected, cross posted too much %d\n",ng);
                msg_show("Cross posted to too many groups");
                return FALSE;
        }
        if (size>cur.max) {
                if (strlen(msgid)<100) imsg("feed: [%s] Rejecting large item %d bytes  %s \n",feeds[f].site,size,msgid);
                msg_show("Item too big");
                return FALSE;
        }


	vtrace(); 
        gotone = FALSE;
	/*if (rulesonly)  goto got_one;*/
        for (i=0;groups[i]!=NULL;i++) {
		if (!config_true(C_FEED_NOMASK)) {
			if (groupid==NULL) r = group_find(groups[i]);
			else r = groupid[i];
			r = feed_mask_test(f,r,&blockit);
		} else {
			r = feed_wild_test(f,groups[i],&blockit);
		}
                if (r) gotone = TRUE;
                else if (blockit) {
                        if (f==0) if (config_true(C_ME_CONTINUE)) {
                                me_cont = TRUE;
                                goto got_one;   
                        }
						if (groups[i]==1) emsg("Bad groups arrany entry %d\n",i);
                        dmsg("feedtst: Reject: Due to @ match (%s) \n",xtrunc(groups[i]));
                        gotone = FALSE;
                        break;
                }
        }



	vtrace(); 
        if (gotone) goto got_one;
        dmsg("feedtst: Rejected, no groups matched\n");
		if (groups[0]==1) emsg("Bad groups arrany entry2 %d\n",i);
        if (groups[0]!=NULL) {
                lprintf(bf,BFSZ-1,"No newsgroups matched {%s}",xtrunc(groups[0]));
        } else strcpy(bf,"No newsgroups match");
        msg_show(bf);
        return FALSE; /* Not a one of the groups matched me */

got_one:
        if (cur.distrib==NULL) {
                dmsg("feedtst: Accepted, no distribution restriction\n");
                goto doreject;
        }
        if (strlen(distrib)==0) {
                dmsg("feedtst: Accepted, no distribution in item \n");
                goto doreject; /* Send the article */
        }

/*cur= alt,biz,hort,nz |  !alt,!sex   distrib = hort,nz */
        ncpy(xpath,distrib,3999);
        strlwr(xpath);
        s = strtok(xpath," \t,;:!@");
        strcpy(list,cur.distrib);
        strcat(list,",");
	vtrace(); 
        for (;s!=NULL;) {
                r = feed_distmatch(cur.distrib,s);
                if (r==2) dmsg("feedtst: Distribution matched (not) %d {%s} {%s} \n",r,cur.distrib,s);
                if (r==1) goto doreject;
                if (r==2) {
                        msg_show("Distribution Rule");
                        return FALSE;
                }
                s = strtok(NULL," \t,;:!@");
        }
        /* If no match and some 'not' then answer is 'yes' we want it*/
        s = strchr(cur.distrib,'!');
        if (s!=NULL) goto doreject; 
        dmsg("feedtst: No distribution match so rejecting {%s} \n",msgid);
        msg_show("Distribution default");
        return FALSE;
doreject: /* So far we want the article */
	vtrace(); 

        if (xitem==NULL) return TRUE;
        if (!config_true(C_SPAM_DEBUG)) {
                set_rej_silent(TRUE);
        }
        if (config_true(C_SPAM_LOCALONLY)) if (!post_incoming) return TRUE;
	if (no_rules) return TRUE;

	if (cur.rules_code!=NULL) {
		char reason[BFSZ];
		int r;
	
		vtrace(); 
		{
			r = TRUE;
			if (config_true(C_RULES_LOCAL_ONLY)) if (!post_incoming) goto skip_rules;
			stat_start(X_RULES);
			r = dfilter_run(cur.rules_code,str_s(xitem),str_len(xitem),reason);
			stat_end(X_RULES);
skip_rules:;
		}
		if (!r) {
			if (config_true(C_RULES_DEBUG)) set_rej_silent(FALSE);
			imsg("rules: reject %s\n",reason);
			msg_show(reason);
			spam_store(xitem);
			if (f==0) if (post_incoming) {
				ncpy(bf,reason,100);
				goto report_spam;
			}
			return FALSE;
		} else {		
			if (config_true(C_LOG_ACCEPT))
			    imsg("rules: accept %s\n",reason);
			is_skipfilter = skip_filter();
			is_skipfrom = skip_from();
			is_skipdup = skip_dup();
		}				
	}
	if (rulesonly) goto skip_spam;
        isspam = TRUE;

	vtrace(); 
        if (cur.live) if (ishdr()) if (f!=0) {
                for (i=0;groups[i]!=NULL;i++) {
                       if (hdr_matchgroup(groups[i])) {
                                msg_show("Can't feed item if it is cross posted to header groups");
				imsg("Can't feed item if it is cross posted to header groups\n");
				return FALSE;
		       }
                }
	}
        
	
	vtrace(); 
	if (f==0) if (config_true(C_SPAM_STOP)) {
                if (strlen(allowgroups)>0) {
                        for (i=0;groups[i]!=NULL;i++) {
                                if (!group_match(allowgroups,groups[i])) goto check_spam;
                        }
                        goto skip_spam;
                }
check_spam:     
		if (spam_test(xitem,ng,reason,msgid)) {
                 if (!config_true(C_SPAM_TEST)) {
			/* if true block from feeds but store locally */
			if (post_incoming) if (config_true(C_SPAM_POST_LOCAL_OK)) {
				imsg("Allowing locally posted spam to be stored, but not allowing it to feed out\n");	
				imsg("Reason %.100s\n",reason);	
				feed_one_disable = TRUE;
				goto skip_spam;
			}                                
			msg_show(reason);
			if (f==0) if (post_incoming) report_spammer(xitem,reason);
			return FALSE;
                 }
                }
        }
skip_spam:
        set_rej_silent(FALSE);
        if (cur.nreject==0) {
                if (me_cont) return FALSE;
                return TRUE;
        }
/*
        If rule rejects, then return false
        if rule accepts return true
        if end of rules, and one accept, return false
        else return true
*/
        anyaccept = FALSE;
        isdefault = FALSE;
	vtrace(); 
        for (i=0; i<cur.nreject; i++) {
dmsg("feed: Testing Rule [%d] (%s) (%s) state=%d\n",i,cur.rejhead[i],cur.reject[i],isdefault);
                if (!cur.isreject[i]) anyaccept = TRUE;
                if (strcmp(cur.rejhead[i],"default")==0) {
                        isdefault = TRUE;
                        defreject = cur.isreject[i];
                }
                if (strcmp(cur.rejhead[i],"body")==0) {
                        s = str_s(xitem);
                        if (s==NULL) goto ret_true;
                        s = strstr(s,"\r\n\r\n");
                        if (s==NULL) goto ret_true;
                        goto testit;
                }
                if ((strcmp(cur.rejhead[i],"header")==0) ||
                 (strcmp(cur.rejhead[i],"head")==0)) {
                        s = str_s(xitem);
                        if (s==NULL) goto ret_true;
                        s2 = strstr(s,"\r\n\r\n");
                        if (s2==NULL) goto ret_true;
                        len = s2-s;
                        if (len>10000) len = 10000;
                        ncpy(bigbf,s,len);
                        s = bigbf;
                        goto testit;
                }
                if (strcmp(cur.rejhead[i],"article")==0) {
                        s = str_s(xitem);
                        if (s==NULL) goto ret_true;
                        goto testit;
                }
                if (strcmp(cur.rejhead[i],">")==0) {
                  if (str_len(xitem)>atoi(cur.reject[i])) {
                        defreject = cur.isreject[i];
                        if (cur.isreject[i]) {
                                lprintf(bf,BFSZ-1,"Reject rule (%s) < %d",cur.reject[i],str_len(xitem));
                                imsg("me: %s \n",bf);
                        }
                  }
                }
                if (strcmp(cur.rejhead[i],"<")==0) {
                  if (str_len(xitem)<atoi(cur.reject[i])) {
                        defreject = cur.isreject[i];
                        if (cur.isreject[i]) {
                                lprintf(bf,BFSZ-1,"Reject rule (%s) > %d",cur.reject[i],str_len(xitem));
                                imsg("me: %s \n",bf);
                        }
                  }
                }
                ncpy(bf,get_ahead(xitem,cur.rejhead[i]),BFSZ-1);
                strlwr(bf);
                s = bf;
testit:;
                t = (mystrstrnc(s,cur.reject[i])!=NULL);
                if (cur.isnot[i]) t = !t;
                if (t) {
                        defreject = cur.isreject[i];
                        if (isdefault) dmsg("feed: rule matched (%s)\n",cur.reject[i]);
                        if (cur.isreject[i]) {
                                lprintf(bf,BFSZ-1,"Reject rule (%s)",cur.reject[i]);
                                if (!isdefault) {
                                        msg_show(bf);
					if (f==0) if (post_incoming) goto report_spam;
                                        return FALSE;
                                } 
                        }
                        if (!isdefault) goto ret_true;
                } else dmsg("feed: Rule didn't match (%s)\n",cur.reject[i]);
        }
	vtrace(); 
        if (isdefault) {
                if (defreject) {
                        msg_show("Last matched rule was a reject");
			if (f==0) if (post_incoming) {strcpy(bf,"reject rule matched"); goto report_spam;}
                        return FALSE;
                }
                goto ret_true;
        }
        if (anyaccept) {
                msg_show("No accept rule matched");
		if (f==0) if (post_incoming) {strcpy(bf,"No accept rule matched"); goto report_spam;}
                return FALSE;
        }
ret_true:;
	vtrace(); 
        if (me_cont) return FALSE;
        return TRUE;

report_spam:;
	if (config_true(C_SPAM_POST_LOCAL_OK)) {
		imsg("Allowing locally posted spam to be stored, but not allowing it to feed out\n");	
		imsg("Reason %.100s\n",bf);	
		feed_one_disable = TRUE;
		goto ret_true;
	}       
	report_spammer(xitem,bf);
	return FALSE;
}
/*-- SPAM -------------------------------------------*/
int spam_dup_total(void);
int spam_dup_cross(void);
int spam_dup_hits(void);
int spam_from_cross(void);
int spam_from_hits(void);
int spam_crc(String *xitem,int ng,char *reason);
void spam_crc_trim(void);
void spam_rej_add(char *s);
int spam_rej_hashx(char *s);
int spam_reject(String *xitem,int ng,char *reason);
int spam_from(String *xitem,int ng,char *reason);
char *spam_dup(char *s);
void spam_dup_free(void);
void spam_crc_age(void);
void spam_from_age(void);

static int totspam,totrej,totdup,totfrom,totsubj;
static char ipname[BFSZ];
void feed_setipname(char *s)
{
        strcpy(ipname,s);
}
int spam_test(String *xitem,int ng,char *reason, char *msgid)
{
        static int n;
        int nkeep;
        static int lasthour;
        char bf[BFSZ];
        static char allow[BFSZ];
        static char allowip[BFSZ];
	char subject[BFSZ];
	char subj[BFSZ];
	char fname[BFSZ];
	int part,partmax;
        static String *ahead;
	int age_time = atoi(config_get(C_SPAM_AGE));

	if (age_time==0) age_time = 3600;
      

        if (lib_gmtime()-lasthour>age_time) {
                imsg("Age spam data\n");
                lasthour = lib_gmtime();
                spam_crc_age();
                spam_from_age();
                ncpy(allow,config_get(C_SPAM_ALLOWHOSTS),BFSZ-1);
                ncpy(allowip,config_get(C_SPAM_ALLOWIHAVE),BFSZ-1);
        }

        totspam++;

        if (allow[0]!=0) {
                ncpy(bf,get_ahead(xitem,"X-NNTP-Posting-Host:"),80);
                if (strlen(bf)==0) {
                        ncpy(bf,get_ahead(xitem,"NNTP-Posting-Host:"),80);
                }
                if (group_match(allow,bf)) return FALSE; 
        }
        if (allowip[0]!=0) {
                if (group_match(allowip,ipname)) return FALSE; 
        }

        
        ncpy(subject,get_ahead(xitem,"Subject:"),BFSZ-1);
	if (parts_subject(subject, subj, fname, &part, &partmax)) {
		if (partmax>1) {
			if (config_true(C_SPAM_IGNORE_PARTS)) {
				dmsg("spam: not checking multi part message %d/%d\n",part,partmax);
				return FALSE; /* Don't count in filters if it's a multi part message */	
			}
		}
	}
	
	/* dupliecate body check  */    
	if (config_true(C_SKIP_DUP)) is_skipdup = TRUE;
	if (!is_skipdup)
          if (spam_crc(xitem,ng,reason)) {totdup++; goto rej;}

        /* In our list of reject body rules */          
	if (!is_skipfilter)
          if (spam_reject(xitem,ng,reason)) {totrej++; goto rej;}

        if (ahead==NULL) ahead = str_new();
        ncpy(bf,get_ahead(xitem,"Subject:"),100);
        str_copysn(ahead,bf);
	if (!is_skipfilter)
          if (spam_reject(ahead,ng,reason)) {totrej++; goto rej;}
        ncpy(bf,get_ahead(xitem,"From:"),100);
	if (!is_skipfilter)
          if (spam_reject(ahead,ng,reason)) {totrej++; goto rej;}
        

        /* Duplicate from,reply, org or whatever we are checking */ 
	if (!is_skipfrom) {
          if (spam_from(xitem,ng,reason)) {totfrom++; goto rej;}
          if (spam_subject(xitem,ng,reason)) {totsubj++; goto rej;}
	}

        /* Every hour subtract 'allowed hits' from 
        totals */

        return FALSE;

rej:
	if (config_true(C_FILTER_SILENT)) {
		set_rej_silent(TRUE);
	}
        nkeep = atoi(config_get(C_SPAM_KEEPN));
        ncpy(subj,get_ahead(xitem,"Subject:"),40);
        if (nkeep<0) {
                imsg("reject spam (%s) (%.100s) %.100s\n",reason,subj,msgid);
                return TRUE;
        }
        if (nkeep==0) nkeep = 5;
        if (n>nkeep) n = 0;
        n++;
        lprintf(bf,BFSZ-1,"spam_%d.tmp",n); 
        file_work(bf);
        imsg("reject spam (%s) (%s) stored as %s %.100s\n",reason,subj,file_work(bf),msgid);
        str_write(xitem,file_work(bf));
        return TRUE;
}
void spam_store(String *xitem)
{
	static int n;
	static char bf[BFSZ];
	int nkeep;
        nkeep = atoi(config_get(C_SPAM_KEEPN));
	if (nkeep<0) return;
        if (nkeep==0) nkeep = 5;
        if (n>nkeep) n = 0;
        n++;
        lprintf(bf,BFSZ-1,"rules_%d.tmp",n); 
        file_work(bf);
        imsg("Rule rejected message stored as (%s)\n",file_work(bf));
        str_write(xitem,file_work(bf));
}
int spam_dup_total(void)
{
        int r = atoi(config_get(C_SPAM_DUP_TOTAL));
        if (r>0) return r;
        return 5000;
}
int spam_dup_cross(void)
{
        int r = atoi(config_get(C_SPAM_DUP_CROSS));
        if (r>0) return r;
        return 200;
}
int spam_from_total(void)
{
        int r = atoi(config_get(C_SPAM_FROM_TOTAL));
        if (r>0) return r;
        return 5000;
}
int spam_dup_hits(void)
{
        int r = atoi(config_get(C_SPAM_DUP_HITS));
        if (r>0) return r;
        return 6;
}
int spam_from_cross(void)
{
        int r = atoi(config_get(C_SPAM_FROM_CROSS));
        if (r>0) return r;
        return 1000;
}
int spam_from_hits(void)
{
        int r = atoi(config_get(C_SPAM_FROM_HITS));
        if (r>0) return r;
        return 60;
}
int spam_subject_hits(void)
{
        int r = atoi(config_get(C_SPAM_SUBJECT_HITS));
        if (r>0) return r;
        return 200;
}

int spam_skip_line(char *body,int len) {
    char *c = body+len;
    int line = 0;
    int other = 0;

    if (len<50) return len;
    if (*c != 0) {dmsg("spam_skip_line: warning no terminating null\n"); return len;}
    /* skip any blank lines to find the end of the last non-blank line */
    for(--c;c>body;c--) {
	if (*c=='\r') continue;
	if (*c=='\n') continue;
	break;
    }
    if (*c=='.') for(--c;c>body;c--) {
	if (*c=='\r') continue;
	if (*c=='\n') continue;
	break;
    }
    /* skip to beginning of the last non-blank line */
    for(--c;c>body;c--) {
	if (*c=='\r') break;
    }
    if ((c-body) < 50) return len;
    return (c-body);
}

#define HASH_CRC 5000
typedef struct SPAMCRC {
        unsigned int crc; unsigned short len; 
        short hits; short cross; 
        struct SPAMCRC *next;
} Spamcrc;
Spamcrc *hashcrc[HASH_CRC];
static int spam_ncrc;
int spam_crc(String *xitem,int ng,char *reason)
{
        /* CRC the article, 
        lookup/add to table, 
        add hit, 
        reject if above max hit/crossposts 
        trim table if it's too big */

        Spamcrc *s;
        unsigned int crc;
        int len,h;
        char bf[BFSZ];
        char *body = strstr(str_s(xitem),"\r\n\r\n");
        if (body==NULL) return FALSE;
        len = strlen(body);
        if (len<12) return FALSE;
        if (len>32000) len = len % 32000;
        len = spam_skip_line(body,len);
	crc = lib_ncrc32(body,len);
	/*crc = lib_crc32(body);*/
        ncpy(bf,body,40);
        dmsg("spam: body len=%d (%s)  \n",len,bf);
        h = crc % HASH_CRC; if (h<0) h = -h;
        for (s=hashcrc[h];s!=NULL;s=s->next) {
          if (s->crc==crc) if (s->len==len) {
                s->hits++;
                s->cross += ng;
                if (s->hits >= spam_dup_hits()) {
                        lprintf(reason,BFSZ-1,"spam_dup_hits, hits=%d cross=%d (%d:%d)",s->hits,s->cross,crc,len);
                        return TRUE;
                }
                if (s->cross > spam_dup_cross()) {
                        lprintf(reason,BFSZ-1,"spam_dup_cross, hits=%d cross=%d (%d:%d)",s->hits,s->cross,crc,len);
                        return TRUE;    
                }
                dmsg("spam: [%d:%d] spam_dup hits %d cross %d\n",h,crc,s->hits,s->cross);
                return FALSE;
          }
        }
        dmsg("spam: crc=%d h=%d %d\n",crc,h,spam_ncrc); 
        s = mymalloc(sizeof(Spamcrc));
        memset(s,0,sizeof(Spamcrc));
        s->next = hashcrc[h];
        s->len = len;
        s->crc = crc;
        hashcrc[h] = s;
        spam_ncrc++;
        if (spam_ncrc>spam_dup_total()) spam_crc_trim();
        return FALSE;
}
void spam_crc_trim(void)
{
        static int upto;
        Spamcrc *s,*snext;
        for (;spam_ncrc>=spam_dup_total();) {
         upto++;
         if (upto>=HASH_CRC) upto = 0;
         for (s=hashcrc[upto];s!=NULL;s=snext) {
                snext = s->next;
                spam_ncrc--;
                myfree(s);
         }
         hashcrc[upto] = NULL;
        }
}
void spam_crc_age(void)
{
        int h;
        int agecross = spam_dup_cross();
        int agehits = spam_dup_hits();
        Spamcrc *s;

        for (h=0;h<HASH_CRC;h++) {
         for (s=hashcrc[h];s!=NULL;s=s->next) {
                s->hits -= agehits;
                s->cross -= agecross;
                if (s->hits<0) s->hits = 0;
                if (s->cross<0) s->cross = 0;
         }
        }
}

#define HASH_REJ 15000
typedef struct SPAMREJ {
        char *s;
        short len;
        short hits;
        struct SPAMREJ *next;
} Spamrej;
Spamrej *hashrej[HASH_REJ];


void spam_rej_load(char *fname)
{
        FILE *f;
        char bf[BFSZ];
        char *s;
        char *c1, *c2;
        if (!config_true(C_SPAM_STOP)) return;
        f = myfopen(fname,"r");
        if (f==NULL) {imsg("spam: Could not find %s\n",fname); return;}
        for (;!feof(f);) {
                if (fgets(bf,BFSZ-2,f)) {
                        s = strtok(bf,"\n\r");
                        if (s!=NULL) {
                                c1 = strstr(s,"/*");
                                if (c1!=NULL) {
                                        c2 = strstr(c1+1,"*/");
                                        if (c2!=NULL) {
                                                memmove(c1,c2+2,strlen(c2+1));
                                        }
                                }
                                if (strlen(s)<3) imsg("spam: spam reject rules must be more than 2 chars (%s) \n",s);
                                else spam_rej_add(s);
                        }
                }
        }
        fclosep(f);
}
int spam_nfile;
void spam_rej_add(char *s)
{
        int h;
        Spamrej *r;
        strlwr(s);
        h = spam_rej_hash(s);
/*
worst[0] = -1;
worst[1] = -1;
worst[2] = -1;
emsg("spam_rej worst case %d\n",spam_rej_hash(worst));
*/
        r = mymalloc(sizeof(Spamrej));
        memset(r,0,sizeof(Spamrej));
        r->next = hashrej[h];
        r->len = strlen(s);
        r->s = mystrdup(s,SD_FEED);
        hashrej[h] = r;
        spam_nfile++;
}
void spam_rej_free(void)
{
        int h;
        Spamrej *r,*rnext;
        spam_nfile = 0;
        for (h=0; h<HASH_REJ; h++) {
                for (r = hashrej[h]; r!=NULL; r=rnext) {
                        rnext = r->next;
                        mystrfree(r->s,SD_FEED);
                        myfree(r);
                }
                hashrej[h] = NULL;
        }
}
int spam_rej_hashx(char *s)
{
        int h;
        char tmp[6];
        ncpy(tmp,s,3);
        strlwr(tmp); s = tmp;
        h = (s[0] + s[1]*64 + s[2]*256) % HASH_REJ;
        if (h<0) return -h;
        return h;
}
void spam_rej_show(int chan)
{
        int h;
        char bf[BFSZ];
        Spamrej *r;
        for (h=0; h<HASH_REJ; h++) {
                for (r = hashrej[h]; r!=NULL; r=r->next) {
                        lprintf(bf,BFSZ-1,"%08d %s\r\n",r->hits,r->s);
                        chan_write_s(chan,bf);
                }
        }
        
}
int spam_reject(String *xitem,int ng,char *reason)
{
        /* go thru article, hash 4 bytes, lookup
           if matches reject rule then reject.

	   Limit to first 32K by default, more if sepcified.
	   spam_rej_size
        */
        int h;
	int sz;
        char *s;
	char *start;
	int nlookup=0;
        Spamrej *r;
        s = str_s(xitem);
	start = s;

	sz = atoi(config_get(C_SPAM_REJ_SIZE));
	if (sz==0) sz = 30000;

        for (;(*s!=0) && (s[3]!=0);s++) {
		if ((s-start)>sz) break;
                h = spam_rej_hash(s);
                for (r=hashrej[h];r!=NULL;r=r->next) {
		 nlookup++;
                 if (strncmpnc(s,r->s,r->len)==0) {
                        r->hits++;
                        if (r->hits>30000) r->hits=30000;
                        lprintf(reason,BFSZ-1,"spam_reject keyword (%s) %d hits",r->s,r->hits);
                        return TRUE;            
                 }
                }
        }       
        return FALSE;
}

#define HASH_FROM 5000
typedef struct SPAMFROM {
        char *s;
        short hits; short cross; 
        struct SPAMFROM *next;
} Spamfrom;
Spamfrom *hashfrom[HASH_FROM];

static int spam_ndup;
int spam_subject(String *xitem,int ng,char *reason)
{
        /* Add subject and from together, only allow 2-3 of these 
	in window */
        int h;
        Spamfrom *r;
        char bf2[BFSZ];
        char *s;
        char bf[BFSZ];
	char *lasts=NULL;
	int strip=TRUE;
	int len;
	int i;

	ncpy(bf,get_ahead(xitem,"From:"),60);
	ncpy(bf2,get_ahead(xitem,"Subject:"),130);
	strlwr(bf2);
	if (strncmp(bf2,"re:",3)==0) return FALSE; 
	
	for (i=strlen(bf2)-1;i>=0; i--) {
		s = bf2 + i;
		if (*s==')') goto found_brace;
		if (*s==']') goto found_brace;
	}
	goto stripit;
found_brace:
	if (strchr(bf2,'/')!=NULL) 
	  if ((s-bf2)>10) {*s = 0; strip=FALSE;}
stripit:
	if (strip) {
		len = strlen(bf2)-10;
		if (len>40) len -= 15;
		if (len>20) bf2[len] = 0;
	}
        fix_address(bf);
 
	strcat(bf,bf2);
        strlwr(bf);
        dmsg("spam: using subject data (%s)\n",bf);
        h = lib_crc(bf) % HASH_FROM;
        if (h<0) h = -h;
        
        for (r=hashfrom[h];r!=NULL;r=r->next) {
          if (strcmp(r->s,bf)==0) {
                r->hits++;
                if (r->hits >= spam_subject_hits()) {
                        lprintf(reason,BFSZ-1,"spam_subject_hits %d hits (%s)",r->hits,r->s);
                        return TRUE;
                }
                dmsg("spam: spam_subj,  hits=%d \n",r->hits);
                return FALSE;
          }
        }
        r = mymalloc(sizeof(Spamfrom));
        memset(r,0,sizeof(Spamfrom));
        r->next = hashfrom[h];
        r->s = spam_dup(bf);
        hashfrom[h] = r;
        dmsg("spam: inserted from data [%d] (%s)\n",h,bf);
        spam_ndup++;
        if (spam_ndup>spam_from_total()) {
                spam_dup_free();        
                spam_ndup = 0;
        }
        return FALSE;
}
int spam_from(String *xitem,int ng,char *reason)
{
        /* [X-]NNTP-Posting-Host: whole thing*/
        /* From: email strip, Reply-To:*/
        int h;
        Spamfrom *r;
        char bf2[BFSZ];
        char *s;
        char bf[BFSZ];
        if (config_true(C_SPAM_HOST)) {
                ncpy(bf,get_ahead(xitem,"X-NNTP-Posting-Host:"),80);
                if (strlen(bf)==0) {
                        ncpy(bf,get_ahead(xitem,"NNTP-Posting-Host:"),80);
                }
                if (strlen(bf)==0) {
                        if (config_true(C_SPAM_NOHOST_REJECT)) {
                         strcpy(reason,"No NNTP-Posting-Host defined");
                         return TRUE;
                        }       
                        ncpy(bf,get_ahead(xitem,"From:"),80);
                        fix_address(bf);
                }
                s = strchr(bf,' '); if (s!=NULL) *s = 0;
        } else {
                ncpy(bf,get_ahead(xitem,"From:"),80);
                fix_address(bf);
        }
        if (config_true(C_SPAM_LINES)) {
                lprintf(bf2,BFSZ-1,"%d",str_len(xitem)%60);
                strcat(bf,bf2);
        }

        strlwr(bf);
        dmsg("spam: using from data (%s)\n",bf);
        h = lib_crc(bf) % HASH_FROM;
        if (h<0) h = -h;
        
        for (r=hashfrom[h];r!=NULL;r=r->next) {
          if (strcmp(r->s,bf)==0) {
                r->hits++;
                r->cross += ng;
                if (r->hits > spam_from_hits()) {
                        lprintf(reason,BFSZ-1,"spam_from_hits %d hits %d cross posts (%s)",r->hits,r->cross,r->s);
                        return TRUE;
                }
                if (r->cross > spam_from_cross()) {
                        lprintf(reason,BFSZ-1,"spam_from_cross, hits=%d cross=%d (%s)",r->hits,r->cross,r->s);
                        return TRUE;    
                }
                dmsg("spam: spam_from,  hits=%d cross=%d\n",r->hits,r->cross);
                return FALSE;
          }
        }
        r = mymalloc(sizeof(Spamfrom));
        memset(r,0,sizeof(Spamfrom));
        r->next = hashfrom[h];
        r->s = spam_dup(bf);
        hashfrom[h] = r;
        dmsg("spam: inserted from data [%d] (%s)\n",h,bf);
        spam_ndup++;
        if (spam_ndup>spam_from_total()) {
                spam_dup_free();        
                spam_ndup = 0;
        }
        return FALSE;
}
static char *dup_stuff[1000],*dup_at;
static int dupi= -1;
static int dup_left;
char *spam_dup(char *s)
{
        int len = strlen(s)+1;
        
        if (dup_left<len) {
                dupi++;
                if (dupi>500) dnews_ex("spam_from_total is too large");
                if (dup_stuff[dupi]==NULL) {
                        dup_stuff[dupi] = mymalloc(200000);
                }
                dup_left = 200000;
                dup_at = dup_stuff[dupi];
        }
        strcpy(dup_at,s);
        s = dup_at;
        dup_at += len;
        dup_left -= len;
        return s;
}
void spam_dup_free(void)
{
        int h;
        Spamfrom *r,*rnext;
        dupi = -1;
        dup_left = -1;
        spam_ndup = 0;

        for (h=0; h<HASH_FROM; h++) {
         for (r=hashfrom[h];r!=NULL;r=rnext) {
                rnext = r->next;
                myfree(r);
         }
         hashfrom[h] = NULL;
        }
}
void spam_from_age(void)
{
        int h;
        int agecross = spam_from_cross();
        int agehits = spam_from_hits();
        Spamfrom *r;
        for (h=0;h<HASH_FROM;h++) {
         for (r=hashfrom[h];r!=NULL;r=r->next) {
                r->hits -= agehits;
                r->cross -= agecross;
                if (r->hits<0) r->hits = 0;
                if (r->cross<0) r->cross = 0;
         }
        }
}

/*-------------------------------------------------*/

int feed_distmatch(char *list, char *dis)
{
        char tmp[100];
        char tmp2[100];
        char *s;

        ncpy(tmp,dis,90);
        strcat(tmp,",");

        ncpy(tmp2+1,tmp,90);
        tmp2[0] = '!';

        
        s = strstr(list,tmp2);
        if (s!=NULL) return 2; /* Matched a group we DONT want*/
        s = strstr(list,tmp);
        if (s!=NULL) return 1; /* Matched a group we DO want */
        return 0;       /* No match */
}
static FILE *fout[MAX_FEEDS];
static int nout;

int feed_find(char *s)
{
        int i;
        for (i=0; i<nfeeds; i++) {      
                if (strcmp(s,feeds[i].site)==0) return i;
        }
        return -1;
}
void feed_setlog(char *s, int t)
{
        int f = feed_find(s);
        if (f<0) {emsg("No such feed {%s}\n",s); return;}
        feeds[f].islog = t;
}
int feed_flush(void)
{
        feed_flush2(FALSE);
        return TRUE;
}
int feed_doit_if(void)
{
        if (config_true(C_SEND_LATER)) return TRUE;
        return feed_doit();
}
int feed_doit(void)
{
        int i;
       for (i=0; i<nfeeds; i++) {
                if (feeds[i].builtin) {
			feeds[i].nsent = 0;
                        feeds[i].last_time = 0;  /* so it will retry now*/
                        suck_queue(feeds[i].site,10);
                        imsg("feed: Queued outgoing NNTP {%s} \n",feeds[i].site);
                }
                if (feeds[i].smtp) {
			feeds[i].nsent = 0;
                        feeds[i].last_time = 0;  /* so it will retry now*/
                        suck_queue(feeds[i].site,11);
                        imsg("feed: Queued outgoing SMTP {%s} \n",feeds[i].site);
                }
        }
        return TRUE;
}
int feed_doit_name(char *feedname)
{
        int i;
        for (i=0; i<nfeeds; i++) {
		if (strcmpnc(feeds[i].site,feedname)!=0) continue; 
                if (feeds[i].builtin) {
			feeds[i].nsent = 0;
                        feeds[i].last_time = 0;  /* so it will retry now*/
                        suck_queue(feeds[i].site,10);
                        imsg("feed: Queued outgoing NNTP {%s} \n",feeds[i].site);
                }
                if (feeds[i].smtp) {
			feeds[i].nsent = 0;
                        feeds[i].last_time = 0;  /* so it will retry now*/
                        suck_queue(feeds[i].site,11);
                        imsg("feed: Queued outgoing SMTP {%s} \n",feeds[i].site);
                }
        }
        return TRUE;
}
int feed_flushsite(char *s)
{
        int f = feed_find(s);
	static int lastdbflush;
        char sendname[BFSZ];
        char feedname[BFSZ];
        char requename[BFSZ];

        if (f<0) return FALSE;

	item_flush();
        feed_flush2(TRUE);

#ifdef NETWARE
        strcpy(sendname,feed_fname(feeds[f].site,".snd"));
        strcpy(feedname,feed_fname(feeds[f].site,".fee"));
        strcpy(requename,feed_fname(feeds[f].site,".req"));
#else
        strcpy(sendname,feed_fname(feeds[f].site,".send"));
        strcpy(feedname,feed_fname(feeds[f].site,".feed"));
        strcpy(requename,feed_fname(feeds[f].site,".reque"));
#endif

        /* if x.send exists, exit */

        if (lib_exists(sendname)) goto append;
	
        /* rename x.feed to x.send */
        if (!lib_exists(feedname)) {
                dmsg("Nothing to feed just now %s\n",feedname);
                if (lib_exists(requename)) goto append;
                return TRUE;
        }
	imsg("Rename %s %s\n",feedname,sendname);
	if (feeds[f].isdfeed) {
		db_flush_quick(); /* could be quick maybe */
	}
        if (rename(feedname,sendname)!=0) {

                filemsg("Couldn't rename feed %s \n",sendname);
                return TRUE;
        }

append: /* append x.reque exit */
        if (lib_exists(requename)) {
                imsg("feed: Appending reque file %s %s \n",requename,sendname);
                if (!lib_append(requename,sendname)) {
                        emsg("Append failed in flushsite\n");
                } else {
                        if (remove(requename)!=0) filemsg("removed %s failed \n",requename); 
                }
        }
        return TRUE;
}       
int feed_flush2(int force)
{
        int i;
        
        if (!force) if (nout==0) return TRUE;
        for (i=0; i<nfeeds; i++) {
                if (fout[i]!=NULL) {
                        xfclose(fout[i]); fout[i] = NULL;
                        dmsg("feed: Close feed [%d] \n",i);
                }
                /* feed_open(i); don't reopen it so it can be renamed */
                /* Should only queue it if it has items in it. */
                if (feeds[i].nsent>0) {
                        feed_queue(i);
                }
                feeds[i].nsent=0;
        }
        nout = 0;
        return TRUE;
}
void feed_queue(int i)
{
        if (!config_true(C_SEND_LATER)) {
              if (feeds[i].builtin) {
                suck_queue(feeds[i].site,10);
                imsg("feed: Queued outgoing feed {%s}\n",feeds[i].site);
              }
              if (feeds[i].smtp) {
                suck_queue(feeds[i].site,11);
                imsg("feed: Queued outgoing mail {%s}\n",feeds[i].site);
              }
        }
}
void feed_addmail(void)
{
        char bf[BFSZ];
        if (!feed_smtp()) {
                strcpy(bf,"site mail");
                feed_add(bf);
                strcpy(bf,"smtp");
                feed_add(bf);
                feed_new();
        }
}
int feed_smtp(void)
{
        int i;
        for (i=0; i<nfeeds; i++) {
                if (feeds[i].smtp) return TRUE;
        }
        return FALSE;
}
char *feed_smtpname(void)
{
        int i;
        for (i=0; i<nfeeds; i++) {
                if (feeds[i].smtp) return feeds[i].site;
        }
        return "127.0.0.1";
}
int feed_local(char *groupnum, int ismod)
{
        int i;
        for (i=0; i<nfeeds; i++) {      
                if (ismod) {
                        if (feeds[i].smtp) break;
                } else {
                        if (feeds[i].posting) break;
                }
        }
        if (i==nfeeds) {
                if (ismod) {
                        imsg("Failed to find SMTP feed in newsfeeds.conf to send moderated post to\n");
                        return FALSE;
                }
                for (i=0; i<nfeeds; i++) {      
                        if (feeds[i].builtin) break;
                }
        }
        if (i==nfeeds) {
                imsg("No feed found for local post, probably ok\n");
                return FALSE;
        }               
        if (post_feed>0) if (!ismod) i = post_feed;
        return feed_send(i,"<local>",groupnum,0);
}
int feed_send(int f, char *msgid, char *groupnum, int nfails)
{
        int feed_retry=atoi(config_get(C_FEED_NRETRY));
        if (feed_retry==0) feed_retry = 5;
        if (nfails>feed_retry) {
                imsg("feed: dropping after %d attempts %s %s\n",feed_retry,msgid,groupnum);
                return TRUE;
        }
        feeds[f].nsent++;
        if (nout>200) feed_flush(); /* Increase to 200 */
        if (fout[f]==NULL) feed_open(f);
        if (fout[f]!=NULL) {
                if (nfails==0) fprintf(fout[f],"%s %s\n",msgid,groupnum);
                else fprintf(fout[f],"%s %s f:%d\n",msgid,groupnum,nfails);
        }
        if (!config_true(C_SEND_LATER)) if (feeds[f].smtp) feed_queue(f);

        nout++;
        return TRUE;
}
int feed_open(int f)
{
        char bf[1000];
#ifdef NETWARE
        strcpy(bf,feed_fname(feeds[f].site,".fee"));
#else
        strcpy(bf,feed_fname(feeds[f].site,".feed"));
#endif
        dmsg("feed: Open feed {%d} {%s} \n",f,bf);
        fout[f] = xfopen(bf,"a");
        if (fout[f]==NULL) {
                filemsg("fatal: feed: Feed failed to append to %s \n",bf);
                return FALSE;
        }
        return TRUE;
}

static char *uucp_file[MAX_FEEDS];
static int uucp_total[MAX_FEEDS];
static int uucp_last[MAX_FEEDS];
static int uucp_n[MAX_FEEDS];
char myeol[100];


void feed_flushuucp(void)
{
        int f;
        for (f=0; f<nfeeds; f++) {
                if (feeds[f].uucp) if (uucp_total[f]>0) uucp_next(f);
                if (fout[f]!=NULL) fflush(fout[f]);
        }
}
void feed_flushuucp2(void)
{
        int f;
	imsg("uucp_flush2 called\n");
        for (f=0; f<nfeeds; f++) {
		if (feeds[f].uucp_timer!=0) {
			if ((feeds[f].uucp_timer + uucp_last[f])>lib_gmtime()) {
				imsg("uucp_timer skipping feed %d %d seconds\n",f,(feeds[f].uucp_timer + uucp_last[f])-lib_gmtime());
				continue;
			}
		}
                if (feeds[f].uucp) if (uucp_total[f]>0) uucp_next(f);
                if (fout[f]!=NULL) fflush(fout[f]);
        }
}

void uucp_next(int f)
{
        int n;
        char *uudir = feeds[f].uucp_dir;
        static char uucp_curname[BFSZ];
        static char uucp_newname[BFSZ];

        uucp_buff_flush(f);

        n = uucp_nexti(f) - 1;
	uucp_last[f] = lib_gmtime();
        lprintf(uucp_curname,BFSZ-1,"%s/NEWS%04d.NEW",uudir,n);
	imsg("uucp_next rotating to next bag file %s\n",uucp_curname);
        lprintf(uucp_newname,BFSZ-1,"%s/NEWS%04d.BAG",uudir,n);
#ifdef VMS
        lprintf(uucp_curname,BFSZ-1,"%sNEWS%04d.NEW",uudir,n);
        lprintf(uucp_newname,BFSZ-1,"%sNEWS%04d.BAG",uudir,n);
#endif
        if (rename(uucp_curname,uucp_newname)!=0) {
		if (errno!=ENOENT) emsg("uucp rename failed %s\n",uucp_curname);
	}
        n++;
        lprintf(uucp_curname,BFSZ-1,"%s/NEWS%04d.NEW",uudir,n);
#ifdef VMS
        lprintf(uucp_curname,BFSZ-1,"%sNEWS%04d.NEW",uudir,n);
#endif
        if (uucp_file[f]==NULL) uucp_file[f] = mymalloc(200);
        strcpy(uucp_file[f],uucp_curname);
        uucp_total[f] = 0;
        uucp_n[f] = 0;
}
int uucp_nexti(int ff)
{
        char fname[BFSZ];
        char bf[BFSZ];
        int lcount=0;
        FILE *f;

        lprintf(fname,BFSZ-1,"%s/count.dat",feeds[ff].uucp_dir);
#ifdef VMS
        lprintf(fname,BFSZ-1,"%scount.dat",feeds[ff].uucp_dir);
#endif
        f = myfopen(fname,"r");
        if (f!=NULL) {
                fgets(bf,199,f);
                lcount = atoi(bf);
                fclosep(f);
        }
        f = myfopen(fname,"w");
        if (f==NULL) filemsg("Cannot write to uucp count file %s\n",fname);
        if (f!=NULL) {
                fprintf(f,"%d\n",lcount+1);
                fclosep(f);
        }
        return lcount;
}

int uucp_len(String *str)  /*  */
{
        char *s1 = str_s(str);
        char *s2;
        int len=0;
        int xlen=0;

        for (;s1!=NULL;) {
                s2 = strchr(s1,'\015');
                if (s2==NULL) break;
                len = (s2-s1+1);
                if (len>MAX_LINE-3) len = MAX_LINE-3;
                if (len==3) if ((*s1 == '.') && (s1[1] == '.')) len--;
                xlen += len;
                if (*(s2+1)!='\012') break; 
                s1 = s2 + 2;
        }
        return xlen;
}
void old_feed_uucp_write(int f, String *xitem)
{
        char *s1 = str_s(xitem);
        char *s2;
        char *mde;
        int len = uucp_len(xitem);
        FILE *fout;
        char bf[MAX_LINE];

        strcpy(myeol,"\n");

        if (uucp_n[f] >= feeds[f].uucp_n) uucp_next(f);
        if (uucp_total[f] > feeds[f].uucp_size) uucp_next(f);
        if (strlen(uucp_file[f])==0) uucp_next(f);

        mde = "ab";
        if (config_true(C_UUCP_CRLF)) mde = "a";
        fout = myfopen(uucp_file[f],mde);
        if (fout==NULL) {filemsg("UUCP Write failed %s\n",uucp_file[f]); return; }
        fprintf(fout,"#! rnews %d%s",len,myeol);
        uucp_total[f] += len;
        uucp_n[f] ++;
        feeds[f].nok++;
        for (;s1!=NULL;) {
                s2 = strchr(s1,'\015');
                if (s2==NULL) break;
                len = (s2-s1);
                if (len>MAX_LINE-3) len = MAX_LINE-3;
                strncpy(bf,s1,len);
                bf[len] = '\n';
                bf[len+1] = 0;
                if (strcmp(bf,"..\n")==0) strcpy(bf,".\n"); 
                if (fwrite(bf,1,len+1,fout)!=(size_t) len+1) {filemsg("Write failed %s\n",uucp_file[f]); break;}
                if (*(s2+1)!='\012') break; 
                s1 = s2 + 2;
        }
        fclosep(fout);
}
void feed_uucp_write(int f, String *xitem)
{
        char *s1 = str_s(xitem);
        char *s2;
        int len = uucp_len(xitem);
        char bf[MAX_LINE];

        if (uucp_n[f] >= feeds[f].uucp_n) uucp_next(f);
        if (uucp_total[f] > feeds[f].uucp_size) uucp_next(f);
        if (strlen(uucp_file[f])==0) uucp_next(f);

        lprintf(bf,MAX_LINE-1,"#! rnews %d\n",len);
        uucp_buff_add(f,bf,strlen(bf));
        uucp_total[f] += len;
        uucp_n[f] ++;
        feeds[f].nok++;
        for (;s1!=NULL;) {
                s2 = strchr(s1,'\015');
                if (s2==NULL) break;
                len = (s2-s1);
                if (len>MAX_LINE-3) len = MAX_LINE-3;
                strncpy(bf,s1,len);
                bf[len] = '\n';
                bf[len+1] = 0;
                len++;
                if (strcmp(bf,"..\n")==0) {strcpy(bf,".\n"); len = strlen(bf); }
                if (!uucp_buff_add(f,bf,len)) {feeds[f].nrejects++; break;}
                if (*(s2+1)!='\012') break; 
                s1 = s2 + 2;
        }
}
int uucp_buff_add(int f, char *bf, int len)
{
        int wanted;
        if (ubuf[f]==NULL) {
                ubuf[f] = mymalloc(UBFSZ+10);
                ubuf_len[f] = 0;
        }
        for (;len>0;) {
                wanted = UBFSZ-ubuf_len[f]-2;
                if (wanted>len) wanted = len;
                memcpy(ubuf[f]+ubuf_len[f],bf,wanted);
                ubuf_len[f] += wanted;
                if (ubuf_len[f] > (UBFSZ-1000)) {
                        if (!uucp_buff_flush(f)) return FALSE;;
                }
                len -= wanted;
                bf += wanted;
        }
        return TRUE;
}
int uucp_buff_flush(int f)
{
        char *mde;
        FILE *fout;
        if (ubuf[f]==NULL) {
                ubuf[f] = mymalloc(UBFSZ+10);
                ubuf_len[f] = 0;
        }
        if (ubuf_len[f]==0) return TRUE;
        mde = "ab";
        if (config_true(C_UUCP_CRLF)) mde = "a";
        fout = fopen(uucp_file[f],mde);
        if (fout==NULL) goto write_failed;
        if (fwrite(ubuf[f],1,ubuf_len[f],fout)!=(size_t) ubuf_len[f]) {fclosep(fout); goto write_failed;}
        fclosep(fout);
        ubuf_len[f] = 0;
        return TRUE;
write_failed:
        filemsg("UUCP Write failed %s\n",uucp_file[f]); 
        ubuf_len[f] = 0;
        return FALSE; 
}

static char wait_fname[400];
static char send_fname[400];
static char site_fname[400];
static char upto_fname[400];
static int fcur;
static FILE *fsend,*fupto;
static int nsent;
static int isposting,issmtp;
static int doagain = FALSE;
int feed_start(char *feedname)
{
        int i;
        char bf[400];

        if (fsend!=NULL) {
                dmsg("feed: Feed_start called when feed end had not been called\n");
                feed_end();
        }
        fcur = feed_find(feedname);
        if (fcur<0) {emsg("Nonexistent feed %s \n",feedname); return FALSE;}
        nsent = 0;

        feed_flush2(TRUE); /* Close the feed files */

        /* Don't retry more than once every 5 minutes */
        if ((lib_gmtime()-feeds[fcur].last_time) < 300) {
                imsg("Won't reopen link to {%s} for %d seconds as we just sent something recently\n",feedname,300-(lib_gmtime()-feeds[fcur].last_time));
                return FALSE;
        }
        feeds[fcur].last_time = lib_gmtime(); 
        isposting = feeds[fcur].posting;
        issmtp = feeds[fcur].smtp;

        if (issmtp) wmsg("feed: Starting SMTP-feed to {%s} \n",feedname);
        else        wmsg("feed: Starting NNTP-feed to {%s} \n",feedname);

        ncpy(wait_fname,feed_fname(feedname,".wat"),399);
#ifdef NETWARE
        ncpy(send_fname,feed_fname(feedname,".sen"),399);
        ncpy(site_fname,feed_fname(feedname,".fee"),399);
        ncpy(upto_fname,feed_fname(feedname,".upt"),399);
#else
        ncpy(send_fname,feed_fname(feedname,".sending"),399);
        ncpy(site_fname,feed_fname(feedname,".feed"),399);
        ncpy(upto_fname,feed_fname(feedname,".upto"),399);
#endif
        if (lib_exists(send_fname)) {doagain=TRUE; goto finish_last;}
        if (!lib_exists(site_fname)) {imsg("No feed file, nothing to send\n"); return FALSE;}
	imsg("Start Rename %s %s\n",site_fname,send_fname);
        if (rename(site_fname,send_fname)!=0) {filemsg("feed: Rename failed {%s}->{%s}\n",site_fname,send_fname); return FALSE;}
        feed_flush2(TRUE); /* Close and reopen files */
        remove(upto_fname); /* Clear the check point file */
finish_last:
        fsend = xfopen(send_fname,"r");
        if (fsend==NULL) {filemsg("feed: Can't open-read feeding file %s \n",send_fname); return FALSE;}
        fupto = xfopen(upto_fname,"r");
        if (fupto!=NULL) {
                fgets(bf,199,fupto);
                nsent = atoi(bf);
                for (i=0; i<nsent ; i++) {
                        fgets(bf,399,fsend); /* Throw away nsent lines of feed*/
                        dmsg("feed: Skipping lined {%s} \n",bf);
                }
                xfclose(fupto); fupto = NULL;
        }
        return TRUE;
}

void feed_del(int x);
void feed_del(int x)
{
        feed_flush2(TRUE); /* Close the feed files */
        emsg("Deleting files that have exceeded PENDING_MAX setting in newsfeeds.conf %d\n",feeds[x].pending_max);
        emsg("Deleting %s,.send,.sending\n",feed_fname(feeds[x].site,".feed"));
        remove(feed_fname(feeds[x].site,".sending"));
        remove(feed_fname(feeds[x].site,".send"));
        remove(feed_fname(feeds[x].site,".feed"));
        remove(feed_fname(feeds[x].site,".wat"));
}
int feed_filesz(int x)
{
        int total;
        feed_flush2(TRUE); /* Close the feed files */
        total = lib_filesize(feed_fname(feeds[x].site,".sending"));
        total += lib_filesize(feed_fname(feeds[x].site,".send"));
        total += lib_filesize(feed_fname(feeds[x].site,".feed"));
        total += lib_filesize(feed_fname(feeds[x].site,".wat"));
        return total;
}
/*
<3ovl3u$6th@status.gen.nz> nz.general:19399 f:0
msgid group:num failures
*/
void feed_stats(int ok)
{
        if (fcur<0) return;
        if (ok) feeds[fcur].nok++;
        else feeds[fcur].nrejects++;
}
void feed_stats_dup(void)
{
        if (fcur<0) return;
        feeds[fcur].ndup++;
}
static int fcount,fgrp,fnum;
static char fmsgid[BFSZ];
void feed_writeupto(void);
void feed_writeupto(void)
{
        dmsg("feed: Checkpoint at %d items sent \n",nsent);
        remove(upto_fname);
        fupto = xfopen(upto_fname,"w");
        if (fupto==NULL) {filemsg("feed: Can't open check file %s \n",upto_fname); return ;}
        if (nsent>0) fprintf(fupto,"%d\n",nsent-1);
        else fprintf(fupto,"%d\n",nsent);
        xfclose(fupto);
        fupto = NULL;
}
int feed_next(int *grp, int *num)
{
        char bf[400];
        char *s;
        if (nsent/10.0*10.0==nsent) {
                feed_writeupto();
        } 
next_line:
        if (fsend==NULL) {
          emsg("feed: fatal, feed_next should not be called here\n");
          return FALSE;
        }
        if (fgets(bf,399,fsend)==NULL) {
                feed_done();
                return FALSE;
        }
        nsent++;
        s = strtok(bf," \t\015\012"); if (s==NULL) goto next_line;/*loopsus*/
        ncpy(fmsgid,s,BFSZ-1);
        s = strtok(NULL,": \t\015\012");      if (s==NULL) goto next_line;
        if (strcmp(s,"#local")==0) {
                fgrp = -2;
        } else  fgrp = group_find(s); 
        s = strtok(NULL,":\015\012");      if (s==NULL) goto next_line;
        fnum = atoi(s); 
        
        fcount = 0;
        for (;s!=NULL;) {
                s = strtok(NULL,": \t\015\012");      /*Optional F: */
                if (s==NULL) break;
                if (*s=='F') {
                        s = strtok(NULL,": \t\015\012");      /*Optional F: */
                        if (s!=NULL) fcount = atoi(s);  
                        break;
                }
        }       

        *grp = fgrp; *num = fnum;
        dmsg("feed: next: %d %d \n",fgrp,fnum);
        return TRUE;
}
int feed_delete(void)
{
        if (isposting || issmtp) post_delete(fnum);
        return TRUE;
}
int feed_firsttime(void)
{
        if (fcount==0) return TRUE;
   return FALSE;
}
int feed_reque(int itfailed)
{
        char bf[400];
        if (itfailed) fcount++;
        if (fcount==0) fcount ++; /* So it gets recorded as 'requed' */
        if (fcount>100) { 
                emsg("feed: Can't get message thru after 100 attempts\n");
                emsg("feed: Throwing message away, sorry\n");
                return FALSE;
        }
        if (isposting || issmtp) 
                lprintf(bf,400-1,"#local:%d",fnum);
        else    lprintf(bf,400-1,"%s:%d",group_name(fgrp),fnum);
        feed_send(fcur,fmsgid,bf,fcount);
        return TRUE;
}
int feed_posting(void)
{
        return isposting;
}
int feed_byposting(void)
{
        if (npost>0) return TRUE;
        return FALSE;
}
int feed_done(void) 
{
        dmsg("feed: Feed_done called %p\n",fsend);
        if (fsend!=NULL) {xfclose(fsend); fsend = NULL;}
        if (remove(send_fname)!=0) imsg("Unable to delete (%s) %s\n",send_fname,strerror(errno));
	else imsg("feed_done, removed %s\n",send_fname);
        remove(upto_fname);
        if (doagain) feed_doit();
        doagain = FALSE;
        return TRUE;
}
int feed_end(void) 
{
        feed_writeupto();
        dmsg("feed: Feed_end called %p\n",fsend);
        if (fsend!=NULL) {xfclose(fsend); fsend = NULL;}
        return TRUE;
}
void feed_out_summary(void)
{
        static int last[1000];
        int total;
        char bf[800];
        char xtype[BFSZ];
        int i;
        int fz;

        for (i=1; i<nfeeds; i++) {
                strcpy(bf,"");
                if (feeds[i].uucp) continue;
                if (feeds[i].builtin) strcpy(bf,"Bultin ");
                if (feeds[i].posting) strcpy(bf,"Posting ");
                if (feeds[i].smtp) strcpy(bf,"SMTP ");
                if (feeds[i].live) strcpy(bf,"Live ");
                if (feeds[i].isstream) strcpy(bf,"LiveStrm ");
                strcpy(xtype,bf);

                fz = feed_filesz(i);
                if ((lib_gmtime()-feeds[i].start)<1) continue;
                total = feeds[i].nok + feeds[i].ndup +  feeds[i].nlost;
                if (total==last[i]) continue;
                last[i] = total;
                imsg("Feed out summary %s Posts=%d Rejects=%d Dup=%d Queued=%d Requed=%d File=%dM Posts=%.2f/s Total=%.2f/s Total=%.0fk Type=%s\n"
                        ,feeds[i].site
                        ,feeds[i].nok
                        ,feeds[i].nrejects
                        ,feeds[i].ndup
                        ,feeds[i].nlost,feeds[i].nreque
                        ,fz/1000000
                    ,(double) feeds[i].nok/(lib_gmtime()-feeds[i].start)
                    ,(double) (feeds[i].nok + feeds[i].nrejects + feeds[i].ndup + feeds[i].nreque) /(lib_gmtime()-feeds[i].start)
                    ,(double) feeds[i].chomps/10.0
                    ,xtype);
        }

}
String *feed_out(void)
{
        static String *xbody;
	FILE *f;
        char bf[800];
        char bf2[800];
	char *s;
        int i;
        int fz;
	Thout *th;

        if (xbody==NULL) xbody = str_new();
        str_clear(xbody);
        lprintf(bf,800-1,"Typ Posts Queued Rejects   Reque     Dup Post/s K/s  Tot MB File Del  Feed\r\n");
        str_addsn(xbody,bf);
        for (i=1; i<nfeeds; i++) {
                strcpy(bf,"  ");
                if (feeds[i].uucp) continue;
                if (feeds[i].builtin) strcpy(bf,"B ");
                if (feeds[i].posting) strcpy(bf,"P ");
                if (feeds[i].smtp) strcpy(bf,"M ");
                if (feeds[i].live) strcpy(bf,"L ");
                if (feeds[i].isstream) strcpy(bf,"S ");
                if (feeds[i].isdfeed) strcpy(bf,"D ");
                if (feeds[i].issendthread) strcpy(bf,"X ");
                
                fz = feed_filesz(i);
                if (fz>feeds[i].pending_max) {
                        feed_del(i);
                        feeds[i].deleted += fz;
                }
                fz = feed_filesz(i);

		

                  lprintf(bf2,800-1,"%7d %6d %7d %7d %7d "
                    ,feeds[i].nok,feeds[i].nlost
                    ,feeds[i].nrejects,feeds[i].nreque
                    ,feeds[i].ndup); strcat(bf,bf2);
/* Post/s Tot/s K/s */
                  lprintf(bf2,800-1,"%5.2f %5.2f %6d"
                    ,(double) feeds[i].nok/(lib_gmtime()-feeds[i].start)
                    ,(double) feeds[i].chomps/(lib_gmtime()-feeds[i].start)/10.0,feeds[i].chomps/10000);
                strcat(bf,bf2);
                  if ((fz>0)) {
                        lprintf(bf2,800-1,"%4dk ",fz/1000); 
                        if (fz>1000000) 
                        lprintf(bf2,800-1,"%4dM ",fz/1000000); 
                        ncat(bf,bf2,700);
                  } else strcat(bf,"      ");
                  if (feeds[i].deleted>0) {lprintf(bf2,200,"%3dM ",feeds[i].deleted/1000000); strcat(bf,bf2);}
                  else strcat(bf,"     ");
                  lprintf(bf+strlen(bf),BFSZ-1,"%s",feeds[i].site);              
                strcat(bf,"\015\012");
                str_addsn(xbody,bf);
		th = feeds[i].th;
		if (th!=NULL) {
			int w,n;
			n = 0;
			for (w=0; w<THMAX; w++) {
				if (th->next[w]!=NULL) n++;
			}

			lprintf(bf,BFSZ-1,"--Thread Out nok=%d ndup=%d nbad=%d nbusy=%d q=%d Speed %6gk/s \r\n  Last: %.60s\r\n",
				th->nok,th->ndup,th->nbad,th->nbusy,n,(double) th->ksent/(lib_gmtime()-th->start),th->reason);
	                str_addsn(xbody,bf);
		}
        }
        str_addsn(xbody,"\015\012");

	f = fopen(file_log("dfeed.status"),"r");
	if (f==NULL) return xbody;
	for (;!feof(f);) {
		if (fgets(bf,200,f)==NULL) break;
		s = strchr(bf,'\n'); if (s!=NULL) *s = 0;
		s = strchr(bf,'\r'); if (s!=NULL) *s = 0;
		strcat(bf,"\r\n");
		str_addsn(xbody,bf);
	}
	fclosep(f);
        return xbody;
}
String *feed_status(void)
{
        static String *xbody;
        char bf[800];
        char bf2[800];
        int i;
        int fz;

        if (xbody==NULL) xbody = str_new();
        str_clear(xbody);
        for (i=0; i<nfeeds; i++) {
                lprintf(bf,800-1,"Feed[%d]: %s ",i,feeds[i].site);
                if (feeds[i].onlyposts) strcat(bf,"OP,");
                if (feeds[i].posting) strcat(bf,"Posting ");
                if (feeds[i].builtin) strcat(bf,"Builtin ");
                if (feeds[i].smtp) strcat(bf,"SMTP ");
                if (feeds[i].builtin || feeds[i].smtp || feeds[i].live || feeds[i].uucp) {
                  lprintf(bf2,800-1,"Posts=%d %.2f/s Rej=%d Req=%d Dup=%d  R=%d:%d:%d Tot=%.2f/s %.2fk/s "
                    ,feeds[i].nok,(double) feeds[i].nok/(lib_gmtime()-feeds[i].start)
                    ,feeds[i].nrejects,feeds[i].nreque
                    ,feeds[i].ndup,feeds[i].nrecent
                    ,feeds[i].nlost,feeds[i].nspool
                    ,(double) (feeds[i].nok + feeds[i].nrejects + feeds[i].ndup + feeds[i].nreque) /(lib_gmtime()-feeds[i].start)
                    ,(double) feeds[i].chomps/(lib_gmtime()-feeds[i].start)/10.0            
                    );
                        strcat(bf,bf2);
                }
                fz = feed_filesz(i);
                if (fz>feeds[i].pending_max) {
                        feed_del(i);
                        feeds[i].deleted += fz;
                }
                fz = feed_filesz(i);
                if ((fz>0)) {lprintf(bf2,200,"file=%dk ",fz/1000); strcat(bf,bf2);}
                if (feeds[i].deleted>0) {lprintf(bf2,200,"del=%dMB ",feeds[i].deleted/1000000); strcat(bf,bf2);}
                if (feeds[i].isstream) {lprintf(bf2,200,"Strm "); strcat(bf,bf2);}
                if (feeds[i].groups!=NULL) {
                        ncpy(bf2,feeds[i].groups,60);
                        lprintf(bf+strlen(bf),BFSZ-1,"%s",bf2);
                }
                strcat(bf,"\015\012");
                str_addsn(xbody,bf);
        }
        lprintf(bf,800-1,"Live cache misses %d/%d\r\n",send_missed,send_total);
        str_addsn(xbody,bf);
        if (config_true(C_SPAM_STOP)) {
                if (totspam==0) totspam = 1;
                lprintf(bf,800-1,"Spam filter.dat(%d)=%d, body dup=%d, from=%d, subj=%d, accepted=%d %3.2g%% Lcl Rej %d\r\n"
                        ,spam_nfile,totrej,totdup,totfrom,totsubj,totspam,100.0-(totfrom+totdup+totrej)*100.0/(double)totspam,post_nsilent());
                str_addsn(xbody,bf);
        }
        str_addsn(xbody,"\015\012");
        return xbody;
}
/*------------------------------------------------------------*/
/*                Feed Pool routines (reading a feed back in  */
/*------------------------------------------------------------*/
enum {L_CLOSED,L_OPEN_STATUS,L_MODE_STATUS,L_IHAVE_STATUS
        ,L_TAKETHIS_STATUS,L_READY};
#define RING_SIZE 5
#define MAX_MSGID 100
#define POOL_SIZE 500
typedef struct FeedMsg {
        char msgid[MAX_MSGID]; 
        char gnum[100]; int isdud,nretry;
        String *msg;
} FEEDMSG;
int ring_size(void);

static FEEDMSG *chan_ring[MAX_CHAN];
static int chan_hangup[MAX_CHAN];
static int chan_ringbig[MAX_CHAN];
static int chan_feed[MAX_CHAN];
static int chan_state[MAX_CHAN];
static int chan_stream[MAX_CHAN];
static int chan_sent[MAX_CHAN];
static double chan_wanted[MAX_CHAN]; /* running average, if > .5 skip check */

static FEEDMSG *pool[MAX_FEEDS];
static int pool_n[MAX_FEEDS];
static int pool_upto[MAX_FEEDS];
static int pool_at[MAX_FEEDS]; /*The place we are upto in the .send file */

static double chan_hitrate[MAX_CHAN];


void feed_init(int ff)
{
        if (feeds[ff].uucp) {
                uucp_file[ff] = mymalloc(100);
                strcpy(uucp_file[ff],"");
        }
        if (!feeds[ff].live) return;
        if (pool[ff]==NULL) {
                pool[ff] = mymalloc(sizeof(FEEDMSG)*POOL_SIZE);
                memset(pool[ff],0,sizeof(FEEDMSG)*POOL_SIZE);
        }
}
void feed_pool_checkpoint(int ff) 
{
        FILE *f;
        char upto_fname[BFSZ];
        ncpy(upto_fname,feed_fname(feeds[ff].site,".at"),399);
        dmsg("pool: checkpoint %d at %d\n",ff,pool_at[ff]);
        if (pool_at[ff]<0) {
                if (pool_at[ff] == -1) {
                        if (remove(feed_fname(feeds[ff].site,".send"))==0) {
                                imsg("Removing completed send file {%s}\n",feed_fname(feeds[ff].site,".send"));
                        } else filemsg("Remove failed %s\n",feed_fname(feeds[ff].site,".send"));
                        remove(upto_fname);
                        pool_at[ff] = -2;
                }
        } else {
	        remove(upto_fname);
                f = fopen(upto_fname,"w");
                if (f==NULL) {filemsg("Unable to write {%s}\n",upto_fname); return;}
                fprintf(f,"%d\n",pool_at[ff]);
                fclosep(f);
        }
}
void feed_pool_fill(int ff) /* open &or rename and find position and read 100 in */
{
        FEEDMSG *p;
        static int next_time[MAX_FEEDS];
	static int wait_time[MAX_FEEDS];
	static char send_fname[BFSZ];
	static char wait_fname[BFSZ];
	static char site_fname[BFSZ];
	static char upto_fname[BFSZ];
        char bf[BFSZ];
        FILE *fin;
        int i;
        char *s;

        dmsg("feed: pool_fill[%d] %d\n",ff,feeds[ff].live_only);
        if (feeds[ff].live_only) return; /* Don't send backlog */
        if (lib_gmtime() < next_time[ff]) return;
        if (pool_upto[ff] < pool_n[ff]) return; /* We have some already */ 

        feed_flush2(TRUE); /* Close the feed files */
        ncpy(wait_fname,feed_fname(feeds[ff].site,".wat"),399);
        ncpy(send_fname,feed_fname(feeds[ff].site,".send"),399);
        ncpy(site_fname,feed_fname(feeds[ff].site,".feed"),399);
        ncpy(upto_fname,feed_fname(feeds[ff].site,".at"),399);
        if (lib_exists(send_fname)) {doagain=TRUE; goto finish_last;}
	if (lib_exists(wait_fname)) goto send_wait;
        if (!lib_exists(site_fname)) {
                next_time[ff] = lib_gmtime() + 100; 
                imsg("pool: No feed file {%s} , nothing to send, try again 180 seconds\n",site_fname); 
                return ;
        }
	if (feeds[ff].xdelay>0) {
send_wait:	if (lib_gmtime() < wait_time[ff]) return;
		wait_time[ff] = lib_gmtime() + feeds[ff].xdelay;
		if (lib_exists(wait_fname)) {
			if (rename(wait_fname,send_fname)!=0) {filemsg("feed: Rename failed {%s}->{%s}\n",site_fname,send_fname); return ;}
			imsg("Renamed delay file to send file (%s)\n",send_fname);
		}
		if (rename(site_fname,wait_fname)!=0) {filemsg("feed: Rename failed {%s}->{%s}\n",site_fname,send_fname); return ;}
		imsg("Renamed feed file to delay file (%s)\n",wait_fname);
	} else {
		if (rename(site_fname,send_fname)!=0) {filemsg("feed: Rename failed {%s}->{%s}\n",site_fname,send_fname); return ;}
		imsg("pool_fill rename %s %s\n",site_fname,send_fname);
	}
        feed_flush2(TRUE); /* Close and reopen files */
        remove(upto_fname); /* Clear the check point file */
finish_last:
        fin = xfopen(send_fname,"r");
        if (fin==NULL) {filemsg("feed: Can't open-read feeding file %s \n",send_fname); return ;}
        dmsg("pool: Opened {%s} \n",send_fname);
        if (pool_at[ff]>=0) {
                fseek(fin,pool_at[ff],SEEK_SET);
        } else {
                fupto = xfopen(upto_fname,"r"); 
                if (fupto!=NULL) {
                        fgets(bf,199,fupto); 
                        fseek(fin,atoi(bf),SEEK_SET);
                        xfclose(fupto); fupto = NULL;
                }
        }
        p = pool[ff];
        for (i=0;(!feof(fin)) && (i<POOL_SIZE);i++) {
                if (fgets(bf,200,fin)==NULL) continue;
                bf[200] = 0;
                s = strtok(bf," \t\015\012"); if (s==NULL) continue; 
                if (strlen(s)>90) {
                        emsg("Rejecting bad message id %s\n",bf);
                        continue;
                }
                if (!mid_isok(s)) {
                        emsg("Rejecting bad message id %s\n",s);
                        continue;
                }
                ncpy(p[i].msgid,s,99);
                s = strtok(NULL," \t\015\012"); if (s==NULL) continue; 
                ncpy(p[i].gnum,s,99);
                p[i].nretry = 0;
                s = strtok(NULL," \t\015\012"); 
                if (s!=NULL) {
                        p[i].nretry = atoi(s+2);
                        dmsg("live: retry = %d\n",p[i].nretry);
                }
                if (p[i].nretry>5) {
                        imsg("Dropping item after 5 attempts to send %s %s\n",p[i].msgid,p[i].gnum);
                        continue;
                }
                pool_n[ff] = i+1;
                dmsg("live: Pool Add [%d]  {%s} {%s} r=%d\n",i,p[i].msgid,p[i].gnum,p[i].nretry);
        }
        if (feof(fin)) {
                pool_at[ff] = -1;
        } else pool_at[ff] = ftell(fin);
        fclosep(fin);
        if (pool_at[ff]<0) feed_pool_checkpoint(ff); 
        pool_upto[ff] = 0;
}
void feed_pool_get(int ff,char *msgid,char *grpnum,int *nretry)
{
        FEEDMSG *p;
        p = pool[ff];
        if (pool_upto[ff]<1) {
                emsg("feed_pool_get() in wrong state %d\n",pool_upto[ff]);
                strcpy(msgid,"");
                return;
        }
        ncpy(msgid,p[pool_upto[ff]-1].msgid,100);
        ncpy(grpnum,p[pool_upto[ff]-1].gnum,100);
        *nretry = p[pool_upto[ff]-1].nretry;
}
int feed_pool_next(int f) /* Get any new items for this feed */
{
        pool_upto[f]++;
        dmsg("live: feed_pool_next [%d] %d %d\n",f,pool_upto[f],pool_n[f]);
        if (pool_upto[f] > pool_n[f]) {
                if (pool_n[f]>0) feed_pool_checkpoint(f);
                feed_pool_fill(f);
        }
        if (pool_upto[f] > pool_n[f]) return -1;
        return pool_upto[f]-1;
}
int feed_pool_flush(void)
{
        char msgid[BFSZ];
        char grpnum[BFSZ];
        int nretry;
	int f;
	for (f=0; f<nfeeds; f++) if (feeds[f].live) {
		for (;pool_upto[f]<pool_n[f];) {
			pool_upto[f]++;
			feed_pool_get(f,msgid,grpnum,&nretry);
		        dmsg("live: pool_flushing [%d] {%s} {%s} \n",f,msgid,grpnum);
		        feed_send(f,msgid,grpnum,nretry+1);
		}
	}
	return TRUE;
}
void feed_pool_reque(int chan, int r)
{
        FEEDMSG *f = chan_ring[chan];
        f = f+r;

        dmsg("feed: Reque {%s} {%s} retry=%d\n",f->gnum,f->msgid,f->nretry);
        feed_send(chan_feed[chan],f->msgid,f->gnum,f->nretry+1);
}
int live_ring_next(int chan)
{
        FEEDMSG *f = chan_ring[chan];
        int i;
        for (i=0; i<ring_size(); i++) {
                if (strlen(f[i].msgid)==0) return i;
        }       
        return -1;
}
int live_ring_find(int chan, char *instr, int len)
{
        FEEDMSG *f = chan_ring[chan];
        char bf[BFSZ];
        int i;
        char *s;
        char *s2;
        
        if (len>200) len = 200;
        ncpy(bf,instr,len);

        s = strtok(bf," \t\015\012");
        if (s!=NULL) s = strtok(NULL," \t\015\012");
        if (s==NULL) goto failed;
        s2 = strchr(s,'>');
        if (s2!=NULL) *(s2+1) = 0; 
        for (i=0; i<ring_size(); i++) {
                if (strcmp(f[i].msgid,s)==0) return i;
        }       
        if (s!=NULL) {
          for (i=0; i<ring_size(); i++) {
                if (strcmpnc(f[i].msgid,s)==0) {
                        imsg("live: remote end returned message id in lower case {%s} \n",bf);
                        return i;
                }
          }
        }       
failed:
        emsg("%s Couldn't find ring message %s\n",linknamec(chan),bf);
/*
          for (i=0; i<RING_SIZE; i++) {
                wmsg("ring [%d] = %s\n",i,f[i].msgid);
          }
  */
    return -1;
}
void str_iffree(String **x)
{
        if (*x == NULL) return;
        str_free(*x);
        *x = NULL;
}
int live_ring_add(int chan,char *msgid,char *groupnum, int nretry, String *xitem)
{
        FEEDMSG *m = chan_ring[chan];
        int r = live_ring_next(chan);
        if ((r<0) || (r>=ring_size())) {
                emsg("%s Live ring add failed, fatal error\n",linknamec(chan)); return 0;
        }
        dmsg("live: live_ring_add: chan [%d] ring[%d] %s %s\n",chan,r,groupnum,msgid);
        m = &m[r];
        ncpy(m->msgid,msgid,MAX_MSGID-1);
        ncpy(m->gnum,groupnum,99);
        m->nretry = nretry;
        str_iffree(&m->msg);
        if (xitem!=NULL) if (str_len(xitem)<big_maxsz()) {
                m->msg = str_new();
                str_copy(m->msg,xitem);
        }
        return r;
}
void ring_free(int chan, int r)
{
        FEEDMSG *f = chan_ring[chan];
        f = &f[r];
        strcpy(f->msgid,"");
        f->isdud = FALSE;
        str_iffree(&f->msg);
        dmsg("live: ring_free: chan[%d] ring [%d] \n",chan,r);
}
char *live_ring_msgid(int chan, int r)
{
        FEEDMSG *f = chan_ring[chan];
        f = &f[r];
        return f->msgid;
}
/*-----------------------------------------------------*/
/*                Live link code                       */
/*-----------------------------------------------------*/

/* Iniitialize and read in these correctly
        int uucp,uucp_size,uucp_n;
        char *uucp_dir;
        char *uucp_name; 
        int live;
        int live_chan[MAX_LIVECHAN];
        int live_nchan;
*/
/* Que item for sending directly if enough memory, else write to feed file.*/
void live_feed(int ff,char *msgid,char *groupnum,String *xitem)
{
        Feed *f = &feeds[ff];   
        int i;
        int c;
	if (f->xdelay==0) {
        for (i=0; i<f->live_nchan; i++) {
                c = f->live_chan[i];
                if (c>=0) {
                        if (c>=0) if (!chan_isopen(c)) {live_close(c); continue;}

                        dmsg("live: [%d] state %d, outsz %d, getbusy %d, ring_next %d \n",
                                c,chan_state[c],nntp_outsz(c),nntp_getbusy(c),live_ring_next(c));
                        if (chan_state[c]!=L_TAKETHIS_STATUS) if (chan_state[c]!=L_READY) continue;
                        if (nntp_outsz(c)>max_strbf()) continue; /* Too much data waiting */
                        if (nntp_getbusy(c)) continue; /* In middle of sending large item*/
                        if (live_ring_next(c)<0) continue; /* Ring buffer full*/
                        dmsg("live: Live_send from live_feed {%s} \n",msgid);
                        live_send(c,msgid,groupnum,xitem,0);
                        f->nrecent += 10;
                        if (f->nrecent>100) f->nrecent = 100;
                        return;
                }
        }
	}
        f->nlost++;
        f->nrecent -= 50;
	if (f->xdelay!=0) f->nrecent = 100;
	if (f->nrecent<0) f->nrecent = 0;
        if (f->live_only) {
        } else {
                dmsg("live: Link busy, so queuing item to feed file %s %s\n",groupnum,msgid);
                feed_send(ff,msgid,groupnum,0);
        }
}


int live_getring(int chan, int r, int *grp, int *num, char *gnum)
{
        FEEDMSG *f = chan_ring[chan];
        char bf[BFSZ];  
        char *s;

        if ((r<0) || (r>=ring_size())) {emsg("%s Live getring failed %d %d \n",linknamec(chan),chan,r); return FALSE;}
        f = &f[r];
        ncpy(gnum,f->gnum,BFSZ-1);
        ncpy(bf,f->gnum,BFSZ-1);
        s = strchr(bf,':');
        if (s==NULL) {emsg("%s Live getring colon missing %d %d {%s} \n",linknamec(chan),chan,r,bf); return FALSE;}
        *s = 0;
        *grp = group_find(bf);
        *num = atoi(s+1);
        return TRUE;
}
String *live_getringmsg(int chan, int r)
{
        FEEDMSG *f = chan_ring[chan];
        if ((r<0) || (r>=ring_size())) {emsg("%s Live getring failed %d %d \n",linknamec(chan),chan,r); return NULL;}
        f = &f[r];
        return f->msg;
}
void live_freeringmsg(int chan, int r)
{
        FEEDMSG *f = chan_ring[chan];
        if ((r<0) || (r>=ring_size())) {emsg("%s Live getring failed %d %d \n",linknamec(chan),chan,r); return ;}
        f = &f[r];
        str_iffree(&f->msg);
}
void live_idlex(void); /* Called in main loop */
void live_idle(void) /* Called in main loop */
{
	stat_start(X_LIVEIDLE);
        live_idlex();
	stat_start(X_LIVEIDLE);
}
int live_maxidle(void);
int live_maxidle(void)
{
        int x = atoi(config_get(C_LIVE_TIMEOUT));
        if (x==0) x = 500;
        return x;
}
void live_sendmore(int chan)
{
        /* If anything waiting try and fill up queue */
        Feed *f;
        int start_time = lib_msec();
        int start_clock = stats_clock();
	
	
	if (chan<0) return;
        if (chan_feed[chan]<0) goto ret;
        f = &feeds[chan_feed[chan]];
        if (nntp_outsz(chan)>50000) goto ret;
        if (f->xdelay==0) if (f->nrecent<50) goto ret;
        live_send_spooled(chan);
ret:;
/*        stats_add("live_sendmore",start_time,start_clock);*/
}
char *feed_to(Feed *f)
{
	if (f->to!=NULL) return f->to;
	return f->site;
}
char *feed_toname(char *sitename)
{
	int ff;
	for (ff=0; ff<nfeeds; ff++) {
		if (strcmp(feeds[ff].site,sitename)==0) return feed_to(&feeds[ff]);
	}
	return sitename;
}
void live_idlex(void) /* Called in main loop */
{
        int n;
        Feed *f;
        int ff;
        int c,i;
        static int lastsec;
        int z;

        if (lastsec==lib_gmtime()) return;

        if ((lib_gmtime() - idle_lastt) > 100) {
                idle_lastt = lib_gmtime();
                for (ff=0; ff<nfeeds; ff++) {
                        if (feeds[ff].live) {
                                feed_pool_fill(ff);
                                feeds[ff].nretry = 0;
                        }
                }               
                
                for (ff=0; ff<nfeeds; ff++) {
                        f = &feeds[ff];
                        if (!f->live) continue;
                        if (f->live_nchan<1) f->live_nchan = 2;
                        if (f->sleeping>lib_gmtime()) {
                                imsg("Feed %s sleeping for %d more seconds\n",f->site,f->sleeping - lib_gmtime());
                                continue;
                        }
                        for (i=0; i<f->live_nchan; i++) {
                                if (f->nrecent<100) f->nrecent += 30;
                                if (f->live_chan[i]>=0) {
                                        c = f->live_chan[i];
                                        if ((chan_state[c]==L_TAKETHIS_STATUS) || (chan_state[c]==L_IHAVE_STATUS)) {
                                                if (chan_hangup[c]+live_maxidle()<lib_gmtime()) {
                                                        wmsg("%s Live no response for %d seconds, closing %d \n",linknamec(c),lib_gmtime()-chan_hangup[c],c); 
                                                        live_close(c);
                                                } else dmsg("live: %s Live idle for %d seconds %d \n",linknamec(c),lib_gmtime()-chan_hangup[c],c); 
                                        }
                                        continue;
                                }
                                dmsg("live: Do we need to open feed [%d] chan [%d] %d \n",ff,i,f->live_chan[i]);
                                c = f->live_chan[i] = chan_open_bind(feed_to(f),f->bind);
                                if (c>=0) {
                                 chan_setstate(c,C_LIVE);
                                 chan_hitrate[c] = 0.89; /* So it will stream soon*/
                                 chan_hangup[c] = lib_gmtime(); 

                                 f->nrecent = 100;
                                 if (f->islog) chan_setlog(c,TRUE);
                                 chan_state[c] = L_OPEN_STATUS;
                                 chan_feed[c] = ff;
                                 if (chan_ring[c]==NULL)        {
                                        chan_ring[c] = mymalloc(sizeof(FEEDMSG)*ring_size());
                                        memset(chan_ring[c],0,sizeof(FEEDMSG)*ring_size());
                                 }
                                }
                        }
                }

        }


        for (ff=0; ff<nfeeds; ff++) {
                if (feeds[ff].live) {
                        f = &feeds[ff];
                        for (i=0; i<f->live_nchan; i++) {
                                c = f->live_chan[i];
                                if (c>=0) if (!chan_isopen(c)) {live_close(c);}
                        }

                        n = f->live_nchan-1; /* Leave one free for live link send */
                        if (f->live_nchan>0) if (n==0) n = 1;
                        if (f->nrecent>50) 
                          for (i=0; i<n; i++) {
                                c = f->live_chan[i];
                                if (c>=0) if (!chan_isopen(c)) {live_close(c); continue;}
                                if (c>=0) {

                        dmsg("live: feed[%d] chan[%d] state %d, outsz %d, getbusy %d, ring_next %d \n",
                                ff,c,chan_state[c],nntp_outsz(c),nntp_getbusy(c),live_ring_next(c));

                                 if (chan_state[c]!=L_TAKETHIS_STATUS) if (chan_state[c]!=L_READY) continue;
 dmsg("live: %d\n",L_READY);
                                 if (nntp_outsz(c)>50000) {dmsg("live: Channel outsz too big [%d]\n",c); continue;} /**/
                                 if (nntp_getbusy(c)) {dmsg("live: Channel busy %d\n",c); continue;} /* In middle of sending large item*/
                                 if (live_ring_next(c)<0) {dmsg("live: Ring buffer full [%d]\n",c); continue;} /* Ring buffer full*/
                                 /* Here we have a free channel, so send something */
 dmsg("live: send spooled\n");
                                 live_send_spooled(c);
                                 if (!config_true(C_LIVE_SPOOL_SLOW)) {
                                  if (f->isstream) {
                                         for (z=0;live_ring_next(c)>=0 && z<6;z++) {
                                                 if (!live_send_spooled(c)) break;
                                         }
                                  }
                                 }
                                }
                        }
                }
        }
        lastsec = lib_gmtime();
}
int live_send_spooled(int c)
{
        char msgid[BFSZ];
        char grpnum[BFSZ];
        int r,nretry;
        if ((r = feed_pool_next(chan_feed[c]))<0) return FALSE;
        feeds[chan_feed[c]].nspool++;
        feed_pool_get(chan_feed[c],msgid,grpnum,&nretry);
        dmsg("live: Send Spooled [%d] {%s} {%s} pool=%d \n",c,msgid,grpnum,r);
        live_send(c,msgid,grpnum,NULL,nretry);
        return TRUE;
}
int live_find_fchan(int ff, int chan)
{       
        Feed *f;
        int i;
        f = &feeds[ff];
        for (i=0; i< f->live_nchan;i++) {
                if (chan==f->live_chan[i]) return i;
        }
        dmsg("live: feed chan not found, feed %d, chan %d\n",ff,chan);
        return -1;
}
void live_close(int chan)
{
        int ff;
        int i;
        Feed *f;
        FEEDMSG *ring = chan_ring[chan];

        chan_close(chan);
        ff = chan_feed[chan];
        f = &feeds[ff];
        dmsg("Live: Lost connection to [%d] {%s}\n",chan,f->site);
        i  = live_find_fchan(ff,chan);
        if (i>=0) f->live_chan[i] = -1; 
        chan_state[chan] = L_OPEN_STATUS;
        /* Clear the ring buffer, treat them all as 'try agains' */
        if (chan_ring[chan] == NULL) return;
        for (i=0; i<ring_size(); i++) {
                if (strlen(ring[i].msgid)!=0) live_reque(chan,i);
        }       
}
char *linknamec(int chan)
{
        static char bf[BFSZ];
        if (chan<0 || chan>=MAX_CHAN) return "bad channel number";
        if (chan_feed[chan]<0) return "bad channel feed number";
        if (chan_feed[chan]>200) return "bad channel feed number";
        if (feeds[chan_feed[chan]].site==NULL) return "site name is null";
        lprintf(bf,BFSZ-1,"c=%d %s",chan,feeds[chan_feed[chan]].site);
        return bf;
}
void live_read(int chan, char *bf, int sz)
{
        int start_time = lib_msec();
        int start_clock = stats_clock();
        if (chan_ring[chan]==NULL) {
                chan_ring[chan] = mymalloc(sizeof(FEEDMSG)*ring_size());
                memset(chan_ring[chan],0,sizeof(FEEDMSG)*ring_size());
        }

        switch (chan_state[chan]) {
          case L_OPEN_STATUS:
                if (atoi(bf)<400) {
                        chan_write_s(chan,"MODE STREAM\015\012"); /* Change to  MODE STREAM */
                        chan_state[chan] = L_MODE_STATUS;
                } else {
                        wmsg("Live (%s) Open failed %s",linknamec(chan),str_trunc(bf,sz));
                        live_close(chan);
                }
                break;
          case L_MODE_STATUS:
                live_mode_status(chan,bf,sz);
                break;
          case L_IHAVE_STATUS:
                live_ihave_status(chan,bf,sz);
                break;
          case L_TAKETHIS_STATUS:               
                live_takethis_status(chan,bf,sz);
                break;
        }
}
void live_mode_status(int chan,char *bf,int sz)
{
        chan_stream[chan] = FALSE;
        if (atoi(bf)==203) chan_stream[chan] = TRUE;
        if (!config_true(C_LIVE_STREAM)) chan_stream[chan] = FALSE;
        if (chan_feed[chan]>0) {
                if (!feeds[chan_feed[chan]].isxreplic) {
                  feeds[chan_feed[chan]].isstream = chan_stream[chan];
                } else {
                  feeds[chan_feed[chan]].isstream = FALSE;
                }
                chan_stream[chan] = feeds[chan_feed[chan]].isstream;
        } else imsg("No feed for chan %d\n",chan);
        if (chan_stream[chan]) {
                if (sz<200) imsg("live: (%s) streaming %s\n",linknamec(chan),bf);
        } else {
                if (sz<200) imsg("live: (%s) using basic ihave %s\n",linknamec(chan),bf);
        }
        chan_state[chan] = L_READY;
}
/*
   235 article transferred ok
   335 send article to be transferred.  End with <CR-LF>.<CR-LF>
  435 article not wanted - do not send it
   436 transfer failed - try again later
  437 article rejected - do not try again
*/
void check_headout(char *head, int len, char *reason);

void live_ihave_status(int chan,char *bf,int sz)
{
        int status = atoi(bf);
        if (status==335) { /* Send article now */
                live_start_send(chan,0);
                return;
        }
        if (status==435) { /* He's already got it */ 
                dmsg("live: duplicate %s\n",nocrlf(bf));
                live_dup(chan,0);
                chan_state[chan] = L_READY;
                live_sendmore(chan);
                return;
        }
        if (status==437) { /* He didn't want it */
                dmsg("live: reject %s\n",nocrlf(bf));
                live_reject(chan,0);
                chan_state[chan] = L_READY;
                return;
        }
        if ((status==436) || (status==431)) { /* Try again later  */            dmsg("live: retry %s\n",nocrlf(bf));
                live_reque(chan,0);
                chan_state[chan] = L_READY;
                live_trylater(chan);                    
                return;
        }
        if (status==235) { /* Article recieved ok */
                dmsg("live: ok %s\n",nocrlf(bf));
                chan_state[chan] = L_READY;
                live_ok(chan,0);
                live_sendmore(chan);
                return;
        } 
        if (sz<200) 
                emsg("%s Unknown status returned %s, assuming the message was rejected",linknamec(chan),str_trunc(bf,sz));
        live_reject(chan,0); 
        wmsg("Closing channel to resync link %d\n",chan);       
        live_close(chan);
}
void live_reque(int chan, int r)                     
{
        FEEDMSG *f = chan_ring[chan];
        f = &f[r];
        if (f->isdud) {live_reject(chan,r); f->isdud = FALSE; return;}
        feed_pool_reque(chan,r);
        if (chan_ringbig[chan]==r) { big_free(chan); chan_ringbig[chan] = -1; }
        ring_free(chan,r);      
        feeds[chan_feed[chan]].nreque++;
}
int ring_size(void)
{
	static int n;
	if (n>0) return n;
	n = atoi(config_get(C_RING_SIZE));
	if (n==0) n = RING_SIZE;
	return n;
}
void live_reject_later(int chan, int r)
{
        FEEDMSG *f = chan_ring[chan];
        if ((r<0) || (r>=ring_size())) r = 0;
        f = &f[r];
        f->isdud = TRUE;
}
void live_trylater(int chan)
{
        int delay;
        feeds[chan_feed[chan]].nretry++;
        if (feeds[chan_feed[chan]].nretry>60) {
                delay = atoi(config_get(C_LIVE_DELAY));
                if (delay==0) delay = 3600;
                imsg("Got more than 20 try later messages so wont retry for %d seconds (live_delay)\n",delay);
                live_close(chan);
                feeds[chan_feed[chan]].sleeping = lib_gmtime()+delay;
        }
}
void live_reject(int chan, int r)
{
        if (chan_ringbig[chan]==r) { big_free(chan); chan_ringbig[chan] = -1; }
        ring_free(chan,r);      
        feeds[chan_feed[chan]].nrejects++;
}
void live_dup(int chan, int r)
{
        if (chan_ringbig[chan]==r) { big_free(chan); chan_ringbig[chan] = -1; }
        ring_free(chan,r);      
        feeds[chan_feed[chan]].ndup++;
}
void live_ok(int chan, int r)
{
        feeds[chan_feed[chan]].nretry = 0;

        if (chan_ringbig[chan]==r) { big_free(chan); chan_ringbig[chan] = -1; }
        ring_free(chan,r);      
        feeds[chan_feed[chan]].nok++;
}
int chan_isnocheck(int chan)
{
        int f = chan_feed[chan];
        if (f<0) return FALSE;
        return feeds[f].live_nocheck;
}
void live_send(int chan, char *msgid, char *groupnum, String *xitem,int nretry)
{
        char bf[8000];
        int grp,num;
        char *s;
        char bigbf[8000];
        int r;
        int f = chan_feed[chan];
        grp = -1;
        if (chan_hitrate[chan]<0.000001) chan_hitrate[chan] = 0;       
        r = live_ring_add(chan,msgid,groupnum,nretry,xitem);
        chan_sent[chan] = 0; 

        if (chan_stream[chan]) {
                chan_state[chan] = L_TAKETHIS_STATUS;
                chan_sethitrate(chan,chan_hitrate[chan]);
                if ((chan_hitrate[chan]>.9) || chan_isnocheck(chan)) {
                        lprintf(bf,8000-1,"TAKETHIS %s\015\012",msgid);
                        chan_write_s(chan,bf);
                        live_start_send(chan,r);
                } else {
                        lprintf(bf,8000-1,"CHECK %s\015\012",msgid);
                        chan_write_s(chan,bf);
                }
        } else {
                /* xref: host group:n group:n ...*/
          if (feeds[f].isxreplic) {
                if (xitem==NULL) {
                        ncpy(bf,groupnum,BFSZ-1);
                        s = strchr(bf,':'); if (s==NULL) goto giveup;
                        *s = 0; s++;
                        grp = group_find(bf); num = atoi(s);
                        if (grp<0) goto giveup;
                        xitem = item_head_cheat(grp,num,groupnum);
                        if (xitem==NULL) goto giveup;
                }
                ncpy(bigbf,get_ahead(xitem,"Xref:"),7999-100);
                if (strlen(bigbf)==0) {
                        if (grp<0) {
                                ncpy(bf,groupnum,BFSZ-1);
                                s = strchr(bf,':'); if (s==NULL) goto giveup;
                                *s = 0; s++;
                                grp = group_find(bf); num = atoi(s);
                                if (grp<0) goto giveup;
                        }
                        lprintf(bf,8000-1,"XREPLIC %s:%d\015\012",group_name(grp),num);
                        goto doit;
                }
                s = strchr(bigbf,' ');
                if (s==NULL) {emsg("fatal feed failure\n"); goto giveup;}
                s++;
                for (;*s!=0;s++) {
                        if (*s == ' ') *s = ',';
                }
                s = strchr(bigbf,' ');
                if (s==NULL) {emsg("fatal feed failure\n"); goto giveup;}
                s++;
                lprintf(bf,8000-1,"XREPLIC %s\015\012",s);
          } else {
                lprintf(bf,8000-1,"IHAVE %s\015\012",msgid);
          }
doit:     chan_write_s(chan,bf);
          chan_state[chan] = L_IHAVE_STATUS;
        }
        return;
giveup: 
        live_reject(chan,0); /* pretend it was rejected */
        chan_state[chan] = L_READY;
}
void chan_savelog(int x);
void live_takethis_status(int chan, char *bf, int sz)
{
        int start_time = lib_msec();
        int start_clock = stats_clock();

        int status = atoi(bf);
        int r = live_ring_find(chan,bf,sz);
        if ((r<0) || (r>=ring_size())) {
                if ((status==431) || (status==436)) {
                        feeds[chan_feed[chan]].nretry = 100;
                        live_trylater(chan);
                        return;
                }
                emsg("%s Got status for unknown message, closing channel %s\n",linknamec(chan),str_trunc(bf,sz));
                chan_savelog(chan); /* save the trace of that channel for debugging */ 
                live_close(chan);
                return;
        }
        switch (status) {
          case 238:  /* Haven;t got it,  send the article */
                chan_hitrate[chan] =  chan_hitrate[chan]*.99 + .01;
                lprintf(bf,800-1,"TAKETHIS %s\015\012",live_ring_msgid(chan,r));
                chan_write_s(chan,bf);
                live_start_send(chan,r);
                break;
          case 436:  /* Try again later */
          case 431:  /* Try again later */
                dmsg("live: later %s\n",nocrlf(bf));
                live_reque(chan,r);
                live_trylater(chan);                    
                break;
          case 438:  /* Already have it, forget it */
                dmsg("live: duplicate %s\n",nocrlf(bf));
                chan_hitrate[chan] =  chan_hitrate[chan]*.99;
                live_dup(chan,r);
                live_sendmore(chan);
                break;
          case 239:  /* Article received ok */
                dmsg("live: ok %s\n",nocrlf(bf));
                chan_hitrate[chan] =  chan_hitrate[chan]*.99 + .01;
                live_ok(chan,r);
                live_sendmore(chan);
                break;
          case 439:  /* Article rejected, don't retry  */
                dmsg("live: reject %s\n",nocrlf(bf));
                chan_hitrate[chan] =  chan_hitrate[chan]*.99;
                live_reject(chan,r);
                break;
          default:
                /* SHOULD TRUNCATE ERROR MESSAGES */
                emsg("%s Takethis status unknown %s\n",linknamec(chan),str_trunc(bf,sz));
                imsg("Closing connection to resync\n");
                chan_savelog(chan); /* save the trace of that channel for debugging */ 
                live_close(chan);
                break;  
        }
}
void live_start_send(int chan, int r)
{
        String *head,*body;
        char groupnum[BFSZ];
        int grp,num;
        String *msg;
        int f;
        int start_time = lib_msec();
        int start_clock = stats_clock();

	grp = num = 0;
        dmsg("live: start send [%d] ring %d \n",chan,r);
        chan_sent[chan] = 0; 
        chan_hangup[chan] = lib_gmtime(); 
        send_total++;
        msg = live_getringmsg(chan,r);
        if (msg==NULL) {
                send_missed++;
                live_getring(chan,r,&grp,&num,groupnum);
                head = item_head_cheat(grp,num,groupnum);
                body = NULL;
                if (head!=NULL) body = item_body_cheat(grp,num,groupnum);
                if ((head==NULL) || (body==NULL)) {
                        imsg("Feed article missing %s:%d\n",group_name(grp),num);
                        live_reject_later(chan,r); /* Remember this one is bad and don't reque it */
                        chan_write_s(chan,".\015\012");
                        return;
                }
                dmsg("live: Item read from file %d:%d\n",grp,num);
                big_init(chan); /* Check this clears the last one */
                str_lock(head);
                big_add(chan,str_s(head),str_len(head));
                big_add(chan,"\015\012",2);
                big_add(chan,str_s(body),str_len(body));
                str_unlock(head);
        } else {
                big_init(chan); 
                str_lock(msg);
                if (str_s(msg)!=NULL) big_add(chan,str_s(msg),str_len(msg));
                str_unlock(msg);
        }
        start_time = lib_msec();
        start_clock = stats_clock();        
	live_freeringmsg(chan,r);
        f = chan_feed[chan];
        if (f>0) {
                feeds[f].chomps += (big_len(chan)/100);
        }

	dmsg("live: live_send_chunk_started len=%d\n",big_len(chan));
        cmd_complete(chan,grp,num,0,big_len(chan),M_FEED);
        live_send_chunk(chan);  
        
}
void check_headout(char *head, int len, char *reason)
{
        /* Check each line contains ": " unless starts with a space */
        /* and if fromfile, then check it ends with crlf */
        char *s,*s2;
        char bf[BFSZ];
        int n;
        int badend=FALSE;

        if (head!=NULL) return; /* Disabled as it found nothing */

        if (head==NULL) {emsg("check_head, header was null (%s)\n",reason);}
        s = head;
        s2 = s;
        for (;(*s != 0) && ((s-head)<=len);) {  
                for (s2=s;*s2!=0;s2++) if (*s2=='\n') break;
                n = s2-s;
                if (n>=1) {
                        if (*s2==0) {
                                if (strcmp(s2-2,"\r\n")!=0) {
                                        badend = TRUE;
                                }
                        }
                }
                if (n>200) n = 200;
                ncpy(bf,s,n-1);
                if (strcmp(bf,"")==0) break;
                if (*bf==' ') goto cont;
                if (*bf=='\t') goto cont;
                if (strstr(bf,": ")==NULL) emsg("check_head, header missing colon space (%s)\n",bf);
cont:           if (*s2==0) break;
                s = s2 + 1;
        }       
        if (strcmp(reason,"From File")==0) {
                if (badend) emsg("check_head, header did not end correctly\n");
        }       
}
void live_send_chunk(int chan) /* Read the next chunk from the article and send it */
{
        char *s;
        int len;
        char lastbit[10];
        /* Must initialize chan_sent */
        int chunk = big_maxsz();
        int start_time = lib_msec();
        int start_clock = stats_clock();

        
	if (chunk==0) chunk = 20000;

        chan_hangup[chan] = lib_gmtime(); 

        strcpy(lastbit,"");
        dmsg("live: send chunk called from %d \n",chan_sent[chan]);
        s = big_get_chunk(chan,chan_sent[chan],chunk,&len);

        start_time = lib_msec();
        start_clock = stats_clock();
        
	if (len>0) if (s==NULL) {  
                wmsg("%s Live link failed to send item [%d] feed [%d] \n",linknamec(chan),chan,chan_feed[chan]);
                chan_write_s(chan,"\015\012.\015\012");
                live_reject_later(chan,chan_ringbig[chan]); /* Remember this one is bad and don't reque it */
                nntp_setbusy(chan,FALSE);
                return;
        }
        if (len>0) if (chan_sent[chan]==0) {
                check_headout(s,len,"Big get chunk");
        }
        if (len>0) {
                chan_write_slen(chan,s,len);
                if (len>3) {
                        ncpy(lastbit,s+len-2,3);
                }
        }
        chan_sent[chan] +=  len;
        if (chan_sent[chan] >= big_len(chan)) {
                dmsg("live: send chunk finished \n");
                if (len<=3) goto ok;
                if (strcmp(lastbit,"\r\n")!=0) {
                        imsg("lastbit (%s) len %d \n",lastbit,len);
                        imsg("%s Live link added crlf [%d] feed [%d] \n",linknamec(chan),chan,chan_feed[chan]);
                        big_save(chan);
                        chan_write_s(chan,"\015\012.\015\012");
                } else {
ok:                     chan_write_s(chan,".\015\012");
                }
                nntp_setbusy(chan,FALSE);
        }
}
#define GOT_PIPES TRUE

#ifdef WIN32
#define popen _popen
#define pclose _pclose
int fex_write(char *bf, int sz, int len, HANDLE h);
HANDLE ex_open(char *cmd,char *mde);
void ex_close(HANDLE h);
int ex_flush(HANDLE h);
#endif

#ifdef OS2
#undef GOT_PIPES
#endif

#ifdef Mac
#undef GOT_PIPES
#endif
#ifdef VMS
#undef GOT_PIPES
#endif
#ifdef NETWARE
#undef GOT_PIPES
#endif

#ifndef GOT_PIPES
void program_feed(int f,char *msgid,char *groupnum,String *xitem)
{                    
        emsg("program feeds not available on this system {%s} \n",msgid);
}
void prog_close(int f)
{
}
#else
#ifdef WIN32
static HANDLE prog_f[MAX_FEEDS];
static HANDLE prog_r[MAX_FEEDS];
#undef popen
#define mypopen(a,b) ex_open(a,b)
#undef pclose
#define mypclose ex_close
#define pwrite fex_write
#define pfflush ex_flush
#define done_pipes TRUE
#endif

#ifdef isunix
typedef struct {int h; int f;} XPIPE;
static  XPIPE *prog_f[MAX_FEEDS];
#define mypopen(a,b) ex_open(a,b)
#define mypclose ex_close
#define pwrite fex_write
#define pfflush ex_flush
#define done_pipes TRUE
int fex_write(char *bf, int sz, int len, XPIPE *h);
XPIPE *ex_open(char *cmd,char *mde);
void ex_close(XPIPE * h);
int ex_flush(XPIPE * h);

#endif

#ifndef done_pipes
#define mypopen(a,b) popen(a,b)
#define pwrite fwrite
#define pfflush fflush
#define mypclose pclose
static FILE *prog_f[MAX_FEEDS];
#endif
static char proglog[BFSZ];
void program_feed(int f,char *msgid,char *groupnum,String *xitem)
{
        char bf[MAX_LINE];
        int len;
        char *s2,*s1;
        
#ifdef isunix
        if (prog_f[f]!=NULL) if (!ex_isalive(prog_f[f])) {
                wmsg("Program feed died\n");
                mypclose(prog_f[f]);
                prog_f[f] = NULL;
        }
#endif
        if (prog_f[f]==NULL) {
                lprintf(bf,MAX_LINE-1,"prog_%d.log",f);
                strcpy(proglog,file_work(bf));
		if (feeds[f].prog_args==NULL)  feeds[f].prog_args = mystrdup("",SD_FEED);
                lprintf(bf,MAX_LINE-1,"%s %s",feeds[f].prog_name,feeds[f].prog_args);
#ifndef WIN32
                imsgf("feed: Pipe open {%s} (NT only, log=%s) \n",bf,proglog);
#endif
                prog_f[f] = mypopen(bf,"wb");
                imsgf("feed: Pipe open completed\n");
                if (prog_f[f]==NULL) filemsg("Pipe open failed (%s)\n",bf);
        }
        if (prog_f[f]==NULL) return;

        lprintf(bf,MAX_LINE-1,"ARTICLE %s %s\n",groupnum,msgid);
        
        imsg("feed: Writing article info to pipe\n");
        if (pwrite(bf,1,strlen(bf),prog_f[f])<=0) {
                filemsg("feed: [%d] pipe failed write0\n",f);
                mypclose(prog_f[f]);
                prog_f[f] = NULL;
                return;
        }
        imsg("feed: Writing article body to pipe\n");
        s1 = str_s(xitem);
        for (;s1!=NULL;) {
                s2 = strchr(s1,'\015');
                if (s2==NULL) break;
                len = (s2-s1);
                if (len>MAX_LINE-3) len = MAX_LINE-3;
                strncpy(bf,s1,len);
                bf[len] = '\n';
                bf[len+1] = 0;
                /* if (strcmp(bf,"..\n")==0) strcpy(bf,".\n"); */
                if (pwrite(bf,1,len+1,prog_f[f])!=len+1) {
                        filemsg("feed: [%d] pipe failed write\n",f);
                        mypclose(prog_f[f]);
                        prog_f[f] = NULL;
                        return;
                }
                if (*(s2+1)!='\012') break; 
                s1 = s2 + 2;
        }
        if (pwrite(".\n",1,2,prog_f[f])<=0) {
                filemsg("feed: [%d] pipe failed write2\n",f);
                mypclose(prog_f[f]);
                prog_f[f] = NULL;
                return;
        }

        if (pfflush(prog_f[f])!=0) {
                filemsg("feed: [%d] pipe failed flush\n",f);
                mypclose(prog_f[f]);
                prog_f[f] = NULL;
                return;
        }
        feeds[f].nok++;
        if (feeds[f].prog_single) {mypclose(prog_f[f]); prog_f[f] = NULL;}
}
void prog_close(int f)
{
        if (prog_f[f]!=NULL) mypclose(prog_f[f]);
        prog_f[f] = NULL;
}

#endif

#ifdef isunix
static int h_write[5000];
XPIPE *ex_open(char *cmd, char *mde)
{
        char *args[10];
        static int xwrite[2];
        int n=0,r;
        char *xcmd;
        XPIPE *hp;
        char *s;
        int h;  
        wmsg("Create program process (%s) \n",cmd);
        s = strtok(cmd," ");
        for (;s!=NULL;) {
                args[n++] = strdup(s);
                if (n==1) xcmd = strdup(s);
                s = strtok(NULL," ");
        }
        args[n++] = NULL;
        args[n++] = NULL;
        if (pipe(xwrite)!=0) {filemsg("pipe failed\n"); return FALSE;}
	fflush(NULL);
#ifdef HAS_VFORK
	h = vfork();
#else
	h = fork();
#endif
        if (h==0) { /* Child process */
                dup2(xwrite[0],STDIN_FILENO);
                close(xwrite[0]);
                /*printf("New stdout works"); fflush(stdout);*/
                if (execv(xcmd,args)<0) {
                        emsg("Could not create child process %s %s\n",xcmd,strerror(errno));
                        exit(1);
                }
        }
	if (h<0) filemsg("pipe: Create child process failed\n");
        imsg("pipe: Created child process %d\n",h);
        hp = mymalloc(sizeof(XPIPE));
        hp->h = h; 
        hp->f = xwrite[1];
        return hp;
}
int ex_flush(XPIPE *p)
{
        dmsg("pipe: ex_flush\n");
        return 0;
}
void ex_close(XPIPE *hp)
{
        int h;
        int r;
        int ecode;
        dmsg("pipe: ex_close %p\n",hp);
        if (hp==NULL) return;
        h = hp->h;
        if (hp->f!=0) {
                fex_write("EXIT\n",1,5,hp);
        }
        if (hp->f!=0) close(hp->f); hp->f = 0;
        myfree(hp);
        if (h>0) {
                r = waitpid(h,&ecode,WUNTRACED);
                if (r==h) {
                        wmsg("feed: Process %d has exited, status %x\n",h,ecode);
                        return ;
                }

        }
}
int fex_write(char *bf, int sz, int len, XPIPE *hp)
{
        int r = write(hp->f,bf,sz*len);
        dmsg("pipe: fex_write %p %d\n",hp,len);
        if (r<sz*len) {
                if (hp->f!=0) close(hp->f); hp->f = 0;
        }
        return r;
}
int ex_isalive(XPIPE *hp)
{
        int ecode;
        int r;
        int h = hp->h;
        dmsg("pipe: ex_isalive %p\n",hp);
        r = waitpid(h,&ecode,WNOHANG | WUNTRACED);
        if (r==h) {
                wmsg("pipe: Feed Process %d has exited, status %x\n",h,ecode);
                if (hp->f!=0) close(hp->f); hp->f = 0;
                hp->h = -1;
                return FALSE;
        }
        return TRUE;
}
#endif



#ifdef WIN32
int fex_write(char *bf, int sz, int len, HANDLE h)
{
        DWORD rlen;
        if (!WriteFile(h,bf,sz*len,&rlen,NULL)) {
                emsg("Read failed %d\n",GetLastError());        
        }
        return rlen;
}
HANDLE ex_open(char *cmd,char *mde)
{
        static DWORD len;
        HANDLE read1,write1,hout;
        STARTUPINFO st;
        PROCESS_INFORMATION p;
        SECURITY_ATTRIBUTES sec;

        
        sec.nLength = sizeof(sec);
        sec.lpSecurityDescriptor = NULL;
        sec.bInheritHandle = TRUE;
        GetStartupInfo(&st);
        CreatePipe(&read1,&write1,&sec,1000);
        hout = CreateFile(proglog,GENERIC_WRITE
                , FILE_SHARE_READ, NULL, CREATE_ALWAYS
                , FILE_ATTRIBUTE_NORMAL, NULL);

        if (hout==0) {
                filemsg("Could not create (%s) \n",proglog);
                return NULL;
        }
        
        st.hStdInput = read1;
        st.hStdOutput = hout;
        st.hStdError = hout;
        st.dwFlags = STARTF_USESTDHANDLES;
        if (CreateProcess(NULL,cmd,NULL,NULL
                ,TRUE,DETACHED_PROCESS
                ,NULL,NULL  /* Env, dir*/
                ,&st,&p)) {
        } else {
                emsg("CreateProcess Failed err %d",GetLastError());
                return NULL;
        }
        /* / Read from read2*/

        CloseHandle(p.hProcess);
        CloseHandle(p.hThread);
        return write1;
}
void ex_close(HANDLE h)
{
        fex_write("EXIT\n",1,5,h);
        CloseHandle(h);
}
int ex_flush(HANDLE h)
{
        return 0;
}
#endif 


static int chan_msg_n;
int chan_msg(const char * arg_list, ...)
{
	char *s;
        va_list arg_ptr;
        char *format;
	char output[30000];
        int len;
        va_start(arg_ptr, arg_list);
        format = (char *) arg_list;
        len = vsnprintf(output,20000, format, arg_ptr);
	output[20000] = 0;
	s = strchr(output,'\n');
	if (s!=NULL) *s = 0;
	strcat(output,"\r\n");
	chan_write_s(chan_msg_n,output);
	return 0;
}

void cmd_rules_reload(int chan)
{
	int i;
	chan_msg_n = chan;
	score_free();
	score_load(FILE_SCOREWORDS);
	dfilter_emsg(chan_msg);
	dfilter_trace(TRUE);
	for (i=0; i<nfeeds; i++) {
		if (feeds[i].rules!=NULL) {
			feeds[i].rules_code = dfilter_compile(feeds[i].rules);
		}
	}
	dfilter_trace(FALSE);
	dfilter_emsg(imsg);
}
void cmd_rules_trace(int chan, char *rulefile, char *fname)
{
	FProg *f;
	String *temp;
	int r;
	FILE *ff;
	char reason[BFSZ];
	char bf[BFSZ];
	if ((rulefile==NULL)||(fname==NULL)||(strlen(rulefile)<1)||(strlen(fname)<1)) {
	    lprintf(bf,BFSZ-1,"usage: tellnews rules_trace rule_file test_message_file\r\n",reason);
	    chan_write_s(chan,bf);
	    return;
	}
	chan_msg_n = chan;
	dfilter_emsg(chan_msg);
	dfilter_trace(TRUE);
	f = dfilter_compile(rulefile);
	if (f==NULL) {
		lprintf(bf,BFSZ-1,"---> Compile failed\r\n");
		chan_write_s(chan,bf);	
	}
	temp = str_new();
	if (!str_read(temp,fname)) {
		ff = fopen(fname,"r");
		lprintf(bf,BFSZ-1,"---> Could not read %s %s\r\n",fname,strerror(errno));
		if (ff!=NULL) fclose(ff);
		chan_write_s(chan,bf);	
		goto failed;
	}
	if (f!=NULL) r = dfilter_run(f,str_s(temp),str_len(temp),reason);
	if (r) lprintf(bf,BFSZ-1,"---> Message Accepted - %s\r\n",reason);
	else lprintf(bf,BFSZ-1,"---> Message Rejected - %s\r\n",reason);
	chan_write_s(chan,bf);	
failed:
	str_free(temp);
	if (f!=NULL) dfilter_free(f);
	dfilter_trace(FALSE);
	dfilter_emsg(imsg);
}
void cmd_rules_group(int chan, char *gname, char *action)
{


}

/* brian - new scoring filter - start */
void score_free(void)
{
        int h;
        Scorerej *r,*rnext;
        score_nfile = 0;
        for (h=0; h<SCORE_HASH_REJ; h++) {
                for (r = scorehashrej[h]; r!=NULL; r=rnext) {
                        rnext = r->next;
                        mystrfree(r->s,SD_FEED);
                        myfree(r);
                }
                scorehashrej[h] = NULL;
        }
}

void score_load(char *fname)
{
        FILE *f;
        char bf[BFSZ];
	char word[BFSZ];
	int score;
        char *s;
        char *c1, *c2;
        
        f = fopen(fname,"r");
        if (f==NULL) {imsg("score: could not find score file %s\n",fname); return;}
        for (;!feof(f);) {
                if (fgets(bf,BFSZ-2,f)) {
			s = strtok(bf,"\n\r");
                        if (s!=NULL) {
                                c1 = strstr(s,"/*");
                                if (c1!=NULL) {
                                        c2 = strstr(c1+1,"*/");
                                        if (c2!=NULL) {
                                                memmove(c1,c2+2,strlen(c2+1));
                                        }
                                }
				/* format should now be score:word */
				if ((c1=strchr(s,':'))==NULL) {
				    imsg("score: score filter rules must be in the format score:word ignoring (%.40s)\n",s);
				    continue;
				}
				ncpy(word,c1+1,BFSZ-10);
                                if (strlen(word)<3) {
				    imsg("score: score filter rules must be more than 2 chars (%s) \n",word);
				    continue;
				}
				score = atoi(s);
				dmsg("score: adding word '%s' with score of %d\n",word,score);
				score_add(word,score);
                        }
                }
        }
        fclosep(f);
}

void score_add(char *s,int score)
{
        int h;
        Scorerej *r;
        strlwr(s);
        h = score_hash(s);
        r = mymalloc(sizeof(Scorerej));
        memset(r,0,sizeof(Scorerej));
        r->next = scorehashrej[h];
        r->len = strlen(s);
        r->s = mystrdup(s,SD_FEED);
	r->score = score;
        scorehashrej[h] = r;
        score_nfile++;
}

int score_one(char *s,char *reason)
{
        /* go thru article, hash 4 bytes, lookup
           if matches reject rule then reject.*/
	int h;
        Scorerej *r;

        for (;(*s!=0) && (s[3]!=0);s++) {
                h = score_hash(s);
                for (r=scorehashrej[h];r!=NULL;r=r->next) {
                 if (strncmpnc(s,r->s,r->len)==0) {
                        r->hits++;
                        if (r->hits>30000) r->hits=30000;
                        if (reason!=NULL) lprintf(reason,BFSZ-1,"score_one match word (%s) score (%d) hits (%d)",r->s,r->score,r->hits);
                        return r->score;            
                 }
                }
        }       
        return 0; /* no matches so return zero for score */
}

int score_all(char *s)
{
        /* go thru article, hash 4 bytes, lookup
           if matches reject rule then reject.*/
	int h;
        /*char *s;*/
        Scorerej *r;
        /*s = str_s(xitem);*/
	int score_all=0;

	if ((s[1]==0) || (s[2]==0)) return 0;
        for (;(*s!=0) && (s[3]!=0);s++) {
                h = score_hash(s);
                for (r=scorehashrej[h];r!=NULL;r=r->next) {
                 if (strncmpnc(s,r->s,r->len)==0) {
                        r->hits++;
                        if (r->hits>30000) r->hits=30000;
                        dmsg("score: score_all match word (%s) score (%d) hits (%d)\n",r->s,r->score,r->hits);
                        score_all += r->score;            
                 }
                }
        }       
        return score_all; /* no matches so return zero for score */
}

#if score_testing
int score_loadtestfile(char *fname,char *c);
void score_test() {
    char buf[SCORE_TESTMAX];
    char reason[BFSZ];
    int score;

    if (!score_loadtestfile(FILE_SCORETEST,buf)) {
	imsg("score: scoring filter test file not readable (%s)(%s)\n",FILE_SCORETEST,strerror(errno));
    }
    else {
	imsg("score: successfully loaded scoring test article (%s)\n",FILE_SCORETEST);
	score_free();
	score_load(FILE_SCOREWORDS);
	score = score_one(buf,reason);
	if (score) imsg("score: %s\n",reason);
	else imsg("score: score_one no words matched\n");
	score = score_all(buf);
	imsg("score: score_all total score was %d\n",score);
    }
    return;
}

int score_loadtestfile(char *fname,char *c) {
    FILE *fp;    
    long num=0;
    long nread;
    char *bf;

    bf=c;
    fp=fopen(fname,"rb");
    if (fp==NULL) return FALSE;
    else {
	if (fseek(fp,SEEK_END,0)!=0) {
	    imsg("score: error seeking to eof (%s)(%s)\n",FILE_SCORETEST,strerror(errno));
	}
    	num = ftell(fp);
	if (fseek(fp,SEEK_SET,0)!=0) {
	    imsg("score: error seeking to start (%s)(%s)\n",FILE_SCORETEST,strerror(errno));
	}
	nread = fread(bf,sizeof(char),SCORE_TESTMAX,fp);
	if (nread<=0) {bf[0]=0;return FALSE;}
	else bf[SCORE_TESTMAX-1]=0;
	if (fclosep(fp)) imsg("score: error closing scoring filter test file (%s)(%s)\n",FILE_SCORETEST,strerror(errno));
	return TRUE;
    }
}
#endif

/* brian - new scoring filter - end */



int tout_shutdown(int ff)
{
        Feed *f = &feeds[ff];   

	if (f->th==NULL) return TRUE;

	tout_mutex_wait();
	
	if (f->th!=NULL) {
		f->th->finish = TRUE;
		f->th = NULL; /* Freed in thout thread */
	}
	tout_mutex_release();
	return TRUE;
}    
int tout_reopen(Thout *th)
{
	if (th->chan>=0) return TRUE;
	tout_mutex_wait();	

	if (th->chan<0) {
		th->chan = chan_open_bind(th->to,th->bind);
		nntp_setsndbf(th->chan, 128000);
		th->state = TOUT_OPEN;
		if (th->chan>=0) thout_start_setup(th->chan);
	}
	tout_mutex_release();
	return TRUE;
}



static Zitem *gotref;
#ifdef DOTHREADS
int tout_feed(int ff, char *msgid, char *groupnum, String *item)
{
	int ret = TRUE;
	int i;
        Feed *f = &feeds[ff];   

	tout_mutex_wait();
	if (f->th==NULL) {
		imsg("thout: Creating new thread for this feed (%s)\n",f->site);
		if (config_true(C_DEBUG_THOUT)) tout_setdebug(TRUE); 
		f->th = tout_new(feed_to(f),f->bind); 
	}
	tout_reopen(f->th);
	if (f->th->state!=TOUT_SEND) {ret=FALSE; goto done;}
	
	/* Can we find a free one */
	for (i=0; i<THMAX; i++) {
		if (f->th->next[i]==NULL) break;
	}
	if (i>=THMAX) {ret = FALSE; goto done;}

	if (gotref==NULL) gotref = tout_getref(f->th,item,groupnum,msgid); /* One for the pot*/
	f->th->next[i] = tout_getref(f->th,item,groupnum,msgid);
//	imsg("thout: Put into que[%d]\n",i,f->th->next[i]);
done:
	if (!ret) f->th->nbusy++;
	tout_mutex_release();
	return ret;
}    
#endif
int tout_free_pot(void)
{
	static int x;
	if (gotref==NULL) return TRUE;
	x++;
	if (x>1) {
		emsg("tout_free_called twice\n");
		abort();
	}
	tout_release(&gotref);
	gotref = NULL;
	x--;
	return TRUE;	
}