/* New sucking module, pulling :-) */
/*

Check for:
	strcpy
	snprintf



Test maxitems
Test maxage

Implement 'speed' setting
Make update_clear call these functions.
If pull_true then show users status in downloading part.
Implement 'mask' times
  
    
Add support for header sucking maybe, or avoid calling this?
Gives reasons of messages being accepted/rejected
Correctly detect close

*/
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <ctype.h>
#include "emsg.h"
#include "str.h"
#include "cmd.h"
#include "sary.h"
#include "group.h"
#include "dnews.h"
#include "lib.h"
#include "item.h"
#include "chan.h"
#include "hist.h"
#include "expire.h"
#include "feed.h"
#include "post.h"
#include "suck.h"
#include "chan_dmy.h"
#include "db.h"

#include "pull.h"


int pull_link_close_later(Link *l);
int pull_link_nbusy(Suck *sk);
#undef mymalloc
#undef myfree
#define mymalloc(x) mymalloc2(x,M_PULL)
#define myfree(x) myfree2(x,M_PULL)

int pull_upto_set_raw(int suckid, int g,int v);
int pull_upto_get_raw(int suckid, int g, int lmax);
static int nsucks;
void pull_double_reset(void);

static Link *links;
#define MAX_SUCKS 100
static Suck sucks[MAX_SUCKS];
static Link *chan_links[MAX_CHAN];
void pwmsg(Link *l, char *arg_list, ...);
/*
	pull_con true		Enable 'pull' for all sucking feeds 
	pull1 con=3 percent=90 maxold=2000 maxage=10 speed=n mb=x file=groups.list
	Enable pull for this sucking feed and set options 

  for (i=0; i<16; i++
*/

enum {P_CLOSED,P_OPENING, P_USER, P_PASS, P_PASS2, P_READY
	,P_GROUP, P_STAT, P_ARTICLE_WANT
	,P_ARTICLE, P_ARTICLE_STATUS, P_NEXT, P_BODY
	,P_MODE, P_MODE2, P_MUSER, P_MPASS, P_MPASS2
};

static char *pstates[]={
	"Closed", "Opening", "User", "Pass", "Pass2", "Ready"
	, "Group", "Stat", "Art_want"
	,"Article", "Art_status", "Next", "Body"
	, "Mode", "Mode", "User", "Pass", "Pass"
	,"None"
};

static int log_group;
void pull_init(void)
{
	int i;
	Link *l;

	log_group = group_find("alt.binaries.sounds.country.mp3");
	for (l=links;l!=NULL;l=l->next) {
		if (l->state==P_CLOSED) continue;
		if (l->chan>=0) if (chan_isopen(l->chan)) {
			pull_link_close(l);
		}
	}

	for (i=0; i<20; i++) {
		pull_init_one(i);
	}
}
void pull_init_one(int sk)
{
	int n;
	char *s;
	char *p[200];
	char bf[BFSZ];
	int i;

	if ((sk<0) || (sk>=MAX_SUCKS)) return;
	strcpy(sucks[sk].lasterr,"");
	strcpy(sucks[sk].lastok,"");
	ncpy(sucks[sk].user,"",100);
	ncpy(sucks[sk].pass,"",100);
	sucks[sk].con = 1;
	sucks[sk].percent = 0;
	sucks[sk].speed = 0;
	sucks[sk].maxitems = 0;
	sucks[sk].maxage = 0;
	sucks[sk].mb = 0;
	sucks[sk].nonext = FALSE;
	sucks[sk].id = sk;
	sucks[sk].ispull = FALSE;
	strcpy(sucks[sk].disable,"");
	
	sucks[sk].nrej2 = sucks[sk].nrej;
	sucks[sk].nok2 = sucks[sk].nok;
	sucks[sk].ndup2 = sucks[sk].ndup;
	sucks[sk].bytes2 = sucks[sk].bytes;


	strcpy(sucks[sk].site,"");
	ncpy(bf,config_get(C_NNTP_SUCK1+sk),BFSZ-1);
	if (strlen(bf)==0) return;
	s = strtok(bf," \r\n\t"); 
	if (s!=NULL) ncpy(sucks[sk].site,s,90);
	if (s!=NULL) s = strtok(NULL," \r\n\t"); 
	if (s!=NULL) ncpy(sucks[sk].groups,s,BFSZ-1);
	if (s!=NULL) s = strtok(NULL," \r\n\t"); 
	if (s!=NULL) ncpy(sucks[sk].user,s,BFSZ-1);
	if (s!=NULL) s = strtok(NULL," \r\n\t"); 
	if (s!=NULL) ncpy(sucks[sk].pass,s,BFSZ-1);

	imsg("pull: initialized %.200s %.80s\n",sucks[sk].site,sucks[sk].groups);
/* UPDATE_MASK (TIMES WHEN UPDATING IS VALID)*/
	
	ncpy(bf,config_get(C_PULL1+sk),BFSZ-1);
	if (strlen(bf)>0) sucks[sk].ispull = TRUE;
	/* con=3 percent=90 maxold=2000 maxage=10 speed=n mb=x file=groups.list 
		disable=8-17
	*/

	strlwr(bf);
	n = token_split(bf,p," \t\r\n",20);
	for (i=0; i<n; i++) {
		if (strncmp(p[i],"con=",4)==0) sucks[sk].con = atoi(p[i]+4);
		else if (strncmp(p[i],"percent=",8)==0) sucks[sk].percent = atoi(p[i]+8);
		else if (strncmp(p[i],"maxitems=",9)==0) sucks[sk].maxitems = atoi(p[i]+9);
		else if (strncmp(p[i],"maxage=",7)==0) sucks[sk].maxage = atoi(p[i]+7);
		else if (strncmp(p[i],"speed=",6)==0) sucks[sk].speed = atoi(p[i]+6);
		else if (strncmp(p[i],"mb=",3)==0) sucks[sk].mb = atoi(p[i]+3);
		else if (strncmp(p[i],"disable=",8)==0) ncpy(sucks[sk].disable,p[i]+8,30);
		else if (strncmp(p[i],"nonext",6)==0) sucks[sk].nonext = TRUE;
		else emsg("Invalid pull setting (%.200s)\n",p[i]);
	}
}
void pull_idle(void)
{
	Link *l;
	Suck *sk;
	static int last;
	static int lastday;
	int t = lib_gmtime();
	int i;

	pull_double_check("idle1");
	if (last==0) last = lib_gmtime()-10;
	if (is_readonly()) return;
	for (l=links;l!=NULL;l=l->next) {
		if (l->state==P_CLOSED) continue;
		if ((t - l->xtime)>120) {
			pmsg(l,"Timeout link after %d seconds\n",t - l->xtime);
			if (l->grp>=0) if (l->uptoi>0) pull_upto_set(l,l->grp,l->uptoi+1);
			pull_link_close(l);
		}
		if (l->chan>=0) if (!chan_isopen(l->chan)) {
			if (l->grp>=0) if (l->uptoi>0) pull_upto_set(l,l->grp,l->uptoi);
			pull_link_close(l);
		}
	}
	pull_double_check("idle2");
	for (i=0; i<MAX_SUCKS; i++) {
		if (sucks[i].nmask==0) continue;
		sk = &sucks[i];
		stats_feed(sk->site,lib_gmtime()-last,sk->nrej - sk->nrej2,sk->nok - sk->nok2,sk->ndup - sk->ndup2,sk->bytes - sk->bytes2,0.0,0.0);
		sk->nrej2 = sk->nrej;
		sk->nok2 = sk->nok;
		sk->ndup2 = sk->ndup;
		sk->bytes2 = sk->bytes;

		if (pull_isstop(sk)) {
			pull_stop(sk);
			continue;	
		} 
		if (pull_link_nbusy(sk)<sucks[i].con) pull_start(i);
	}
	pull_double_check("idle3");
	last = lib_gmtime();
	if ((t-lastday)>(60*60*24)) {
		imsgf("pull: Aging pull mb data as one day has passed\n");
		for (i=0; i<MAX_SUCKS; i++) {
			sucks[i].bytes = 0;
			sucks[i].bytes2 = 0;
		}
		lastday = t;
	}
	pull_double_check("idle4");
}

int pull_link_close_later(Link *l)
{
	imsg("pull: Due to failure will not retry link for 60 seconds\n");
	l->suck->later = time(NULL) + 60;
	return pull_link_close(l);
}
int pull_link_close(Link *l)
{
	pull_double_check("close1");
	if (l->chan<0 || l->chan>=MAX_CHAN) {
	    pemsg(l,"Channel was already closed, state was %d - fixing\n",l->state);
    	    l->state = P_CLOSED;
    	    chan_links[l->chan] = NULL;
	    return TRUE;
	}
	
	if (l->chan<0) return FALSE;
	if (l->chan>=MAX_CHAN) return FALSE;
	if (l->state==P_CLOSED) return FALSE;
	
	if ((l->state>=P_GROUP) && (l->state<=P_BODY)) {
		if (l->grp>=0) if (l->uptoi>0) pull_upto_set(l,l->grp,l->uptoi);
	}
	
	l->state = P_CLOSED;
	chan_links[l->chan] = NULL;
	pdmsg(l,"Closing\n");

	chan_close(l->chan);
	l->chan = -1;
	l->state = P_CLOSED;
	return TRUE;
}
void pull_start(int sk)
{
	Link *l;
	/* Find a free link to that site */
	if ((sk<0) || (sk>=MAX_SUCKS)) return;
	if (time(NULL)<sucks[sk].later) {
		imsg("Not trying again for %d seconds\n",sucks[sk].later-time(NULL));
		return;
	}

	pull_double_check("start1");
	dmsg("pull: Start link %d (%.200s)\n",sk,sucks[sk].site);
	for (l=links;l!=NULL;l=l->next) {
		if (l->suck->id!=sk) continue;	
		if (l->state==P_READY) break;
	}
	dmsg("pull: Existing link found %p\n",l);
	if (l==NULL)  for (l=links;l!=NULL;l=l->next) {
		if (l->suck->id!=sk) continue;	
		if (l->state==P_CLOSED) {
			pdmsg(l,"Found closed link to reopen\n");
			pull_open_link(l); /* Open the link up again */
			return;
		}
	}
	if (l==NULL) {
		if (sucks[sk].nlinks>=sucks[sk].con) {
			dmsg("pull: All links busy %d %d\n",sucks[sk].nlinks,sucks[sk].con);
			return; /* all busy*/
		}
		l = pull_new_link(sk); /* Open a new one to that site */
		return; /* Not in right state yet */
	}
	pdmsg(l,"Start the link %d %d \n",l->chan,l->state);
	pull_start_link(l);
} 
Link * pull_new_link(int sk)
{
	Link *l;
	static int id;

	if ((sk<0) || (sk>=MAX_SUCKS)) return NULL;
	pull_double_grab();
	l = mymalloc(sizeof(Link));
	pull_double_grab();
	memset(l,0,sizeof(Link));
	l->suck = &sucks[sk];
	l->next = links;
	l->id = ++id;
	l->chan = -1;
	l->xtime = lib_gmtime()+30;
	l->suck->nlinks++;
	pdmsg(l,"Creating new link structure\n");
	pull_open_link(l);
	links = l;
	pull_double_reset();
	return l;
}
void pull_open_link(Link *l)
{
	l->xtime = lib_gmtime()+300;
	if (l->chan>0) emsg("Channel not cleared!!!\n");
	if (strlen(l->suck->site)==0) return;
	l->chan = chan_open(l->suck->site);
	pdmsg(l,"Opening channel %d %.200s\n",l->chan,l->suck->site);
	if ((l->chan<0)||(l->chan>=MAX_CHAN)) {pemsg(l,"Open (%.200s) failed\n",l->suck->site); return;}
	if (l->suck->speed>0) {
		chan_setinspeed(l->chan,l->suck->speed);
		pmsg(l,"Limiting incoming speed to %d bytes per second\n",l->suck->speed);
	}
	chan_links[l->chan] = l;
	l->state = P_OPENING;
	l->xtime = lib_gmtime()+300;
	chan_setfn(l->chan,pull_data_chan);
}
void pull_start_link(Link *l)
{
	int grp;
	int k=0;
	/* Get the next group to suck from that site */
again:
	k++;
	if (!pull_next(&grp,l->suck)) {
		/* Nothing to suck, close the link down */
		pdmsg(l,"No groups queued to suck\n");
		pull_link_close(l);
		strcpy(l->gname,"none");
		return; 
	}
	ncpy(l->gname,group_name(grp),200);
	
        if (!group_wanted(l->gname)) {
		imsg("pull: Skipping group which is not in 'me' feed {%s}\n",l->gname);
		if (k<50000) goto again;
        }
	l->grp = grp;
	pull_write(l,"GROUP %.200s\r\n",group_name(grp));
	l->suck->ngot++;
	l->state = P_GROUP;
	l->xtime = lib_gmtime()+30;	
	l->uptoi = pull_upto_get(l,l->grp);
	pdmsg(l,"Found next group to pull (%d)\n",grp);
}
int pull_data_chan(int chan, char *data, int len)
{
	Link *l;
	static int in;
	if ((chan<0)||(chan>=MAX_CHAN)) return TRUE;

	pull_double_check("data1");
	l = chan_links[chan];
	if (l==NULL) {
		dmsg("pull: Unknown channel-link %d\n",chan);
		return TRUE;
	}
	if (len<0) {
		if (in>0) return TRUE;
		in=1;
		pemsg(l,"Channel closed s=%d %d\n",l->state,l->chan);
		if (l->chan>=0)  {
			pull_link_close(l);
		}
		in = 0;
		return TRUE;
	}
	l->xtime = lib_gmtime();
	if (l->state==P_BODY) {pull_data_body(l,data,len); return TRUE;}
	data[len] = 0;
	for (;len>0;len--) {
		if (data[len-1]!='\n') if (data[len-1]!='\r') break;
		len--;
		data[len] = 0;
	}
	if (len>200) data[len] = 0;
	pull_double_check("data2");
	pull_data(l,data);
	pull_double_check("data3");
	return TRUE;
}
void pull_store(Link *l)
{
	String *get;
	int r = FALSE;
	pdmsg(l,"<--[%d] ...End of data\n",l->state);
	chan_setinbody(l->chan,FALSE); /* Finished getting body */
        msg_from(l->suck->site);
	if (big_len(l->chan) >= MAX_STRLEN) {
		msg_reject("Item far too big, ignoring it\n"); 
		return;
	}
	get = big_get(l->chan);
	set_useage(l->suck->maxage);
	if (get!=NULL) r = item_suck(get);
	if (r) {
		pdmsg(l,"Stored item ok %.200s\n",l->mid); l->suck->nok++;
		if ((time(NULL) % 60) == 30) ncpy(l->suck->lasterr,"",80);
	} else {
		pmsg(l,"Rejected (%.200s) %.200s\n",l->mid,msg_reason());
		ncpy(l->suck->lasterr,msg_reason(),80);
		l->suck->nrej++;
	}
        big_free(l->chan);
}
void pull_data_body(Link *l,char *data,int len)
{
	l->suck->bytes += len;
	l->batchsize += len;
        if (strcmp(data,".\015\012")==0) {
		pull_double_check("data337");
		pull_store(l);
		l->state = P_NEXT;
		pull_double_check("data338");
		pull_data(l,"200 Internal, Get next item");
		return;
	}
	if (big_len(l->chan) >= MAX_STRLEN) return;
	big_add(l->chan,data,len);		
}
#define OUTSIZE 33000
#ifdef OS2
#define vsnprintf _vbprintf
#endif
#ifdef WIN32
#define vsnprintf _vsnprintf
#endif
void pmsg(Link *l, char *arg_list, ...)
{
        va_list arg_ptr;
        char *format;
        int len;
	char output[OUTSIZE];
	char bf[BFSZ];

        va_start(arg_ptr, arg_list);
        format = arg_list;
        len = vsnprintf(output,900, format, arg_ptr);
	output[900] = 0;
	if (len>800) {
		emsg("pmsg string violation %d bytes, email this to support@netwinsite.com\n",len);
		emsg("-> %.80s\n",output);
		abort();
	}
	lprintf(bf,BFSZ-1,"pull: [%d] %.80s %.80s:%d %.180s",l->id,l->suck->site,l->gname,l->uptoi,output);
	imsg("%.200s",bf);
}
void pwmsg(Link *l, char *arg_list, ...)
{
        va_list arg_ptr;
        char *format;
        int len;
	char output[OUTSIZE];
	char bf[BFSZ];

        va_start(arg_ptr, arg_list);
        format = arg_list;
        len = vsnprintf(output,900, format, arg_ptr);
	output[800] = 0;
	if (len>800) {
		emsg("pmsg string violation %d bytes, email this to support@netwinsite.com\n",len);
		emsg("-> %.80s\n",output);
		abort();
	}
	lprintf(bf,BFSZ-1,"pull: [%d] %.80s %.80s:%d %.180s",l->id,l->suck->site,l->gname,l->uptoi,output);
	wmsg("%.200s",bf);
}
void pemsg(Link *l, char *arg_list, ...)
{
        va_list arg_ptr;
        char *format;
        int len;
	char output[OUTSIZE];
	char bf[BFSZ];

        va_start(arg_ptr, arg_list);
        format = arg_list;
        len = vsnprintf(output,900, format, arg_ptr);
	output[800] = 0;
	if (len>800) {
		emsg("pmsg string violation %d bytes, email this to support@netwinsite.com\n");
		emsg("-> %.80s\n",output);
		abort();
	}
	lprintf(bf,BFSZ-1,"pull: [%d] %.80s %.80s:%d %.180s",l->id,l->suck->site,l->gname,l->uptoi,output);
	emsg("%.200s",bf);
	ncpy(l->suck->lasterr,bf,80);
}
void pdmsg(Link *l, char *arg_list, ...)
{
        va_list arg_ptr;
        char *format;
        int len;
	char output[OUTSIZE];
	char bf[BFSZ];

        va_start(arg_ptr, arg_list);
        format = arg_list;
        len = vsnprintf(output,900, format, arg_ptr);
	output[800] = 0;
	if (len>800) {
		emsg("pmsg string violation %d bytes, email this to support@netwinsite.com\n",len);
		emsg("-> %.80s\n",output);
		abort();
	}
	lprintf(bf,BFSZ-1,"pull: [%d] %.80s %.80s:%d %.180s",l->id,l->suck->site,l->gname,l->uptoi,output);
	dmsg("%.200s",bf);
}
void pull_data(Link *l, char *line)
{
	int code = atoi(line);
	int p,limit,batchn,batchsize;
	int maxold;
	int n;
	int x;
	char mid[BFSZ];
	l->suck->bytes += strlen(line);
	l->xtime = lib_gmtime();
	pdmsg(l,"<--[%d] %.200s\n",l->state,line);
again:
	switch (l->state) {
	case P_OPENING:
		 if ((code<200) || (code>299)) {
			 pemsg(l,"Open responded (%.200s)\n",line);
		 }
		 if (strlen(l->suck->user)>0) {
			 l->state = P_USER;
			 goto again;
		 }
		l->state = P_MODE;
		goto again;
	case P_USER:
		if (l->suck->user==NULL) {emsg("Fatal internal error-usernull\n"); return;}
		pull_write(l,"AUTHINFO USER %.200s\r\n",l->suck->user);
		l->state = P_PASS;
		break;
	case P_PASS:
		if (code!=381) {emsg("pull: Fatal bad response to authinfo user (%.200s)\n",line); pull_link_close_later(l); return;}
		pull_write(l,"AUTHINFO PASS %.200s\r\n",l->suck->pass);
		l->state = P_PASS2;
		break;
	case P_PASS2:
		if (code!=281) {pemsg(l,"Fatal bad response to authinfo pass (%.200s)\n",line); pull_link_close_later(l); return;}
		l->state = P_MODE;
		goto again;
	case P_MODE:
		pull_write(l,"MODE READER\r\n");
		l->state = P_MODE2;
		break;
	case P_MODE2:
		/* We don't care about response to mode command */
		if (strlen(l->suck->user)>0) {
			if(config_true(C_SUCK_NOAUTH2)) l->state = P_READY;
			else l->state = P_MUSER;
			goto again;
		}
		l->state = P_READY;
		goto again;
	case P_MUSER:
		pull_write(l,"AUTHINFO USER %.200s\r\n",l->suck->user);
		l->state = P_MPASS;
		break;
	case P_MPASS:
		if (code!=381) {emsg("pull: Fatal bad response to authinfo user (%.200s)\n",line); pull_link_close_later(l); return;}
		pull_write(l,"AUTHINFO PASS %.200s\r\n",l->suck->pass);
		l->state = P_MPASS2;
		break;
	case P_MPASS2:
		if (code!=281) {pemsg(l,"Fatal bad response to authinfo pass (%.200s)\n",line); pull_link_close_later(l); return;}
		l->state = P_READY;
		goto again;
	case P_READY: /*Dummy entry */ 
		pull_start_link(l);
		break;
	case P_GROUP: /* 211 3892 170808 174699 rec.humor selected*/
		if (code==0) if (strlen(line)<3) {
			pemsg(l,"Got blank line from weird news server\n");
			break;
		}
		if (code!=211) {
			pemsg(l,"GROUP response %.200s\n",line); 
			l->state = P_READY; goto again;
		}
		l->nitems = atoi(get_word(line,1));
		l->min = atoi(get_word(line,2));
		l->max = atoi(get_word(line,3));
		l->uptoi = pull_upto_get(l,l->grp);
		if (log_group==l->grp) 	sucklog("pull: nitems = %d min=%d max=%d uptoi=%d\n",l->nitems,l->min,l->max,l->uptoi);

		if (l->uptoi>(l->max+1)) {
			pmsg(l,"Resetting upto to end of group as it was past it %d > %d\n",l->uptoi,l->max);
			l->uptoi = l->max+1;
			pull_upto_set(l,l->grp,l->uptoi);
		}

		l->batchsize = 0;
		pdmsg(l,"pull: nitems = %d min=%d max=%d uptoi=%d\n",l->nitems,l->min,l->max,l->uptoi);
		if (l->uptoi<l->min) l->uptoi = l->min-1;
		
		p = l->suck->percent; if (p==0) p = 100;
		limit = (int) (((100.0-p)/100.0)*l->nitems);
		if(batchn = atoi(config_get(C_SUCK_BATCHN))) {
			if(batchn<limit || limit==0) {
				x = l->uptoi+batchn;
				if (x<l->max) l->max = x;
				limit=0;
			}
		}

		if (limit!=0) {
			if(!batchn) pmsg(l,"Limiting to %d%% of items, ignoring %d items, sucking to %d of %d\n",p,limit,l->max-limit,l->nitems);
			l->max -= limit;
			if (l->max<l->min) {
				pemsg(l,"Limiting percent went below min %d-%d %d\n",l->min,l->max,p);
				l->max = l->min;
			}
		}
		if (l->uptoi>l->max) {
			pmsg(l,"Correcting uptoi from %d to %d\n",l->uptoi,l->max);
			l->uptoi = l->max;
			pull_upto_set(l,l->grp,l->uptoi);
		}
		maxold = atoi(config_get(C_SUCK_MAXOLD));
		if (maxold==0) maxold = 2000;
		if (l->suck->maxitems>0) maxold = l->suck->maxitems;
		if (l->uptoi<l->max-maxold) {
			pmsg(l,"Correcting uptoi to limit maxold from %d to %d\n",l->uptoi,l->max-maxold);
			l->uptoi = l->max-maxold;
			pull_upto_set(l,l->grp,l->uptoi);
		}
		if (l->uptoi>=l->max) {
			pmsg(l,"Nothing new\n");
			pull_upto_set(l,l->grp,l->uptoi);
			group_setupdate(l->grp,FALSE);
			l->state = P_READY; goto again;
		}
		l->uptoi = l->uptoi+1;
		pwmsg(l,"Pulling from %d to %d (up to %d new)\n",l->uptoi,l->max,l->max - l->uptoi + 1);
		pull_write(l,"STAT %d\r\n",l->uptoi);
		l->state = P_STAT;
		break;
	case P_STAT: /*223 170916 <37AFEF49.5F96@nospam.att.net> article retrieved - request text separately */
		if (code==0) if (strlen(line)<3) {
			pemsg(l,"Got blank line from weird news server2 \n");
			break;
		}
		/* If it's not there, then search until we find one */
		if (code>399) { /* Find a real one */
next_one_please:
			if (l->uptoi>=l->max) {
				pmsg(l,"Got to end of group while searching for end of gap\r\n");
				pull_upto_set(l,l->grp,l->max);
				group_setupdate(l->grp,FALSE);
				l->state = P_READY;
				goto again;
			}
			pull_write(l,"STAT %d\r\n",++(l->uptoi));
			break;
		}
		if (code!=223) emsg("pull: bad code (%.200s)\n",line);
		if (code==211) {
			pemsg(l,"Lost sync, will close connection\n");
			pull_link_close(l);
			break;
		}
		l->state = P_ARTICLE_WANT;
		goto again;
	case P_ARTICLE_WANT:
		pull_double_check("data33");
		n = atoi(get_word(line,1));
		if (n==0) if (strlen(line)<3) {
			pemsg(l,"Got blank line from weird news server\n");
			break;
		}
		ncpy(mid,get_word(line,2),BFSZ-10);
		ncpy(l->mid,mid,200);
		if (code==423) { /* Stupid system can't decide if it has that article get next one */
		    goto next_one_please;
		}
		if (l->uptoi>n) {
			if (code!=421) pemsg(l,"Bad line when asked for stat %d/%d %.200s\n",l->uptoi,l->max,line);
			pull_upto_set(l,l->grp,l->max);
			group_setupdate(l->grp,FALSE);
			l->state = P_READY;
			goto again;
		}
		if (n!=l->uptoi) {
			pmsg(l,"Correcting uptoi was %d now %d\n",l->uptoi,n);
			l->uptoi = n;
		}
		if (l->uptoi>l->max) {
			pmsg(l,"Thats all we want from that group (Percentage or suck_batchn)\r\n");
			pull_upto_set(l,l->grp,l->max);
			group_setupdate(l->grp,FALSE);
			l->state = P_READY;

			if(config_get(C_SUCK_REQUEUE)) {
				pull_add(l->suck->id,l->grp,FALSE);
				break;
			}

			goto again;
		}
		
		if(batchsize=atol(config_get(C_SUCK_BATCHSIZE))) {
			if(l->batchsize>=batchsize) {
				pmsg(l,"Thats all we want from that group (suck_batchsize)\r\n");
				pull_upto_set(l,l->grp,l->uptoi);
				group_setupdate(l->grp,FALSE);
				l->state = P_READY;

				if(config_get(C_SUCK_REQUEUE)) {
					pull_add(l->suck->id,l->grp,FALSE);
					break;
				}

				goto again;
			}
		}
		
		if (l->uptoi%200==5) {
			if (l->grp>=0) if (l->uptoi>0) pull_upto_set(l,l->grp,l->uptoi);
		}

		if (hist_gotit2(mid,l->chan)) {
			l->suck->ndup++;
			pdmsg(l,"Already got %.200s\n",mid);
			l->state = P_NEXT;
			goto again;
		}

		l->state = P_ARTICLE;
		goto again;
	case P_ARTICLE: /*220 170916 <37AFEF49.5F96@nospam.att.net> article retrieved - head and body foll*/
		if (l->suck->nonext) {
			pull_write(l,"ARTICLE %d\r\n",l->uptoi);
		} else {
			pull_write(l,"ARTICLE\r\n");
		}
		l->state = P_ARTICLE_STATUS;		
		break;
	case P_ARTICLE_STATUS:
		if ((code<200) || (code>299)) {
			pmsg(l,"Missing article %.80s %.80s %.80s\n",l->suck->site,l->gname,line);
			l->state = P_NEXT;
			if (code==501 || code==423) {
				pmsg(l,"We'll avoid the next command as it seems to be broken on this system\n");
				l->suck->nonext = TRUE;
			}
			goto again;
		}
		l->state = P_BODY;
		big_init(l->chan);
		chan_setinbody(l->chan,TRUE);/*Send us big chunks*/
		break;
	case P_NEXT:
		l->uptoi++;
		if (l->suck->nonext) {
			pull_write(l,"STAT %d\r\n",l->uptoi);
		} else {
			pull_write(l,"NEXT\r\n");
		}
		l->state = P_ARTICLE_WANT;
		break;
	}
}
int pull_write(Link *l,char *arg_list, ...)
{
        va_list arg_ptr;
        char *format;
        int len;
	char output[4000];
	char *s;
        va_start(arg_ptr, arg_list);
        format = arg_list;
        len = vsnprintf(output,900, format, arg_ptr);
	output[400] = 0;
	if (len>400) {
		emsg("Wrote too much data to pull_write %.80s\n",output);
	}
	if (l->chan<0) return -1;
	chan_write_s(l->chan,output);
	s = strchr(output,'\r'); if (s!=NULL) *s = 0;
	pdmsg(l,"-> %.200s\n",output);
	return TRUE;
}
extern int xmax_groups;
int pull_suck(int suck, int grp, int isurgent)
{
	if (!pull_true(suck)) return FALSE;
	if (is_readonly()) return FALSE;
	pull_add(suck,grp,isurgent);
	return TRUE;
}
int pull_true(int suck)
{
	if ((suck>0) && (suck<MAX_SUCKS)) {
		if (sucks[suck].ispull) return TRUE;
	}
	if (config_true(C_PULL)) {
		if (strlen(config_get(C_HEADER_GROUPS))>0) {
			emsg("Cannot use pull module with header sucking\n");
			return FALSE;
		}	
		return TRUE;
	}
	return FALSE;
}
static int saved_max;
int pull_add(int suck, int grp, int isurgent) 
{
	Suck *p;

	if ((suck<0) || (suck>=MAX_SUCKS)) return FALSE;

	dmsg("pull: suck group %.200s from suck%d\n",group_name(grp),suck);
	if (suck<0) return FALSE; if (suck>=MAX_SUCKS) return FALSE;	 
	p = &sucks[suck];
	if (p->mask==NULL) {
		p->mask = mymalloc(xmax_groups/8+10);
		memset(p->mask,0,xmax_groups/8+1);
		saved_max = xmax_groups;
		dmsg("pull: Created mask\n");
	}
	if (grp>saved_max) return FALSE;
	if (grp<0) return FALSE;
	if (bit_isset(p->mask,grp)) {
		dmsg("pull: Already in sucking que\n");
		return TRUE;
	}
	p->nmask++;
	if (isurgent) {
		p->gupto = grp-1;
		sucklog("pull: reset gupto to %d (%s)\n",grp,group_name(grp));
	}
	if (p->gupto<0) p->gupto = 0;
	bit_set2(p->mask,grp,TRUE);
	pull_start(suck);
	return TRUE;
}
int pull_isstop(Suck *sk) 
{
	Link *l;
	char bf[BFSZ];
	for (l=links;l!=NULL;l=l->next) {
		if (l->suck==sk) break;
	}

	if (sk->mb>0) {
		if (sk->mb < (sk->bytes/1000000)) {
			if (!sk->warned)
			  if (l!=NULL) pemsg(l,"Daily limit exceeded  %4.2gMB > %dMB\n",sk->bytes/1000000.0,sk->mb);
			sk->warned = TRUE;
			return TRUE;
		}
	}
	if (strlen(sk->disable)>0) {
		lprintf(bf,BFSZ-1,"* %.100s",sk->disable);
		if (conf_isat(bf)) {
			if (!sk->warned)
			  if (l!=NULL) pemsg(l,"Disable time %.100s\n",sk->disable);
			sk->warned = TRUE;
			return TRUE;
		}
	}
        if (!lib_checkspace()) {
		if (!sk->warned)
		  if (l!=NULL) pemsg(l,"Disk full\n");
		sk->warned = TRUE;
		return TRUE;
	}
	sk->warned = FALSE;
	return FALSE;
}
int pull_stop(Suck *sk)
{
	Link *l;
	for (l=links;l!=NULL;l=l->next) {
		if (l->suck!=sk) continue;
		if (l->state==P_CLOSED) continue;
		if (l->chan>=0)  {
			pmsg(l,"Closing link to enforce limit\n");
			pull_link_close(l);
		}
	}
	return TRUE;
}
int pull_clear(void)
{
	Link *l;
	int i;
	int k;
	Suck *sk;
	for (l=links;l!=NULL;l=l->next) {
		if (l->state==P_CLOSED) continue;
		if (l->chan>=0)  {
			pmsg(l,"Closing link pull_clear()\n");
			pull_link_close(l);
		}
	}
	for (k=0; k<MAX_SUCKS; k++) {
		sk = &sucks[k];
		if (strlen(sk->site)==0) continue;
		if (sk->mask!=NULL) for (i=0;i<xmax_groups;i++) {
			if (bit_isset(sk->mask,i)) {
				bit_set2(sk->mask,i,FALSE);
				sk->nmask--;
			}
		}
	}
	return TRUE;
}
int pull_next(int *grp, Suck *p) 
{
	if (p->nmask<=0) return FALSE;
	if (pull_isstop(p)) return FALSE;
	p->gupto++;
	for (;p->gupto<xmax_groups;p->gupto++) {
		if (bit_isset(p->mask,p->gupto)) {
			bit_set2(p->mask,p->gupto,FALSE);
			p->nmask--;
			*grp = p->gupto;
			dmsg("pull: Found group %d\n",*grp);
			if (*grp == log_group) sucklog("pull: Found group (%s) to suck next (%s)\n",group_name(*grp),p->site);
			return TRUE;
		}
	}
	sucklog("pull: Got to end of mask at last, starting from top %s\n",p->site);
	for (p->gupto=0;p->gupto<xmax_groups;p->gupto++) {
		if (bit_isset(p->mask,p->gupto)) {
			bit_set2(p->mask,p->gupto,FALSE);
			p->nmask--;
			*grp = p->gupto;
			dmsg("pull: Found group %d\n",*grp);
			return TRUE;
		}
	}
	dmsg("pull: No MORE groups wanted\n");
	p->nmask = 0;
	return FALSE;
}
#define group_to_gid group_getgroupid




int pull_getold(int grp, int feed, int items)
{
	int n = pull_upto_get_raw(feed,grp,-2);
	if (n>0) pull_upto_set_raw(feed,grp,n-items);
	imsg("pull_getold was %d for group (%s) now is set to %d %d\n",n,group_name(grp),n-items,pull_upto_get_raw(feed,grp,-2));
	return TRUE;
}
int pull_show(int chan, char *group)
{
	int i;
	int upto;
	char bf[BFSZ];
	int g = group_find(group);
	if (g<0) return FALSE;
	for (i=0; i<5; i++) {
		upto = pull_upto_get_raw(i,g,-2);
		lprintf(bf,BFSZ-1," [%d] Upto %d, group %s\n",i,upto,group);
		chan_write_s(chan,bf);
	}
	return TRUE;
}
int pull_upto_get(Link *l, int g)
{
	if (g<0) return l->max;
	return pull_upto_get_raw(l->suck->id,g,l->max);
}
int pull_upto_get_raw(int suckid, int g, int lmax)
{
	FILE *f;
	char fname[BFSZ];
	int gid;
	int v;
	gid = group_to_gid(g);

	lprintf(fname,BFSZ-1,"upto_%d.dat",suckid);
	f = fopen(file_work(fname),"rb");
	if (f==NULL) return 0;
	if (fseek(f,gid*sizeof(int),SEEK_SET)!=0) {
		filemsg("pull: pull_upto_get seek failed %d\n",gid);
		fclosep(f);
		return lmax;
	}
	if (fread(&v,sizeof(int),1,f)<=0) {
		filemsg("pull: pull_upto_get read failed %d\n",gid);
		fclosep(f);
		return 0;
	}
	fclosep(f);
	imsg("pull: pull_upto_get id=%d gid=%d %s:%d\n",suckid,gid,group_name(g),v);
	return v;
}
int pull_upto_set(Link *l, int g,int v)
{
	return pull_upto_set_raw(l->suck->id,g,v);
}
int pull_upto_set_raw(int suckid, int g,int v)
{
	FILE *f;
	char fname[BFSZ];
	int gid;
	if (g<0) {
		return FALSE;
	}
	gid = group_to_gid(g);

	lprintf(fname,BFSZ-1,"upto_%d.dat",suckid);
	imsg("pull: pull_upto_set id=%d gid=%d %s:%d\n",suckid,gid,group_name(g),v);
	f = fopen(file_work(fname),"r+b");
	if (f==NULL) {
		f = fopen(file_work(fname),"w+b");
		if (f==NULL) {
			filemsg("pull: Unable to write to (%s)\n",file_work(fname));
			return 0;
		}
	}
	if (fseek(f,gid*sizeof(int),SEEK_SET)!=0) {
		filemsg("pull: pull_upto_set seek failed %d\n",gid);
		fclosep(f);
		return FALSE;
	}
	if (fwrite(&v,sizeof(int),1,f)<=0) {
		filemsg("pull: pull_upto_set write %d %d\n",gid,v);
		fclosep(f);
		return FALSE;
	}
	fclosep(f);
	return TRUE;
}
char *lib_pad(char *s,int width)
{
	static char bf[BFSZ];
	int len = strlen(bf);
	if (width>500) width = 500;
	if (len>500) len = 500;
	ncpy(bf,s,BFSZ-3); 
	memset(bf+len,' ',20);
	bf[width] = 0;
	return bf;
}
int pull_link_nbusy(Suck *sk)
{
	int n=0;
	Link *l;
	for (l=links;l!=NULL;l=l->next) {
		if (l->suck!=sk) continue;	
		if (l->state==P_READY) continue;
		if (l->state==P_CLOSED) continue;
		n++;
	}
	return n;

}
char *pull_status(void)
{
	static char out[6000];
	char bf[BFSZ];
	char mb[BFSZ];
	char qued[BFSZ];
	int i,nbsy;
	Suck *sk;

	for (i=0; i<MAX_SUCKS; i++) if (pull_true(i)) goto gotone;
		
	if (!pull_true(0)) return "";

gotone:	lprintf(out,4000,"    Ok    Dup    Rej Qued/Busy/Done    MB/MAX  Upto    Site\r\n");
	for (i=0; i<MAX_SUCKS; i++) {
		sk = &sucks[i];
		if (strlen(sk->site)==0) continue;
		lprintf(bf,BFSZ-1,"%3.2f/%d",sk->bytes/1000000.0,sk->mb);
		ncpy(mb,lib_pad(bf,7),100);
		nbsy = pull_link_nbusy(sk);
		lprintf(bf,BFSZ-1,"%d/%d/%d",sk->nmask,nbsy,sk->ngot);
		strcpy(qued,lib_pad(bf,16));
		lprintf(bf,BFSZ-1,"%6d %6d %6d  %.90s %.90s %8d %.18s %.34s\r\n        %.200s\r\n"
			,sk->nok,sk->ndup,sk->nrej,qued
			,mb,sk->gupto,sk->site,sk->groups,sk->lasterr);
		if (strlen(bf) + strlen(out) > 4000) break;
		ncat(out,bf,4000);
		if (strlen(out)>1900) break;
		
	}
	return out;
}
int pull_link_status(int chan)
{
	char out[BFSZ];
	char bf[BFSZ];
	Link *l;
	Suck *sk;
	if (is_readonly()) return TRUE;
	lprintf(out,BFSZ-1,"     State   Idle   nok  ndup  nrej                     Group Item Site\r\n");
	chan_write_s(chan,out);
	for (l=links;l!=NULL;l=l->next) {
		sk = l->suck;
		if (l->state==P_CLOSED) continue;
		lprintf(bf,BFSZ-1,"%.100s %d",l->gname,l->uptoi);
		lprintf(out,BFSZ-1,"%10s %6d %5d %5d %5d %30s %.80s\r\n"
			,pstates[l->state],lib_gmtime() - l->xtime
			,sk->nok,sk->ndup,sk->nrej,bf,l->suck->site);
		chan_write_s(chan,out);

	}
	return TRUE;

}

static Link *table[1000];
static Link *next[1000];
static int ntable;
void pull_double_reset(void)
{
	Link *l;
	ntable=0;

	return; 
	for (l=links; l!=NULL; l=l->next) {
		table[ntable] = l;
		next[ntable] = l->next;	
		if (ntable>900) {
			emsg("pull: EXITING BECAUSE TOO MANY LINKS FOUND\n");
			dnews_ex("Exiting because double_reset found too many links");
		}
		ntable++;
	}
	imsg("pull: nlinks %d\n",ntable);
	
}
void pull_double_check(char *when)
{
	/* Check our duplicate structures are ok */
	int i;
	Link *l=links;

	return ;  /* We found the bug at last */
	pull_safe_check(when);
	for (i=0; i<ntable; i++) {
		if (table[i]!=l) {
			emsg("Link[%d] changed %s from %p to %p\n",i,when,table[i],l);
			pull_safe_check(when);
			dnews_ex("pull failed");
		}
		if (next[i]!=l->next) {
			emsg("Linknext[%d] changed %s from %p to %p\n",i,when,next[i],l->next);
			pull_safe_check(when);
			dnews_ex("pull failed2");
		}
		l = l->next;
	}
}
static char *safe[1000];
static int nsafe;
void pull_double_grab(void)
{
	char *s;

	return ;
	if (nsafe>900) return;
	s = safe[nsafe++] = mymalloc(sizeof(Link));
	memset(s,'a',sizeof(Link));
	imsg("pull: added safe block safe[%d] = %p\n",nsafe-1,safe[nsafe-1]);
}
void pull_safe_check(char *when)
{
	char *s;
	int k;
	int i;

	return; 

	for (i=0; i<nsafe; i++) {
		s = safe[i];
		if (s==NULL) {
			emsg("pull lost null %s i=[%d] nsafe=[%d] \n",when,i,nsafe);
			dnews_ex("lost nsafe somehow");
		}
		for (k=0; k<sizeof(Link); k++,s++) {
			if (*s!='a') {
				emsg("pull lost %s [%d] [%d] %x %c\n",when,i,k,*s,*s);
				dnews_ex("lost something here");
				s = safe[i];
				for (k=0; k<sizeof(Link); k++,s++) {
					emsg("pull lost [%d] [%d] %x %c\n",i,k,*s,*s);
				}
				dnews_ex("lost something in safety block");
			}
		}
	}
}
void pull_die(void)
{
	char *s;
	s = safe[0];
	if (s!=NULL) s[4] = 'c';
	links->next = (void *) 23;
}