#include "dfeed.h"
enum {F_OPEN,F_MODE,F_SENDING};
static int bkt_total[MAX_DBN];
void xpending_release(Que *q, Flink *l, int line);
#define pending_release(q,f) xpending_release(q,f,__LINE__);

static Feed feeds[MAX_FEED];
static int nfeeds;
static int nscan;
static int maxque;
static time_t cur_tick;
static int trysend;

char *trimit(char *s);
void xpending_release(Que *q, Flink *l, int x)
{
	if (l->cur==q) {
		imsg("Thats odd, releasing que entry I was sending\n");
		l->cur = NULL;
	}
	que_free(q);
}

int feeds_free(void)
{
	Feed *f;
	int i=0;
	for (;i<nfeeds;i++) {
		f = &feeds[i];
		if (f->flog!=NULL) fclose(f->flog);
		f->flog = NULL;
		if (f->fscan!=NULL) fclose(f->fscan);
		f->fscan = NULL;
		links_free(f->links);
		f->links = NULL;
		free(f->site);
		free(f->to);
		free(f->bind);
	}
	nfeeds = 0;
	return TRUE;
}
void links_free(Flink *l)
{
	Flink *next;
	for (;l!=NULL;l=next) { 
		next = l->next;
		chan_release(l->chan);
		chan_close(l->chan);
		fque_free(l);
		free(l);
	}
}
void fque_free(Flink *l)
{
	Que *fq,*next;
	for (fq = l->que; fq!=NULL; fq=next) {
		next = fq->next;	
		que_free(fq);
	}
	l->que = NULL;
}
void feeds_exit(void)
{
	feeds_free();
}
int feeds_load(void)
{
	char type[BFSZ],bind[BFSZ],to[BFSZ],site[BFSZ];
	int nchan;
	char cmd[BFSZ],p1[BFSZ];
	FILE *f;
	char bf[BFSZ];
	char *s;
	int islog;
	/* Read newsfeeds.conf and create feeds structures */

	trysend = atoi(ini_get(I_TRYSEND));

	maxque = atoi(ini_get(I_DFEED_NQUE));
	if (maxque==0) maxque = 15; 

	strcpy(type,"");
	strcpy(to,"");
	strcpy(bind,"");
	islog = FALSE;
	nchan = 1;
	
	f = fopen(file_config("newsfeeds.conf"),"r");
	if (f==NULL) {emsg("Cannot read %s %s\n",file_config("newsfeeds.conf"),strerror(errno)); return FALSE;}
	
	feeds_free();
	for (;!feof(f);) {
		if (fgets(bf,BFSZ-1,f)==NULL) break;
		s = strtok(bf," \t\r\n"); if (s==NULL) continue;
		ncpy(cmd,s,100);
		s = strtok(NULL," \t\r\n"); 
		if (s==NULL) strcpy(p1,"NONE");
		else ncpy(p1,s,100);
		strlwr(cmd);
		if (strcmp(cmd,"site")==0) {
			if (strcmp(type,"dfeed")==0) feeds_load_site(site,to,bind,nchan,islog);
			strcpy(type,"");
			strcpy(to,"");
			strcpy(bind,"");
			nchan = 1;
			islog = FALSE;
			strcpy(site,p1); continue;
		}
		if (strcmp(cmd,"to")==0) ncpy(to,p1,100);
		strlwr(p1);
		if (strcmp(cmd,"type")==0) ncpy(type,p1,100);
		if (strcmp(cmd,"bind")==0) ncpy(bind,p1,100);
		if (strcmp(cmd,"nchan")==0) nchan = atoi(p1);
		if (strcmp(cmd,"log")==0) {islog = TRUE;}
	}
	fclose(f);
	if (strcmp(type,"dfeed")==0) feeds_load_site(site,to,bind,nchan,islog);

	return TRUE;
}


int feeds_load_site(char *site, char *to, char *bind, int nchan, int islog)
{
	Feed *f = &feeds[nfeeds++];

	imsg("feeds_load site (%s) to (%s) bind (%s) nchan %d islog %d\n",site,to,bind,nchan,islog);
	f->site = strdup(site);
	if (strlen(to)>0) f->to = strdup(to);
	else f->to = strdup(site);
	f->bind = strdup(bind);
	f->nchan = nchan;
	f->time_retry = 0;
	f->len = 0;
	f->in = NULL;
	f->links = NULL;
	f->n = nfeeds-1;
	/* Clear stats */
	f->nbytes = 0;
	f->ndup = 0;
	f->fid = -1;
	f->nrej = 0;
	f->nok = 0;
	f->ndel = 0;
	f->islog = islog;
	f->flog = NULL;
	if (nfeeds>100) nfeeds = 100;
	return TRUE;
}
enum {L_NONE, L_CHECK, L_CHECKED, L_SEND,L_SENDING,L_IHAVE};
enum {I_READY,I_SENT};

void link_send_check(Flink *l, Que *q)
{
	char bf[BFSZ];
	if (q->state!=L_CHECK) return;
	if (l->ihave!=I_READY) return;
	if (l->state!=F_SENDING) return;
	l->lastdata = cur_tick;
	if (!l->stream) {
		sprintf(bf,"IHAVE %s\r\n",q->mid);
		/*dmsg("link_send_check %s",bf);*/
		if (strlen(bf) + l->outlen + 10 <BFSZ) {
			strcpy(l->outbf + l->outlen,bf);
			l->outlen += strlen(bf);
			q->state = L_CHECKED;
		}
		l->ihave = I_SENT;
		chan_setsending(l->chan,TRUE);
		return;
	}
	sprintf(bf,"CHECK %s\r\n",q->mid);
	/*imsg("link_send_check %s",bf);*/
	if (strlen(bf) + l->outlen + 10 <BFSZ) {
		strcpy(l->outbf + l->outlen,bf);
		l->outlen += strlen(bf);
		q->state = L_CHECKED;
	}
	chan_setsending(l->chan,TRUE);
}
int link_send_checks(Chan *c, Flink *l)
{
	Que *fq;
	for (fq = l->que; fq!=NULL; fq=fq->next) {
		dmsg("send_checks l->que = %p fq = %p\n",l->que,fq);
		if (fq->state==L_CHECK) link_send_check(l,fq);
	}
	if (l->outlen>0) return TRUE;
	return FALSE;
}
int feeds_write_next(Chan *c, Flink *l)
{
	Que *fq;
	int sentone = FALSE;

	if (l->cur==NULL) {
		link_que_fill(l);
		for (fq=l->que;fq!=NULL;fq=fq->next) {
			if (fq->state==L_SEND) {
				l->cur = fq;
				l->sendtake = 0;
				fq->state = L_SENDING;
				l->sendat = 0;
				chan_setsending(c,TRUE);
				return TRUE;
			}
			if (link_send_checks(c,l)) {
				chan_setsending(c,TRUE);
				sentone = TRUE;
				return TRUE;
			}


		}
		if (sentone) return TRUE;
		if ((l->outlen==0) && (strlen(l->rest)==0)) {
			chan_setsending(c,FALSE);
			return FALSE;
		}
	}
	chan_setsending(c,TRUE);
	return TRUE;
}
static char nulmsg[BFSZ];
static int nnul;
void feeds_write(Chan *c, void *p1)
{
	Flink *l=(Flink *) p1;
	char *data;
	int i;
	char bf[BFSZ];
	int got,total;
	Que *q;
	int rest;
	int r;
	int totsent = 0;
next_one:
	if (l->state != F_SENDING) return;
	imsg_feed(l->feed,l->id);
	if (strlen(l->rest)>0) {
		r = chan_write(l->chan,l->rest,strlen(l->rest));
		if (r<0) {
			imsg("Chan_write failed\n");
			goto ret;
		}
		if (r<(int)strlen(l->rest)) {
			memmove(l->rest,l->rest + r,strlen(l->rest+r)+1);
			goto ret;
		}	
		strcpy(l->rest,"");
	}

	l->lastdata = cur_tick;
	if (!feeds_write_next(c,l)) {
		imsg("Nothing to write to this channel just now\n");
		goto ret;
	}
	if (l->cur==NULL) {
  		if (l->outlen==0) {
			goto ret;
		}
		r = chan_write(l->chan,l->outbf,l->outlen);
		if (r<0) { emsg("Channel closed while writing data\n"); goto ret; }
		if (r < l->outlen) {
			memmove(l->outbf,l->outbf+r,l->outlen-r+1);
			l->outlen -= r;
			goto ret;
		}
		l->outlen = 0;
		goto ret;
	}

	if (!chan_isopen(l->chan)) goto ret;
	data = que_get(l->cur,l->sendat,&got,&total,l->feed);
	if (data==NULL) {
isnull:	
		q = link_que_remove(l,l->cur->mid); 
		l->cur = NULL;	
		if (q!=NULL) pending_release(q,l);  /* Data is not needed by this channel */
		l->sendtake = 0;
		l->feed->ndel++;
		if (!l->stream) {
			r = chan_write(l->chan,"\r\n.\r\n",5);
			goto ret;
		}
		l->ihave = I_READY;
		feeds_write_next(c,l);		
		goto ret;
	}
	for (i=0; i<got; i++) {
		if (data[i]==0) {
			nnul++;
			sprintf(nulmsg,"Null at %d, sendat=%d, got=%d total=%d, %s, bkt=%d offset=%d"
				,i,l->sendat,got,total
				,l->cur->mid,l->cur->bkt,l->cur->offset);
			emsg("%s\n",nulmsg);
			goto isnull;
		}
	}
	/* Get a chunk of this message, with big ones it might not be cached
	if 'x' mb is already cached */

	dmsg("Sending %s %d bytes %d\n",l->cur->mid,got,l->sendtake);
	if (l->sendtake!= -1) {
		sprintf(bf,"TAKETHIS %s\r\n",l->cur->mid);
		rest  =  strlen(bf+l->sendtake);
		r = chan_write(l->chan,bf+l->sendtake,rest);
		if (r<0) {
			emsg("takethis write failed %s\n",tcperror());
			link_close(l,tcperror());
			goto ret;
		}
		if (r<rest) {
			l->sendtake += r;
			goto ret;
		}
		l->sendtake = -1;
	}
	r = chan_write(l->chan,data,got);
/*	if (got>10000) 	r = chan_write(l->chan,data,10000);
	else r = chan_write(l->chan,data,got);*/
	totsent += got;
	if (r<0) { emsg("Channel closed while writing data\n"); goto ret; }
	l->sendat += r;
	l->feed->nbytes += r; /* Bytes sent */
	if (l->sendat >= total) {
		sprintf(bf,".\r\n",3);
		r = chan_write(l->chan,bf,strlen(bf));		
		if (r<0) {
			goto ret;	
		}
		if (r<(int) strlen(bf)) {
			strcpy(l->rest,bf);
		}
		l->cur = NULL;
		l->sendtake = 0;
		feeds_write_next(c,l);
		if (l->cur!=NULL) {
			if (totsent<trysend) {
				imsg("And next one %d/%d sent now\n",totsent,trysend);
				goto next_one;
			}
		}
	}
ret:;
	link_ifempty(l);
	imsg_feed(NULL,0);

}
void link_ifempty(Flink *l)
{
	if (l->que==NULL) link_sendmore(l);
}

void feeds_idle(void)
{
	/* Open channels to outgoing feeds */
	static int last;
	int fi;
	Flink *l;
	time_t t = time(NULL);

	cur_tick = t;
	if (t-last<10) return;
	last = t;

	imsg_flush();
	feeds_idle_open();

	stats_start(S_FEEDS_SCAN);
	for (fi=0; fi<nfeeds; fi++) {
		imsg_feed(&feeds[fi],0);
		feeds_scan(&feeds[fi]);
		for (l=feeds[fi].links;l!=NULL;l=l->next) {
			link_que_fill(l);
		}
		imsg_feed(NULL,0);
	}
	stats_end(S_FEEDS_SCAN);	
	if (lib_exists(file_work("dfeed.exit"))) {
		dfeed_exit();
	}
}
void link_close(Flink *l, char *reason)
{
	time_t t;
	struct tm *xtm;
	char bf[BFSZ];
	t = time(NULL);
	xtm = localtime(&t);


	imsg("%02d:%02d Closing link %.80s\n",xtm->tm_min,xtm->tm_sec,trimit(reason));
	sprintf(bf,"%02d:%02d %s",xtm->tm_min,xtm->tm_sec,trimit(reason));
	fque_free(l);
	if (chan_isopen(l->chan)) chan_close(l->chan);
	ncpy(l->feed->reason,bf,100);
	ncpy(l->feed->problem,bf,100);
	l->inlen = 0;
	l->outlen = 0;
	l->ihave = I_READY;
	l->feed->time_retry = time(NULL) + 10; /* Change to 60*/
	l->nretry = 0;
	l->sendat = 0;
	l->state = F_OPEN;
	l->cur = NULL;
	l->sendtake = 0;
}

Flink *link_new(void)
{
	Flink *l;
	l = malloc(sizeof(Flink));
	memset(l,0,sizeof(Flink));
	return l;
}
void feeds_idle_open(void)
{
	/* Open channels to outgoing feeds */
	int i;
	Feed *f;
	char logname[BFSZ];
	int nlink=0;
	Flink *l;
	int t=time(NULL);
	for (i=0; i<nfeeds; i++) {
		f = &feeds[i];
		imsg_feed(f,0);
		if (f->islog) {
			ncpy(logname,file_feedlog(f->site),200);
			if (f->flog==NULL) {
				f->flog = fopen(logname,"ab");
				emsg("Open log file %s %s\n",logname,strerror(errno));
				dmsg("\n");
				dmsg("\n");
				dmsg("------ DFEED %s Starting -------\n",DFEED_VERSION);
			}
			if (f->flog!=NULL) {
				if (file_size(f->flog)>1000000) {
					fclose(f->flog);
					f->flog = fopen(logname,"wb");
				}
				fflush(f->flog);
			}
		}
		if (f->time_retry>t) continue; 
		if (f->links==NULL) {
		 for (i=0; i<f->nchan; i++) {
			l = link_new();
			l->feed = f;
			l->next = f->links;
			l->id = i;
			f->links = l;
		 }
		}

		for (l=f->links;l!=NULL;l=l->next) {
			imsg_feed(l->feed,l->id);
			nlink++;
			if (l->chan==NULL) l->chan = chan_new();
			if (chan_isopen(l->chan)) continue;
			f->time_retry = t + 10;
			strcpy(l->feed->reason,"Trying to open link");
			chan_open(l->chan,f->to,f->bind);
			chan_setread(l->chan,feeds_read,l);
			chan_setwrite(l->chan,feeds_write,l);
			l->state = F_OPEN;
			l->stream = FALSE;
		}

		for (l=f->links;l!=NULL;l=l->next) {

			imsg_feed(l->feed,l->id);
			if (l->state!=F_SENDING) l->lastdata = cur_tick;
			else {
			 if ((cur_tick - l->lastdata)>120) {
				imsg("Closing idle channel feed %s\n",f->site);
				link_close(l,"Idle");
			 }
			}
		}

	}
	imsg_feed(NULL,0);
}
/* 

  Read data from socket 

*/
void feeds_read(Chan *c, void *p)
{
	int r;
	char bf[BFSZ];
	Flink *l = (Flink *) p;
	Feed *f = l->feed;
	int len;
	int rest;
	char *s;


	imsg_feed(l->feed,l->id);
	
	len = BFSZ-100-l->inlen;
	if (len<0) {
		emsg("Cannot be true %d\n",l->inlen);
		abort();
	}
	r = chan_read(c,l->in+l->inlen,len);
	if (r<=0) {
		if (!chan_isopen(c)) {
			emsg("Channel closed %s\n",tcperror());
			sprintf(bf,"Channel closed %s",tcperror());
			link_close(l,bf);
		}
		goto ret;
	}
	l->lastdata = cur_tick;

	l->inlen += r;
again:
	s = mystrstr(l->in,"\r\n");
	if (s==NULL) {
		if (l->inlen>300) {
			emsg("Thats odd, connection sent 300 char with no end of line\n");
			link_close(l,"Long line");
		}
		goto ret;
	}
	*s = 0;

	rest = l->inlen - ((s+2)-l->in);
	feeds_read_line(c,f,l,l->in);
	if (!chan_isopen(c)) return;
	memmove(l->in,s+2,rest+1);
	l->inlen = rest;
	if (l->inlen>0) goto again;

ret:;
	imsg_feed(NULL,0);
}
char *trimit(char *s)
{
	static char bf[BFSZ];
	ncpy(bf,s,100);
	return bf;
}
void feeds_read_line(Chan *c, Feed *f, Flink *l, char *data)
{
	int code = atoi(data);
	dmsg("Read %s\n",trimit(data));
	switch(l->state) {
 	  case F_OPEN: /* 200, 201, 480, 500... */
		  if ((code>299) || (code<200)) {
			imsg("Open to %s failed %s\n",f->to,trimit(data));
			link_close(l,data);
		  }
		chan_write_s(c,"MODE STREAM\r\n");
		strcpy(l->feed->reason,"Got Welcome");
		l->state = F_MODE;
		break;
	  case F_MODE:
		/* 500 Syntax error*/
		if (code==500) l->stream = FALSE;
		/* 203 StreamOK */
		if (code==203) l->stream = TRUE;
		/* 
			if (code==203) l->stream = FALSE;
			imsg("Debugging, sending in non streaming mode\n");
		*/
		if (l->stream) imsg("Streaming mode set\n");
		else imsg("IHAVE mode set\n");
		l->state = F_SENDING;
		strcpy(l->feed->reason,"Got Stream Response");
		link_send(l);
		feeds_write_next(c,l);
		break;
	  case F_SENDING:
		feeds_read_sending(c,f,l,data);
		break;
	}
}

/*

*/
void link_send(Flink *l)
{
	link_que_fill(l);


}
void feeds_read_sending(Chan *c, Feed *f, Flink *l, char *status)
{
	int code = atoi(status);

	ncpy(l->feed->reason,trimit(status),60);
	
	switch (code) {
	/* Stream Check*/
	  case 438: /* <mid> article not wanted */
		link_dup(l,get_word(status,1));
		break;
	  case 238: /* <mid> yes, send article */
		link_takethis(l,get_word(status,1));
		break;
	/* Stream takethis */
	  case 431: /* <mid> try later  */
		link_later(l,get_word(status,1));
		break;
	  case 439: /* <mid> rejected  */
		link_rej(l,get_word(status,1));
		break;
	  case 239: /* <mid> transferred ok */
		link_ok(l,get_word(status,1));
		break;
	/* Ihave check */
	  case 435: /* Not wanted, duplicate */
		link_ihave_dup(l);
		break;
	  case 335: /* Yes, send it */
		link_ihave_send(l);
		break;
	/* Ihave sent data */
	  case 437: /* Rejected  */
		link_ihave_rej(l);
		break;
	  case 235: /* Transferred ok */
		link_ihave_ok(l);
		break;
	  case 436: /* Paused/full, try later  */
		imsg("Temporary problem, will try later (%s), closing link\n",trimit(status));
		link_later(l,NULL);
		goto paused;
	  default:  /* Misc errors */
		    /* 	4xx close connection and try later	*/
		    /*	  480 Permission denied		*/
		imsg("Bad Status (%s), closing link\n",trimit(status));
paused:		link_close(l,trimit(status));
		f->time_retry = time(NULL) + 60;
		break;
	}

}

Que *link_que_remove(Flink *l, char *mid)
{
	Que *q;
	Que *fq;
	Que **last=&l->que;
	for (fq = l->que; fq!=NULL; fq=fq->next) {
		if (mid==NULL) {
			if (fq->state==L_CHECKED) goto thisone;
			if (fq->state==L_SENDING) goto thisone;
			goto cont;
		}
		if (strcmpnc(mid,fq->mid)==0) {
thisone:		*last = fq->next;
			q = fq;
			l->nque--;			
			return q;
		}
cont:		last = &fq->next;
	}
	if (mid==NULL) mid = "ihave";
	emsg("FATAL ERROR, could not find that entry in the que (%s) \n",mid);
	return 	NULL;
}

void link_rej(Flink *l, char *mid)
{
	Que *q;
	q = link_que_remove(l,mid); if (q==NULL) return ;
	l->feed->nrej++;

	pending_release(q,l);	/* Record that item is no longer needed for this feed */
	link_sendmore(l);
}
void link_dup(Flink *l, char *mid)
{
	Que *q;
	q = link_que_remove(l,mid); if (q==NULL) return ;
	l->feed->ndup++;

	pending_release(q,l);	/* Record that item is no longer needed for this feed */
	link_sendmore(l);
}
Que *link_que_find(Flink *l,char *mid)
{
	Que  *q;
	for (q=l->que;q!=NULL;q=q->next) {
		if (mid==NULL) return q;
		if (strcmp(q->mid,mid)==0) return q;
	}
	return NULL;
}
void link_takethis(Flink *l, char *mid)
{
	Que *q;
	q = link_que_find(l,mid); if (q==NULL) return ;
	q->state = L_SEND; /* Send item when possible */
	chan_setsending(l->chan,TRUE); /* */ 
}
void link_later(Flink *l, char *mid)
{
	Que *q;
	q = link_que_remove(l,mid); if (q==NULL) return ;
	l->feed->nlater++;
	l->nretry++;
	feed_reque(q,l->feed); /* append q entry back in bucket queue file */
	pending_release(q,l);	/* Record that item is no longer needed for this feed */
	link_sendmore(l);
	if (l->nretry>100) {
		emsg("Trying that link later as 100 try later errors occured\n");
		link_close(l,"Told to try laster");
		l->feed->time_retry = time(NULL) + 300;
		l->nretry = 0;
	}
}
void link_ok(Flink *l, char *mid)
{
	Que *q;
	q = link_que_remove(l,mid); if (q==NULL) return;
	l->nretry = 0;
	l->feed->nok++;

	if ((l->feed->nok % 300) == 2) {
		feed_write_upto(l->feed,mid);
	}
	ncpy(l->feed->reason,"Sent Message Ok",100);

	l->feed->nlater = 0;
	pending_release(q,l);	/* Record that item is no longer needed for this feed */
	link_sendmore(l);
}
int link_isfull(Flink *l)
{
	if (l->nque>=maxque) return TRUE;
	return FALSE;
}
void link_sendmore(Flink *l)
{
	dmsg("link_sendmore, %d currently ready to send\n",l->nque);
	if (!l->stream) {
		link_send_checks(l->chan,l);
	}
	if (link_isfull(l)) return;
	if (l->nque> maxque/2) return ; /* So we always get at least 5 */
	/*if (l->wait>time(NULL)) return;*/
	l->wait = time(NULL) + 1;
	link_que_fill(l);
}

void link_ihave_rej(Flink *l)
{
	Que *q;
	q = link_que_remove(l,NULL); if (q==NULL) return;
	l->feed->nrej++;
	l->ihave = I_READY;
	pending_release(q,l);	/* Record that item is no longer needed for this feed */
	link_sendmore(l);
}
void link_ihave_dup(Flink *l)
{
	Que *q;
	q = link_que_remove(l,NULL); if (q==NULL) return;
	l->feed->ndup++;
	l->ihave = I_READY;
	pending_release(q,l);	/* Record that item is no longer needed for this feed */
	link_sendmore(l);
}
void link_ihave_send(Flink *l)
{
	Que *q;
	q = link_que_find(l,NULL); if (q==NULL) return;
	q->state = L_SEND; /* Send item when possible */
	chan_setsending(l->chan,TRUE); /* */ 
}

void link_ihave_ok(Flink *l)
{
	Que *q;
	q = link_que_remove(l,NULL); if (q==NULL) return;
	l->feed->nok++;
	l->ihave = I_READY;
	pending_release(q,l);	/* Record that item is no longer needed for this feed */
	link_sendmore(l);
}

	/* 
	  
		Delete queues for deleted buckets somehow.

		If dbn file changed, then read and figure out
		which buckets no longer exist, and delete
		the queues.
	*/

	/*
Format of bucket feed files ? 
  feed.name <336@netwin.co.nztest.12> test.12:4:5:17600:8800:0
  sitename  mid group:item:bucket:offset:len:spool 
*/

void feeds_scan(Feed *feed)
{
/*
	scan for waiting .send files

	If found rename and start processing.
	
*/
	char sendtmp[BFSZ];
	char sendfile[BFSZ];

	if (feed->fscan!=NULL) return;
	ncpy(sendfile,file_send(feed->site),200);
	ncpy(sendtmp,file_sendtmp(feed->site),200);
	if (lib_exists(sendtmp)) {
again:		imsg("Opening send file %s\n",sendtmp);
		feed->fscan = fopen(sendtmp,"r");
		if (feed->fscan==NULL) {
			emsg("Cannot read (%s) (%s) \n",sendtmp,strerror(errno));
			return;
		} else {
			feeds_scan_upto(feed);
		}
		return;
	}
	if (!lib_exists(sendfile)) return;
	if (rename(sendfile,sendtmp)!=0) { imsg("Rename failed (%s) (%s) (%s)\n",sendfile,sendtmp,strerror(errno)); return; }
	goto again;
}
void feeds_scan_upto(Feed *feed)
{
	int len;
	char bf[BFSZ];
	FILE *f = feed->fscan;
	char *upto = feed_read_upto(feed);
	if (upto==NULL) return;
	len = strlen(upto);

	emsg("Scanning for upto restart mid %.80s\n",upto);
	if (f==NULL) return;
	for (;!feof(f);) {
		if (fgets(bf,BFSZ-10,f)==NULL) break;
		if (strncmp(bf,upto,len)==0) {
			emsg("Found restart point %.80s\n",upto);
			return;
		}
	}
	fseek(f,0,SEEK_SET);
	emsg("Couldn't find restart point %.80s\n",upto);
}
int send_read(FILE *f, char *mid, int *bkt, int *offset, int *len, int *spool, int *pile)
{
	char xbf[BFSZ];
	char *s;
skipline:
	if (feof(f)) return FALSE;
	if (fgets(xbf,200,f)==NULL) return FALSE;
	/*mid*/ s = strtok(xbf," "); if (s==NULL) return FALSE;
	ncpy(mid,s,100);
	if (strlen(s)>100) return FALSE;
	/*grp*/ s = strtok(NULL,":"); if (s==NULL) return FALSE;
	/*n  */ s = strtok(NULL,":"); if (s==NULL) return FALSE;
	/*bkt*/ s = strtok(NULL,":"); if (s==NULL) return FALSE;
	*bkt = atoi(s);
	/*offset*/ s = strtok(NULL,":"); if (s==NULL) return FALSE;
	*offset = atoi(s);
	/*len*/ s = strtok(NULL,":"); if (s==NULL) goto skipline;
	*len = atoi(s);
	/*spool*/ s = strtok(NULL,":"); if (s==NULL) goto skipline;
	*spool = atoi(s);
	/*pile*/ s = strtok(NULL,":"); if (s==NULL) goto skipline;
	*pile = atoi(s);
	nscan++;
	return TRUE;
}


void feed_reque(Que *q, Feed *feed)
{
	FILE *f;
	f = fopen(file_send(feed->site),"a");
	if (f==NULL) return;
	fprintf(f,"%s %s:%d:%d:%d:%d:%d:%d\n",q->mid,"none",1,q->bkt,q->offset,q->len,q->spool,q->pile);
	fclose(f);
}



void link_add_msg(Flink *l,Que *q)
{
	Que *old = link_que_find(l,q->mid);
	if (old!=NULL) {
		imsg("Ignoring duplicate entry in feed file %s\n",q->mid);
		return;
	}
	q->next = l->que;
	l->que = q;
	q->state = L_CHECK;
	link_send_check(l,q);
	l->nque++;
}
void link_que_fillx(Flink *l);
void link_que_fill(Flink *l)
{
	stats_start(S_QUE_FILL);
	link_que_fillx(l);
	stats_end(S_QUE_FILL);	
}
void link_que_fillx(Flink *l)
{
	Que *q;
	if (l->state!=F_SENDING) return;
	for (;!link_isfull(l);) {
		q = link_next(l);
		if (q==NULL) break;
		link_add_msg(l,q);
	}
}
void feed_write_upto(Feed *feed,char *mid)
{
	FILE *f;
	f = fopen(file_sendupto(feed->site),"w");
	if (f==NULL) return;
	fprintf(f,"%s\n",mid);
	fclose(f);
}
char *feed_read_upto(Feed *feed)
{
	FILE *f;
	char *s;
	static char bf[BFSZ];
	f = fopen(file_sendupto(feed->site),"r");
	if (f==NULL) return NULL;
	fgets(bf,200,f);
	fclose(f);
	s = strchr(bf,'\n');
	if (s!=NULL) *s = 0;
	return bf;
}
Que *link_next(Flink *l)
{
	int r;
	Que *q;
	int bkt,offset,len,spool,pile;
	char mid[BFSZ];
	if (l->feed->fscan==NULL) return NULL;
again:
	r = send_read(l->feed->fscan,mid,&bkt,&offset,&len,&spool,&pile);
	if (!r) {
		if (feof(l->feed->fscan)) {
			fclose(l->feed->fscan); l->feed->fscan = NULL;
			remove(file_sendupto(l->feed->site));
			if (remove(file_sendtmp(l->feed->site))!=0) {
				emsg("Couldn't remove (%s) (%s) \n",file_sendtmp(l->feed->site),strerror(errno));
			}
			feeds_scan(l->feed);
		} else goto again;
		return NULL;
	}
	l->feed->nscan++;
	q = malloc(sizeof(Que));
	dmsg("Malloc que %p\n",q);
	memset(q,0,sizeof(q));
	q->mid = strdup(mid);
	q->bkt = bkt;
	q->offset = offset;
	q->len = len;
	q->spool = spool;
	q->pile = pile;
	return q;
}
void que_free(Que *q)
{
	if (q->mid!=NULL) free(q->mid);
	q->mid = NULL;
	dmsg("Free q=%p\n",q);
	free(q);
}
char *file_bucket(Que *q)
{
	static char bf[BFSZ];
	if (q->pile==0) {
	  sprintf(bf,"%s/db_%d.itm",file_spool(q->spool),q->bkt);
	} else {
	  sprintf(bf,"%s/db_%d_%d.itm",file_spool(q->spool),q->bkt,q->pile);
	}
	return bf;
}
#define HEAD_LEN 28
/*
~00008761:0000000017:000958:
^
*/
char *show_ten(char *s)
{
	static char bf[BFSZ];
	ncpy(bf,s,20);
	return bf;
}
char *xque_get(Que *q, int from, int *got, int *total, Feed *feed);
char *que_get(Que *q, int from, int *got, int *total, Feed *feed)
{
	char *s;
	stats_start(S_XREAD);	
	s = xque_get(q,from,got,total,feed);
	stats_end(S_XREAD);	
	return s;
}
char *xque_get(Que *q, int from, int *got, int *total, Feed *feed)
{
	int r;
	char bf[BFSZ];
	static char *data;
	static int datalen;
	int at_1,at_2,diff;
	static char lastmid[BFSZ];
	FILE *f;
	int len = q->len - HEAD_LEN - 1;
	
	if (datalen==0) {
		datalen = atoi(ini_get(I_ITEM_SIZE));
		if (datalen<1000000) datalen = 1000000;
		datalen += 10000;
		data = malloc(datalen);
	}
	if (len+1000>datalen) return NULL;

	if (strcmp(lastmid,q->mid)!=0) {	
		stats_start(S_OPEN);
		f = bfile_open(q);
		stats_end(S_OPEN);
		if (f==NULL) {
			imsg("Bucket file deleted %s %s\n",file_bucket(q),strerror(errno));
			goto failed;
		}
		stats_start(S_FSEEK);

		at_1 = ftell(f);
		at_2 = q->offset+HEAD_LEN-1;
		diff = at_2 - at_1;
		if ((diff>0) && (diff<BFSZ)) {
		 if ((int)fread(bf,1,diff,f)!=diff) {
			emsg("Could not read to item bkt=%d offset=%d %s\n",q->bkt,q->offset+HEAD_LEN-1,strerror(errno));
			goto failed;			 
		 }
		} else {
		 if (fseek(f,at_2,SEEK_SET)!=0) {
			emsg("Could not seek to item bkt=%d offset=%d %s\n",q->bkt,q->offset+HEAD_LEN-1,strerror(errno));
			goto failed;
		 }
		}	
		stats_end(S_FSEEK);
		stats_start(S_FREAD);	
		r = fread(data,1,len+2,f);
		stats_end(S_FREAD);	
		data[len+2] = 0;
		if (r!=len+2) goto failed;
		if (data[0]!=':') {emsg("Data did not start with a colon (%s)\n",show_ten(data)); goto failed;}
		if (data[len+1]!='^') {emsg("Data did not end with a hat (%s)\n",show_ten(data - 20)); goto failed;}
		data[len+1] = 0;
	} 
	ncpy(lastmid,q->mid,200);
	dmsg("que_get %s\n",q->mid);
	*total = len;
	*got = len - from; /* r - from;*/
	return data + from + 1;
failed:
	return NULL;
}
static FILE *bfile;
static int bfile_bkt;
FILE *bfile_open(Que *q)
{
	static char bbf[64000];
	if (bfile_bkt!=q->bkt) {
		if (bfile!=NULL) fclose(bfile);
		bfile = NULL;
		bfile_bkt = q->bkt;
	}
	if (bfile!=NULL) return bfile;
	bfile = fopen(file_bucket(q),"rb");
	if (bfile!=NULL) setvbuf(bfile,bbf,_IOFBF,32767);
	dmsg("Open %s\n",file_bucket(q));
	return bfile;
}
char *pad_left(char *s, int n)
{
	static char bf[BFSZ];
	int add = n - strlen(s);
	if (add<1) {strcpy(bf,s); return bf;}
	strcpy(bf,"                   ");
	strcpy(bf+add,s);
	return bf;
}

static int start_time;
void stats_reset(void)
{
	start_time = time(NULL);
}
void feeds_status(FILE *fout)
{
	int i;
	Feed *f;
	char fsize[BFSZ];
	static int up_time;
	int fsz,fsz2;
	int total_time;
	double tsentk,tsentn;
	Flink *l;
	
	if (start_time==0) start_time = time(NULL)-1;
	if (up_time==0) up_time = time(NULL)-1;
	total_time = time(NULL) - start_time;

	tsentk = 0;
	tsentn = 0;

	fprintf(fout,"    Sent      Dup      Rej     Lost    File  Sitename\n");
	fprintf(fout,"   Bytes      K/s  Items/s  Scanned    OK/s\n");
	for (i=0; i<nfeeds; i++) {
		f = &feeds[i];
		fsz = lib_filesize(file_feed(f->site));
		fsz2 = lib_filesize(file_send(f->site));
		fsz2 += lib_filesize(file_sendtmp(f->site));
		if (fsz2>1000000) {
			sprintf(fsize,"%d/%dM",fsz2/1000000,fsz/1000000);
		} else {
			sprintf(fsize,"%d/%dk",fsz2/1000,fsz/1000);
		}
		strcpy(fsize,pad_left(fsize,7));
		fprintf(fout,"%8d %8d %8d %8d  %s %s\n%8.0fmb %6.0fk/s %5.1f/s %6d   %5.1f/s\n---------->%s\n---------->%s\n\n"
			,f->nok,f->ndup,f->nrej,f->ndel,fsize,f->site
			,f->nbytes/1000000.0,f->nbytes/1000.0/total_time
			,(double) f->nok/total_time
			,f->nscan,(double) (f->nok+f->ndup+f->nrej+f->ndel)/total_time
			,f->reason,f->problem);
		tsentk += f->nbytes/1024;
		tsentn += f->nok;
		for (l=f->links;l!=NULL;l=l->next) {
			fprintf(fout,"    Link [%d] %d queued\n",l->id,l->nque);
		}

	}
	fprintf(fout,"Number scanned from que files %d\n",nscan);
	fprintf(fout,"Total sent %.0fmb %.0fkbytes/s, Total items sent ok %.0fk %3.1f/s\n",
		tsentk/1024,tsentk/total_time,tsentn/1000,tsentn/total_time); 
	if (nnul>0) {
		imsg("Nulls found %d %s\n",nnul,nulmsg);
	}
	fprintf(fout,"Uptime %d days %2.2f hours\n"
		,(time(NULL) - up_time)/(60*60*24)
		,(double) (time(NULL) - up_time)/(60*60));

	
}


