/*
	TEST upto on feed is correct.
	TEST timeout
	TEST prefetch is never repeated. even for empty non cached group

	Work in a dmulti configuration.

  The main problem with this is expiration? 
  For each header sucked group we need an index of
       itemupstream ---> local item
  And when expire litgroiup is sent we can then figure out
  which items to cancel locally. 


--------------------------------------------------
	test normal non postonly setting
	test normal c_postonly setting stil works.
	test post_aging 
--------------------------------------------------
    Test system and see if really works.


  
    6	tellnews command to expire all messages in header_groups and remember set 0

  
    Make it work in dmulti mode too
	(e.g. super proxy server)
  Sucking Just headers, then bodies on demand.
tick	If group matches list, then 
tick		suck headers only
tick		re-arrange headers so group we are reading is primary group
tick		Don't feed out at all.
		Use replicate mode while sucking the specified group.
	When body/article requested, 
tick		if (group) matches (wildcard)
tick		Search cache
tick		Request it
			(if doesn't exist send listgroup and expire to match)
		Use 'n' streams
tick		Send it 'as it is written' to any requesting channels.
		Cache is controlled with a separate setting 'x' mb's

dnews.conf settings
	header_groups *.binaries.*,*.warez.*
	header_chan_n 3		# Retrieve 3 items at once as a maximum.
	header_expire 45 8	# Expire items at 8:45
	header_body_mb 100 # 100 mb	   
	header_body_path # defaults to workarea

Hash directory structure, e.g.
	cache/nm/c_nmo.txt


*/

#include <stdlib.h>
#include <stdio.h>
#include <ctype.h>
#include <string.h>
#include <errno.h>
#ifdef NeXT
#include <sys/time.h>
#endif 
#include "group.h"
#include "item.h"
#include "dnews.h"
#include "nntp.h"
#include "hist.h"
#include "suck.h"
#include "chan.h"
#include "sary.h"
#include "str.h"
#include "lib.h"
#include "db.h"
#include "feed.h"
#include "expire.h"
#include "ncmd.h"
#include "header.h"

static double bused;
static int totused;
void headmsg(int id,int doretry, char * arg_list, ...);

void vstrdup(char **dst, char *src)
{
	if (*dst!=NULL) mystrfree(*dst,SD_HEADER);
	*dst = mystrdup(src,SD_HEADER);
}

int ishdr_new(void)
{
	return config_true(C_HEADER_NEWNUMBERS);
}
int ishdr_old(void)
{
	return !config_true(C_HEADER_NEWNUMBERS);
}
int ishdr(void)
{
	if (strlen(config_get(C_HEADER_GROUPS))==0) return FALSE;
	return TRUE;	
}
int hdr_matchgroup(char *gname)
{
	if (strlen(config_get(C_HEADER_GROUPS))==0) return FALSE;
	return group_match(config_get(C_HEADER_GROUPS),gname);
}
enum {S_FREE,S_REMOTE,S_LOCAL,S_COMING,S_FAILED
};
typedef struct HDR {int state; int gid; int num; int lastused; int size; char *etxt; int doretry; int tasked; } Hdr;


static Hdr *hc;
static int maxmb,maxhc,nhc;
void hcache_init(void) /*  scan cache and rebuild internal index. */
{
	maxmb = atoi(config_get(C_HEADER_BODY_MB));
	if (maxmb==0) maxmb = 20;
	imsg("header: maxmb=%d %s\n",maxmb,config_get(C_HEADER_BODY_MB));
	maxhc = maxmb*200;  
	if (maxhc>10000) maxhc = maxmb*10;
	if (maxhc>100000) maxhc = 100000;
	if (atoi(config_get(C_HEADER_BODY_N))>0) maxhc = atoi(config_get(C_HEADER_BODY_N));
	hc = mymalloc(sizeof(Hdr) * maxhc);
	memset(hc,0,sizeof(Hdr)*maxhc);
	
	hcache_read(); /* Read hcache index into memory */
}
#define FILE_BODIES_OLD file_work("bodies.idx")
#define FILE_BODIES file_work("bodies2.idx")
int hremove(char *fname, char *reason)
{
	int r;
	r = remove(fname);
	if (r==0) keeplog("Removed (%s) reason (%s) \n",fname,reason);
	return r;
}
int hcache_read(void)
{
	int i;
	FILE *f;
	bused = 0;

	if (lib_exists(FILE_BODIES_OLD)) {
		remove(FILE_BODIES_OLD);
		hcache_rebuild();
		return TRUE;
	}
	f = fopen(FILE_BODIES,"rb");
	if (f==NULL) return FALSE;
	fread(hc,sizeof(Hdr),maxhc,f);
	fclosep(f);
	for (i=0; i<maxhc; i++) {
		hc[i].etxt = NULL;
		if (hc[i].state==S_LOCAL) {
			totused++;
			bused += hc[i].size;
			nhc = i+1;
		} else if (hc[i].state!=S_FREE) {
			if (remove(hcache_fname(i))==0) keeplog("Removed (%s) in hcache_read\n",hcache_fname(i));
			hc[i].state = S_FREE;
			dmsg("hcache: Cleaning out failed cache items %s\n",hcache_fname(i));
		}
	}
	return TRUE;
}
int hcache_update(int i) 
{
	FILE *f;
	if ((i<0) || (i>=maxhc)) return FALSE;
	f = fopen(FILE_BODIES,"r+");
	if (f==NULL) {
		f = fopen(FILE_BODIES,"w+");
	}
	if (f==NULL) return FALSE;
	fseek(f,i*sizeof(Hdr),SEEK_SET);
	fwrite(&hc[i],sizeof(Hdr),1,f);
	fclosep(f);
	return TRUE;
}


int hcache_add(int gid, int num)
{
	int i;
	if (hc==NULL) dnews_ex("hcache not initialized");

	for (i=0; i<nhc; i++) {
		if (hc[i].state==S_FREE) goto found;	
	}
	if (nhc>=maxhc) {
		emsg("FATAL, HCACHE_ADD RAN OUT OF ITEMS, BUG MUST EXIST, REBUILDING CACHE INDEX \n");
		hcache_rebuild();
		hcache_trim(0);
		for (i=0; i<nhc; i++) {
			if (hc[i].state==S_FREE) goto found;	
		}
		return 0;
	}
	i=nhc++;
found:	
	fimsg("hcache[%d] allocated %d:%d   %d %d\n",i,gid,num,nhc,maxhc);
	totused++;
	hc[i].state = S_REMOTE;
	hc[i].gid = gid;
	hc[i].num = num;
	imsg("hcache[%d] allocated %d:%d\n",i,hc[i].gid,hc[i].num);
	return i;
}
int hcache_num(int id)
{
	if ((id<0) || (id>=maxhc)) return 0;
	return hc[id].num;
}
char *hcache_fname(int id)
{
	char path[BFSZ];
	static char bf[BFSZ];
	ncpy(path,config_get(C_HEADER_PATH),BFSZ-1);
	if (strlen(path)==0) ncpy(path,config_get(C_WORKAREA),BFSZ-1);
	lprintf(bf,BFSZ-1,"%s/%d/b_%d.itm",path,id/100,id);
	return bf;
}
void hcache_free(int id)
{
	if ((id<0) || (id>=maxhc)) return;
	if (hc[id].state==S_COMING) return;
	if (hc[id].state==S_FREE) return;
	hc[id].state = S_FREE;
	if (hc[id].etxt!=NULL) mystrfree(hc[id].etxt,SD_HEADER);
	hc[id].etxt = NULL;
	totused--;
	bused -= hc[id].size;
	hremove(hcache_fname(id),"hcache_free");
	hcache_update(id);
}
void hcache_trim(int x)
{
	int i;
	static int nextf;
	if (totused+10<maxhc) if (((bused*1.05/1000000) < maxmb)) return;
	for (i=0; (totused+20>maxhc) || ((bused*1.1/1000000) > maxmb);i++) {
		if (i>5000000) {
			hcache_rebuild();
			break;
	
		}
		if (hc[nextf].state == S_FREE) goto nextone;
		if (nextf==x) goto nextone;
		imsgf("Header: cache trimming item [%d] %d bytes\n",nextf,hc[nextf].size);
		hcache_free(nextf);
nextone:	nextf++;
		if (nextf>=maxhc) nextf = 0;
	}
	
}
static int s_nread,s_fetch,s_readk,s_fetchk;
int hcache_findadd(int gid, int num)
{
	int i = hcache_find(gid,num);

	s_nread++;
	if (i<0) i = hcache_add(gid,num);
	if ((i<0) || (i>=maxhc)) return 0;

	if ((hc[i].state!=S_LOCAL) && (hc[i].state!=S_COMING)) {			        
		imsg("header: requesting article [%d] %d:%d\n",i,gid,num);
		s_fetch++;
		hque_item(i);
		hcache_update(i);
		hcache_trim(i); /* Delete items if necessary to keep cache down to size*/
	}
	hc[i].lastused = lib_gmtime();
	return i;
}

int hcache_find(int gid, int num)
{
	int i;
	for (i=0; i<nhc; i++) {
		if (hc[i].gid==gid) if (hc[i].num==num) {
			return i;
		}
	}
	return -1;
}

/* Must work in dmulti configuration */ 

enum {Q_ITEM,Q_LIST,Q_SUCK};
typedef struct HQUE {int id; int type; int chan;  } Hque;
#define MAX_QUE 20000
static Hque que[MAX_QUE];
static int nque;

int hque_item(int id)	/* Que fetch of this item */
{
	if ((id<0) || (id>=maxhc)) return FALSE;
	if (nque+100>MAX_QUE) {
		emsg("Cannot fetch item, body queue is full\n");
		headmsg(id,TRUE,"Body queue is full");
		return FALSE;
	}
	hc[id].state = S_COMING;
	hc[id].tasked = lib_gmtime(); 
	que[nque].type = Q_ITEM;
	que[nque++].id = id;
	hcache_trim(id); /* Make sure we have enough space */
	imsg("header: queued request for item [%d]  id=%d \n",nque,id);
	hremove(hcache_fname(id),"hque_item_old_one"); /* Safety remove of old item */
	return TRUE;
}
int hque_suck(int grp, int chan )	/* Que a list of this group*/
{
	if (nque+100>MAX_QUE) {
		emsg("Cannot fetch group, body queue is full\n");
		return FALSE;
	}
	if (grp<0) {
		emsg("Cannot fetch group as grp is -1\n");
		return FALSE;
	}
	que[nque].type = Q_SUCK;
	que[nque].chan = chan;
	que[nque++].id = grp;
	return TRUE;
}
void hque_pop(void)
{
	int i;
	imsg("Ppopped item off queue %d\n",nque);
	for (i=0; i<nque; i++) {
		que[i] = que[i+1];
	}
	nque--;
	if (nque<0) nque = 0;
}
/* Open channels to fetch items if necessary  & timeouts etc*/
enum {S_CLOSED,S_IDLE,S_OPEN,S_MODE,S_USER,S_PASS
	,S_GROUP1,S_GROUP2,S_ARTICLE1,S_ARTICLE2,S_ARTICLE3
	,S_SUCK_START,S_SUCK_GROUP,S_SUCK_BLOCK
	,S_SUCK_HEAD,S_SUCK_HEAD_DATA,S_SUCK_NEXT
};
static char *hchan_state[]= 
{"S_CLOSED","S_IDLE","S_OPEN","S_MODE","S_USER","S_PASS","S_GROUP1","S_GROUP2","S_ARTICLE1","S_ARTICLE2","S_ARTICLE3","XX","XX","XX","XX","XX","XX","XX","XX","XX","XX","XX","XX","XX","XX","XX","XX","XX","XX","XX","XX","XX","XX","XX","XX","XX"};
enum {T_BODY, T_GROUP};
typedef struct HCHAN {int chan; int start,fromchan,block,ignore; int state,next,id,grp,got,itemi,type; FILE *f;} Hchan;
#define MAX_HCHAN 100
static Hchan hchan[MAX_HCHAN];
static int nchan,nopen,h_hosti;
static char h_user[BFSZ],h_pass[BFSZ],h_host[BFSZ];
int hdr_nchan(void);
int hdr_nchan(void)
{
	int n = atoi(config_get(C_HEADER_CHAN_N));
	if (n==0) return 1;
	return n;
}
void hque_bkgnd(void)
{
	int i;
	int h;
	char host[BFSZ];
	if (nque==0) return;
	dmsg("hque: Hque_background\n");
	for (i=0; i<nchan; i++) if (hchan[i].state==S_IDLE) goto found;
	/* If no free channels and max open, return */
	if (nopen>=hdr_nchan()) return;
	/* Open a new channel */	
	for (i=0; i<nchan; i++) if (hchan[i].state==S_CLOSED) break;
	if (i==nchan) nchan++;
	hchan[i].state = S_OPEN;
	h = atoi(config_get(C_HEADER_HOST));
	h_hosti = h;
	suck_info(h,host,h_user,h_pass); strcpy(h_host,host);
	hchan[i].chan = chan_open(host);

	imsg("header: open connection to (%s) \n",host);
	if (hchan[i].chan<0) {
		emsg("Unable to open link to header host %s\n",host);
		return;
	}
	chan_setfn(hchan[i].chan,hchan_data);
	chan_settimeout(hchan[i].chan,200);
	nopen++;
				
		
found:	/* Start request*/

	if (que[0].type==Q_SUCK) {
		hchan[i].next = S_SUCK_START;		
		hchan[i].fromchan = que[0].chan;
		imsg("suck group %d %s\n",que[0].id,group_name(que[0].id));
		hchan[i].grp = que[0].id;
		hchan[i].type = T_GROUP;
	} else {
		hchan[i].next = S_GROUP1;
		hchan[i].id = que[0].id;
		hchan[i].type = T_BODY;
	}
	hque_pop();
	if (hchan[i].state== S_IDLE) {
		hchan[i].state = hchan[i].next;
		hchan_data(hchan[i].chan,"",0);
	}
}
#define SUCK_BLOCK 5
int hdr_prefetch(void);
int hdr_prefetch(void)
{
	int x;
	x = atoi(config_get(C_HEADER_PREFETCH));
	return x;
}
int hdr_suck_timeout(void)
{
	int x = atoi(config_get(C_HEADER_TIMEOUT));
	if (x==0) x = 20;
	return x;
}
int hchan_find(int chan)
{
	int i;
	for (i=0; i<nchan; i++) if (hchan[i].chan==chan) return i;
	return -1;	
}
int header_logstuff(int n)
{
	FILE *f;
	String *str;

	
	f = fopen(file_log("header.log"),"w");
	if (f==NULL) return TRUE;
	fprintf(f,"After closing failed channel I have %d open\n",n);
	str = dnews_status();	
	fwrite(str_s(str),1,str_len(str),f);
	fclosep(f);
	return TRUE;
}

int hchan_data(int chan, char *xin, int len) /* Process incoming data from channels */
{
	int i;
	char bf[BFSZ];
	String *get;
	char *x;
	int n;
	int isheader;
	int first,last;
	int hh = hchan_find(chan);
	int status=atoi(xin);
	int id,grp,g;
	int isbody;

	if (len>0) xin[len] = 0;
	if (hh<0) {
		emsg("Data for non existent header channel\n");
		return FALSE;
	}
	id = hchan[hh].id;
	grp = hchan[hh].grp;
	isbody = (hchan[hh].type==T_BODY);
	if (isbody) if ((id<0) || (id>=maxhc)) {
		emsg("Data for non existent cache entry\n");
		return FALSE;
	}
	if (!isbody) if ((grp<0) || (grp>=group_n())) {
		emsg("Can't fetch group grp=%d\n",grp);
		return FALSE;
	}

again:
	if (hchan[hh].state!=S_ARTICLE3) 
	  if (hchan[hh].state!=S_SUCK_HEAD_DATA)
		imsg("header: hchan_data(%d,%s) state=%s\n",chan,nocrlf(xin),hchan_state[hchan[hh].state]);
	if (len<0) {
		if (hchan[hh].state == S_CLOSED) return FALSE;
		hchan[hh].state = S_CLOSED;
		nopen--;
		if (isbody && ((id>=0) && (id<maxhc))) {
			headmsg(id,TRUE,"Cannot open connection to (%s)",h_host);
		}

		chan_close(chan);
		header_logstuff(nopen);
		return TRUE;
	}
	switch (hchan[hh].state) {
	case S_IDLE:  /* Waiting for something to do */ 
		break;
	case S_OPEN:
		if (status>399) {  /* Open failed */
			emsg("head: Bad open status (%s)\n",nocrlf(xin));
			chan_close(chan); chan = -1;
			nopen--;
			if (hchan[hh].next==S_SUCK_START) goto suck_failed;
			hchan[hh].state = S_CLOSED;
			if (isbody && ((id>=0) && (id<maxhc))) {
			 headmsg(id,TRUE,"Upstream site down (%s)",nocrlf(xin));
			}
			return TRUE;
		}
		imsg("header: Successfully connected to server - %s\n",nocrlf(xin));
		chan_write_s(chan,"MODE READER\r\n");
		hchan[hh].state = S_MODE;
		break;
	case S_MODE:
		/* Ignore response, we don't care if its ok or not.*/
		if (strlen(h_user)>0) {
			imsg("header: Sending username (%s)\n",h_user);
			lprintf(bf,BFSZ-1,"AUTHINFO USER %s\r\n",h_user); chan_write_s(chan,bf);			
			hchan[hh].state = S_USER;
			break;
		}
		hchan[hh].state = hchan[hh].next;
		goto again;
	case S_USER:
		imsg("header: Sending password\n");
		lprintf(bf,BFSZ-1,"AUTHINFO PASS %s\r\n",h_pass); chan_write_s(chan,bf);			
		hchan[hh].state = S_PASS;
		break;
	case S_PASS:
		imsg("header: authinfo response %s\n",nocrlf(xin));
		hchan[hh].state = hchan[hh].next;
		goto again;
	case S_GROUP1:
		imsg("Fetching group  [%d] gid=%d (%s)\n",id,hc[id].gid,group_name_gid(hc[id].gid));
		lprintf(bf,BFSZ-1,"GROUP %s\r\n",group_name_gid(hc[id].gid)); chan_write_s(chan,bf);
		hchan[hh].state = S_GROUP2;
		break;
	case S_GROUP2:
		if (status>300) {
			emsg("head: Bad group status (%s)\n",nocrlf(xin));
			chan_close(chan); chan = -1;
			nopen--;
			hchan[hh].state = S_CLOSED;
			if (isbody && ((id>=0) && (id<maxhc))) {
			   headmsg(id,TRUE,"Upstream group not available (%s)",nocrlf(xin));
			}
			return TRUE;
		}
		/* 211 num first last group.name ...*/
		first = atoi(get_word(xin,2));
		last = atoi(get_word(xin,3));
		if (isbody && ((id>=0) && (id<maxhc))) 
			hdr_fixrange(hc[id].gid,first,last); /* Remove items outside range */
		hchan[hh].state = S_ARTICLE1;
		goto again;
	case S_ARTICLE1:
		imsg("Requesting article [%d](%s:%d)\n",id,group_name_gid(hc[id].gid),hc[id].num);
		if ((id<0) || (id>=maxhc)) {emsg("id out of ranged %d\n",id); break;}
		lprintf(bf,BFSZ-1,"ARTICLE %d\r\n",hc[id].num); chan_write_s(chan,bf);
		hchan[hh].state = S_ARTICLE2;		
		break;
	case S_ARTICLE2:
		if (status>299) { 
			imsg("head: Bad article response (%s)\n",nocrlf(xin)); 
			if ((status>399) && (status<500)) {
				imsg("head: Deleting remote expired article from local index\n");
				if ((id<0) || (id>=maxhc)) {emsg("id out of ranged2 %d\n",id); goto fail_item;}

				g = gid_to_group(hc[id].gid);
				if (g>=0) item_delete_all(-1,g,hc[id].num);
				headmsg(id,FALSE,"Expired on upstream machine (%s)",nocrlf(xin));
			} else {
				headmsg(id,TRUE,"Error reading body from upstream site (%s)",nocrlf(xin));
				emsg("head: Error reading upstream item %s\n",nocrlf(xin));
			}
			goto fail_item;
		}
		hchan[hh].f = myfopen(hcache_fname(id),"wb");
		if (hchan[hh].f==NULL) {
			if ((id<0) || (id>=maxhc)) {goto fail_item;}
			lib_createdir(hcache_fname(id));
			hchan[hh].f = myfopen(hcache_fname(id),"wb");
		}
		if (hchan[hh].f==NULL) { 
			filemsg("Could not write to %s\n",hcache_fname(id)); 
			headmsg(id,TRUE,"Error writing cache file");
			goto fail_item;
		}
		hchan[hh].state = S_ARTICLE3;	
		break;
	case S_ARTICLE3: /* Storing data */
		if (((id>=0) && (id<maxhc))) { 
		hc[id].tasked = lib_gmtime(); /* don't timeout if data is arriving */ 

		if (strcmp(xin,".\r\n")==0) {
			hc[id].state = S_LOCAL;
			hc[id].size = file_length(fileno(hchan[hh].f));
			keeplog("Successfully fetched %s %d bytes\n",hcache_fname(id),hc[id].size);
			s_fetchk += hc[id].size/100;
			bused += hc[id].size;
			fclosep(hchan[hh].f); hchan[hh].f = NULL;
			hcache_update(id);
			hchan[hh].state = S_IDLE;
			ncmd_hdr_send(id);
			break;
		}
		n = fwrite(xin,1,strlen(xin),hchan[hh].f);
		if (n<(int) strlen(xin)) { 
			headmsg(id,TRUE,"Error writing cache file");
			filemsg("Error writing to cache file\n"); goto fail_item;
		}
		hchan[hh].got += n;
		if (hchan[hh].got>2000) {
			ncmd_hdr_send(id);
			hchan[hh].got = 0;
		}
		}
		break;		
/*-- Sucking ------------------------------------------------------------*/
	case S_SUCK_START: /*  Suck first 'n' items for this group */
		imsg("hdrsuck: Suck start for group %s %d\n",group_name(grp),grp);
		big_init(chan);
		hchan[hh].start = lib_gmtime();
		lprintf(bf,BFSZ-1,"GROUP %s\r\n",group_name(grp)); chan_write_s(chan,bf);
		hchan[hh].state = S_SUCK_GROUP;
		hchan[hh].got = 0;
		break;		
	case S_SUCK_GROUP: 
		imsg("hdrsuck: group response (%s) \n",nocrlf(xin));
		if (status>299) { emsg("Group response %s\n", nocrlf(xin)); goto suck_failed; }
	case S_SUCK_BLOCK: /*  Suck first 'n' items for this group */
next_block:	
		imsg("hdrsuck: Ask for BLOCK items\n");
		hchan[hh].block = 0;
		hchan[hh].ignore = FALSE;
		isheader = hdr_matchgroup(group_name(grp));

		for (i=0; i<SUCK_BLOCK; i++) {
			lprintf(bf,BFSZ-1,"%s\r\n",isheader ? "HEAD" : "ARTICLE" ); chan_write_s(chan,bf);
			lprintf(bf,BFSZ-1,"NEXT\r\n"); chan_write_s(chan,bf);
		}
		hchan[hh].state = S_SUCK_HEAD;
		break;		
	case S_SUCK_HEAD:
		imsg("hdrsuck: head response (%s) \n",nocrlf(xin));
		big_init(chan);
		if (status<200 || status>499) { emsg("head response %s\n", nocrlf(xin)); goto suck_failed; }
		if (status>399) { /* Missing item, ignore */
			hchan[hh].state = S_SUCK_NEXT;
			break;
		}
		imsg("hdrsuck: Setting upto on feed to %d\n",atoi(get_word(xin,1)));
		group_setfeedendif(grp,h_hosti,atoi(get_word(xin,1)));
		hchan[hh].itemi = atoi(get_word(xin,1));
		hchan[hh].state = S_SUCK_HEAD_DATA;
		break;
	case S_SUCK_HEAD_DATA:
		if (strcmp(xin,".\r\n")==0) {
			isheader = hdr_matchgroup(group_name(grp));
			hchan[hh].got++;
			if (isheader) {
				x = hdr_link("header_feed",group_name(grp),hchan[hh].itemi);
				big_add(chan,x,strlen(x));
			}
	                msg_from(h_host);
			get = big_get(chan);
			if (isheader) item_setheader(TRUE,group_name(grp));				
			if (ishdr_old()) {
				group_xref_clear();
				group_xref_here(grp,hchan[hh].itemi);
			}
			if (!hchan[hh].ignore) 
			set_ignoreage(TRUE);
			if (get!=NULL) item_suck(get);
			set_ignoreage(FALSE);
			if (isheader) item_setheader(FALSE,"");
			if (ishdr_old()) 	group_use_xref(FALSE);
	                big_free(chan);
			hchan[hh].state = S_SUCK_NEXT;
			break;
		}
                if (big_len(chan) >= MAX_STRLEN) {	goto suck_failed;}
		big_add(chan,xin,len);		
		break;
	case S_SUCK_NEXT:
		imsg("hdrsuck: next response (%s) \n",nocrlf(xin));
		if (status<200 || status>499) { emsg("head response %s\n", nocrlf(xin)); goto suck_failed; }
		hchan[hh].block++;
		if ((lib_gmtime()-hchan[hh].start)>hdr_suck_timeout()) {
			imsg("hdrsuck: timeout only got %d of %d items\n",hchan[hh].got,hdr_prefetch());
			goto suck_failed;
		}

		if (hchan[hh].block>=SUCK_BLOCK) {
			if ((hchan[hh].got>=hdr_prefetch()) || (status>300)) { 
				goto suck_finished;
			}
			goto next_block;
		}
		if (status>300) {
			hchan[hh].ignore = TRUE;
		}
		hchan[hh].state = S_SUCK_HEAD;
		break;		
	}

	return TRUE;
fail_item:

	if ((id>=0) && (id<maxhc)) 
		hc[id].state = S_FAILED;
	hchan[hh].state = S_IDLE;
	if (hchan[hh].f!=NULL) fclosep(hchan[hh].f);
	if ((id>=0) && (id<maxhc)) 
		hremove(hcache_fname(id),"fail_item");
	hchan[hh].f = NULL;
	return TRUE;
suck_failed:
	if (chan>=0) {
		chan_close(chan);
		big_free(chan);
		nopen--;
	}
	hchan[hh].state = S_CLOSED;
	ncmd_group_finish(hchan[hh].fromchan);
	return TRUE;
suck_finished:
	imsg("hdr prefetch of group finished %s\n",group_name(grp));
	ncmd_group_finish(hchan[hh].fromchan);
	hchan[hh].state = S_IDLE;
	big_free(chan);
	return TRUE;
}
void hdr_fixrange(int gid, int first, int last)
{
	int cur_first,cur_last;
	int n;
	int grp = gid_to_group(gid);

	if (grp<0) return;
	cur_first = group_getstart(grp);
        cur_last = group_getend(grp);
	imsg("Purging range for %s, now %d->%d\n",group_name(grp),first,last);
	if (cur_first>=first) if (cur_last<=last) return; 
	n = g_list_start(grp);
	for (; n>=0 ; )  {
		if ((n<first) || (n>last)) {
			item_delete_all(-1,grp,n);
		}
		n = g_list_next();
	}
	if (first>cur_first) group_setstart(grp,first);
}

/*
	Possible return values
		upto = -1  Finished
		xitem = NULL failed
		str_len(xitem)==0 then there is no new data yet
*/
char *hcache_error(int id)
{
	if ((id<0) || (id>=maxhc)) return "No Error";
	if (hc[id].etxt==NULL) return "No Error";
	return hc[id].etxt;
}
int hcache_retry(int id)
{
	if ((id<0) || (id>=maxhc)) return FALSE;
	return hc[id].doretry;
}
void hcache_timeout(int id)
{
	if ((id<0) || (id>=maxhc)) return;
	if (hc[id].state!=S_COMING) return;
	if (hc[id].tasked<=0) return;
	if (hc[id].tasked + hdr_suck_timeout() > lib_gmtime()) return;
	imsg("Timing out header request [%d] after %d seconds (header_timeout)\n",id,lib_gmtime()-hc[id].tasked);
	headmsg(id,TRUE,"Timeout waiting for body from upstream");
}
String *hcache_get(int id, int xfrom, int chunk, int *upto)
{
	static int last_rebuild;
	static int nfailed;
	FILE *f;
	static String *xitem;
	int got;
	if (xitem==NULL) xitem = str_new();

	str_setlen(xitem,0);
	if ((id<0) || (id>=maxhc)) return NULL;
	hcache_timeout(id);
	if (hc[id].state==S_FAILED) return NULL;
	f = fopen(hcache_fname(id),"rb");
	if (f==NULL) {
		if (hc[id].state==S_LOCAL) {
			emsg("Missing cache item %d, %d failrues %s\n",id,nfailed,strerror(errno));
			hc[id].state=S_FAILED;
			nfailed++;
			if (nfailed > 10) {
				if (lib_gmtime()-last_rebuild > 60*60*24) {  // Only once ever day at most 
					last_rebuild = lib_gmtime();
					nfailed = 0;
					emsg("Missing cache item %d, rebuild hcache index\n",id);
					hcache_rebuild();
					emsg("Missing cache item %d, rebuild hcache index - done\n",id);
				}
			}
			return NULL;
		}
		return xitem;
	}
	fseek(f,xfrom,SEEK_SET);
	str_setlen(xitem,chunk+10);
	got = fread(str_s(xitem),1,chunk,f);
	if (got<0) got = 0;
	str_setlen(xitem,got);
	*upto += got;
	if (feof(f)) if (hc[id].state==S_LOCAL) {
		*upto = -1;
		s_readk += hc[id].size/100;
	}
	fclosep(f);
	if (got==0) if (*upto >=0) {str_setlen(xitem,0); return xitem;}	
	return xitem;
}
		  
char *hcache_status(void)
{
	static char bf[BFSZ];
	lprintf(bf,BFSZ-1,"Header requests cached/remote  %d/%d  %dk/%dk, Size %d/%d  %dMB/%dMB\r\n",
		s_nread,s_fetch,s_readk/10,s_fetchk/10,
		totused,maxhc,(int) (bused/1000000),maxmb);		
	return bf;
}
void hcache_rebuild(void)
{
	int i;
	bused = 0;
	totused = 0;
	for (i=0; i<maxhc; i++) {
		if (hc[i].state==S_LOCAL) {
			if (!lib_exists(hcache_fname(i))) goto lostit;
			totused++;
			bused += hc[i].size;
			nhc = i+1;
		} else if (hc[i].state!=S_FREE) {
lostit:			hremove(hcache_fname(i),"notfree or lostit");
			hc[i].state = S_FREE;
			imsg("Cleaning out failed cache items %s\n",hcache_fname(i));
		}
	}
	return;
	
}
char *hdr_link(char *feed, char *group, int n)
{
	static char bf[BFSZ];
	
	lprintf(bf,BFSZ-1,"\r\n:dnews_hdr_cache1923:%s:%d:%s:-:-:\r\n",group,n,feed);
	return bf;
}
int hdr_islink(char *xin, int *group, int *n)
{
	char bf[BFSZ];
	char *s;
	if (strlen(config_get(C_HEADER_GROUPS))==0) return FALSE;
	if (strncmp(xin,":dnews_hdr_cache1923:",21)!=0) return  FALSE;
	ncpy(bf,xin,200);
	s = strtok(bf,":"); if (s==NULL) return FALSE;
	s = strtok(NULL,":"); if (s==NULL) return FALSE;
	*group = group_find(s);
	if (*group < 0) return FALSE;
	s = strtok(NULL,":"); if (s==NULL) return FALSE;
	*n = atoi(s);
	return TRUE;	
}

void headmsg(int id,int doretry, char * arg_list, ...)
{
        va_list arg_ptr;
        char *format;
        int len;
	char output[4000];

        va_start(arg_ptr, arg_list);
        format = arg_list;
        len = vsnprintf(output, BFSZ-1, format, arg_ptr);
        output[BFSZ-1] = 0;

       	if ((id>=0) && (id<maxhc))  {
		vstrdup(&(hc[id].etxt),output);
		hc[id].state = S_FAILED;
		hc[id].doretry = doretry;
	}
}
char *file_hdr(int gid)
{
	static char fname[BFSZ];
	char bf[BFSZ];
	lprintf(bf,90,"%d/hdr_%d.dat",gid%100,gid);
	ncpy(fname,file_xwork(bf,-1),BFSZ	-1);
	return fname;
}
static int upstream,hdr_grp;
int hdr_setgrp(int grp)
{
	hdr_grp = grp;
	return TRUE;
}
int hdr_setupstream(int up)
{
	upstream = up;
	return TRUE;
}
int hdr_addmap(int grp, int n)
{
	char fname[BFSZ];
	int pair[2];
	int gid;
	FILE *f;

	if (upstream<0) return TRUE;
	if (grp<0) return TRUE;
	if (grp!=hdr_grp) return TRUE;
	gid = group_getgroupid(grp);
	ncpy(fname,file_hdr(gid),BFSZ-1);
	f = fopen(fname,"ab");
	if (f==NULL) {
		lib_createdir(fname);
		f = fopen(fname,"ab");
	}
	if (f!=NULL) {
		pair[0] = n;
		pair[1] = upstream;
		fwrite(pair,sizeof(pair),1,f);
		fclose(f);
	}
	upstream = hdr_grp = -1;
	return TRUE;
}
int hdr_translate(int grp,int listid[],int nlist)
{
	char fname[BFSZ];
	int pair[2];
	int gid;
	int i;
	int *done;
	FILE *f;

	if (nlist==0) return TRUE;
	if (grp<0) return TRUE;
	if (!ishdr_new()) return TRUE;

	gid = group_getgroupid(grp);
	ncpy(fname,file_hdr(gid),BFSZ-1);
printf("reading %s\n",fname);
	f = fopen(fname,"rb");
	if (f==NULL) return TRUE;
	

	done = mymalloc(nlist*sizeof(int));
	for (i=0; i<nlist; i++) done[i] = FALSE;
	for (;!feof(f);) {
		if (fread(pair,sizeof(pair),1,f)<1) break;
printf("reading pair (%d %d\n",pair[0],pair[1]);
		for (i=0; i<nlist; i++) {
			if (done[i]) continue; 
			if (listid[i]==pair[1]) {
imsg("Translated remote number %d to local number %d\n",listid[i],pair[0]);
printf("Translated remote number %d to local number %d\n",listid[i],pair[0]);
				listid[i] = pair[0];
				done[i] = TRUE;
				break;
			}
		}
	}
printf("end of table (%d)\n",feof(f));
	for (i=0; i<nlist; i++) {
		if (!done[i]) {
			listid[i] = 0;
			imsg("Remote entry not found locally %d\n",i);
		}
	}
	fclose(f);
	myfree(done);
	return TRUE;
}