/* Threading stuff for incoming feeds */

/*.


Call cell check's in main loop
Check header line length and refuse anything over 15K bytes.


A piece of memory is being over-written, by some kind of string/array reference.

It happens when thin(), is enabled.

Many threads are created/deleted so it must be


	It must be an array reference that is faulty and a long way off.
	An un-initialzed array reference?
	Some thread/concurrency problem. 
	It could be a string over/under-write,

Solution:
	add bound() function checking to





  Safety check array indexing for:

	

	static int chan_len[MAX_CHAN];

  Safety check:
	chan_bf[chan] = mymalloc(CHBFSZ);
	chan_item[chan] = mymalloc(maxlen+1);
String
	memmove, memcpy, strcpy, strncpy
	string lengths;
	------------------------
  
    
      
	
	  memcpy, memmove




Add status 'nthread' counter and display 
	Use test function to test these input routines with
	partial packets, and long message id's 
	Add 'check' processing after we have debugged takethis.
	Test for speed with feedtest (could download?)
	Modify item_suck to reject messages with naked 'cr' characters
	in the headers.
	IF thread creation fails, continue without threads!!!

In chan.c
	stat_start(X_DOREST);
	if (xn[c] > 0) chan_dorest(c);

	if (xn[c]==0) if (chan_xisihave[c]) {
		th_start(c);
	}
	stat_end(X_DOREST);
	vtrace();

*/

#ifdef __MWERKS__ /*RHP we will take this to mean we are making a Mac version*/
#include "mac.h"
#endif

#include <stdlib.h>
#include <stdio.h>
#include <ctype.h>
#include <string.h>
#include <assert.h>
#include <errno.h>

#include "mutex.h"


#ifdef VMS
#include <unixio.h>
#define unlink remove
#endif
#ifndef NO_FCNTL
#include <fcntl.h>
#else
/*RHP not for mac */
#ifndef Mac
#include <file.h>
#endif
#endif
#include "emsg.h"
#include "sary.h"
#include "str.h"
#include "lib.h"
#include "group.h"
#include "hist.h"
#include "suck.h"
#include "nntp.h"
#include "chan.h"
#include "feed.h"
#include "expire.h"
#include "control.h"
#include "item.h"
#include "db.h"
#include "ncmd.h"
#include "thin.h"

#undef mymalloc
#undef myfree
#define mymalloc(x) mymalloc2(x,M_THIN)
#define myfree(x) myfree2(x,M_THIN)

int thin_fault_log(int c, char *in, int n, int response, int line);
int thin_fault(int c, char *reason);
int thin_orders_add(int *c);
int thin_orders_find(int c);

void nntp_addidle(int x);
void nntp_setthread(int c, int v);
void suck_setexit(void);
int thin_suck(int chan, char *start, int len, char *mid, char *s, char *out, int toobig, int start_time, int start_clock, int skt, int skc, int isiahve);
int thin_stats(int chan,int n);
int thin_test(int chan, char *bf, int len);
int thin_finish(int chan);
int thin_perror(char *s, char *socketerror);
int thin_abort(char *rsea);
int ncmd_ihave_result(int chan, char *mid, char *rs);
static int are_threads;
int tmsg(const char * arg_list, ...);
static Thread *chan_thread[MAX_CHAN];
static String *chan_str[MAX_CHAN];
static int chan_chan[MAX_CHAN];
static int maxlen;
static char *chan_item[MAX_CHAN];
static char *chan_bf[MAX_CHAN];
static int chan_upto[MAX_CHAN];
static int chan_len[MAX_CHAN];
int thin_report(void);
int ttmsg_do(char *output);

static int chan_used[MAX_CHAN];
static int isdebug = FALSE;
int thin_bounds(int v, int max, char *reason);
#define CHBFSZ 132000
#define xxxxnntp_socket_read thin_test
int thin_fix_nulls(char *s, int len);

#ifdef DOTHREADS

#ifndef TESTING
int thin_setbusy(int chan,int x)
{	
	chan_used[chan] = x;
	return TRUE;
}
int thin_busy(int chan)
{
	return chan_used[chan];
}
int thin_start_setup(int chan);
int thin_start(int chan)
{
	if ((chan<0) || (chan>=MAX_CHAN)) return TRUE;
	thin_start_setup(chan);
	imsg("thin_start %d %d chan %d\n",C_THREAD_FAKE,config_true(C_THREAD_FAKE),chan);
	if (thin_orders_find(chan)) {
		lib_thread_add(1);
		return TRUE;
	}
	if (config_true(C_THREAD_FAKE)) {
		emsg("running FAKE thread\n");
		thin_main(&chan_chan[chan]);
		return TRUE;
	} 
	chan_thread[chan] = th_new(thin_main,&chan_chan[chan]);
	if (chan_thread[chan]==NULL) {
		emsg("Thread creation failed\n");
		return FALSE;
	}
	th_free(&chan_thread[chan]); /* Closes the handle, not the thread itself */
	chan_thread[chan] = NULL;

	lib_thread_add(1);
	return TRUE;
}
int thin_start_setup(int chan)
{
	if ((chan<0) || (chan>=MAX_CHAN)) return TRUE;
	are_threads = TRUE;
	if (chan_used[chan]) {
		emsg("thin_start called twice for chan %d\n",chan);
		dnews_ex("thin_start called twice");
		return TRUE;
	}
	chan_used[chan] = TRUE;
	chan_chan[chan] = chan;
	nntp_setthread(chan,TRUE);
	ncmd_set_mode(chan,M_THREAD);
	nntp_setblock(chan,TRUE);
	return TRUE;
}
typedef struct ORDERS {int *c; struct ORDERS *next; } Orders;
static Orders *orders;
int thin_orders_add(int *c)
{
	Orders *o;
	o = mymalloc(sizeof(Orders));
	o->c = c;
	o->next = orders;
	orders = o;
	return TRUE;
}
int thin_orders_find(int c)
{
	Orders *o;
	if (orders==NULL) return FALSE;
	o = orders;
	*(o->c) = c;
	orders = o->next;
	myfree(o);
	return TRUE;
}
int thin_wait_orders(int *c)
{
	mutex_wait(M_MAIN);
	*c = -1;
	thin_orders_add(c);
	tmsg("Input thread waiting for more orders ...\n");
	mutex_release(M_MAIN);
	for (;TRUE;) {
		lib_msleep(100);
		if (*c>=0) break;
	}
	tmsg("Orders arrived for me, chan [%d]\n",*c);
	return TRUE;
}
int thin_main(void *arg)
{
	int chan = *(int *) arg;

again:	tmsg("Input thread running 1 chan=%d\n",chan);
	thin_init(chan);
	for (;TRUE;) {
		if (isdebug) tmsg("thin: Input thread running 2\n");
		vtrace();
		if (!thin_read(chan)) break;
		vtrace();
		if (!check_cell(chan_item[chan],"thin")) {
			thin_abort("chan_item corrupt");	
		}
		if (!check_cell(chan_bf[chan],"thin2")) {
			thin_abort("chan_bf corrupt");	
		}

	}
	tmsg("thin: Input thread exiting %d\n",chan);
	thin_finish(chan);
	tmsg("thin: thin_finish_finished ok\n");
	if (config_true(C_THREAD_REUSE)) {
		thin_wait_orders(&chan);
		goto again;
	}
	return TRUE;
}
void nntp_close_later(int c);
int chan_stats_add(int x);
int thin_finish(int chan)
{
	if ((chan<0) || (chan>=MAX_CHAN)) return TRUE;
	tmsg("thin: Input thread finishing %d\n",chan);
	mutex_wait(M_MAIN);
	chan_stats_add(chan);
	
	vtrace();
	myfree(chan_item[chan]); chan_item[chan] = NULL;
	myfree(chan_bf[chan]); chan_bf[chan] = NULL;
	if (chan_str[chan]!=NULL) str_free(chan_str[chan]); chan_str[chan] = NULL;
	chan_used[chan] = FALSE;
	nntp_setthread(chan,FALSE);
	nntp_close_later(chan);
/*	chan_close(chan); */
	tmsg("thin: setthread false \n");
	lib_thread_add(-1);
	tmsg("thin: thread_add done \n");
	vtrace();
	tmsg("thin: Now release mutex\n");

	mutex_release(M_MAIN);
	tmsg("thin: Now sleeping 0.1 seconds\n");
	lib_msleep(100);
	tmsg("thin: Now exiting this thread\n");
	return TRUE;
}
int thin_init(int chan)
{
	if ((chan<0) || (chan>=MAX_CHAN)) return TRUE;
	mutex_wait(M_MAIN);
	vtrace();
	maxlen = conf_item_max();
	if (maxlen<40000) maxlen = 40000;
	chan_bf[chan] = mymalloc(CHBFSZ);
	chan_item[chan] = mymalloc(maxlen+1);
	chan_upto[chan] = 0;
	chan_len[chan] = 0;
	vtrace();
	mutex_release(M_MAIN);
	return TRUE;
}
int thin_setdebug(void)
{
	isdebug = !isdebug;
	return TRUE;
}

int thin_read(int chan)
{
	char bf[CHBFSZ];
	char *p[40];
	int n;
	char tmp[BFSZ];

	if ((chan<0) || (chan>=MAX_CHAN)) return TRUE;
	vtrace();
	if (!thin_read_line(chan,bf,CHBFSZ-30)) {
		tmsg("thin_read_line failed\n");
		return FALSE;
	}
	if (isdebug) tmsg("[%d] Read line (%.300s)\n",chan,bf);

	lprintf(tmp,98,"[%d] len=%d %.80s",th_id(),strlen(bf),bf);
	trace_addmsg(tmp);

	n = token_split(bf,p," \t\r\n",30);	
	if (isdebug) tmsg("Tokens %d\n",n);
	if (n<1) return TRUE;
	strlwr(p[0]);
	if (isdebug) tmsg("Got command (%.30s)\n",p[0]);

	if ((chan<0) || (chan>=MAX_CHAN)) {
		tmsg("Chan data corrupt %d\n",chan);
		dnews_ex("die now");
	}
	
	
	if (strcmp(p[0],"takethis")==0) {if (!thin_cmd_takethis(chan,p[1],FALSE)) { tmsg("exit_takethis\n"); return FALSE;}}
	else if (strcmp(p[0],"check")==0) {if (!thin_cmd_check(chan,p[1])) { tmsg("exit_check\n"); return FALSE;}}
	else if (strcmp(p[0],"ihave")==0) {if (!thin_cmd_ihave(chan,p[1])) { tmsg("exit_ihave\n"); return FALSE;}}
	else if (strcmp(p[0],"quit")==0) {if (!thin_cmd_quit(chan)) { tmsg("exit_quit\n"); return FALSE;}}
	else if (strcmp(p[0],"xdrealip2")==0) /*Ignore this no response */;
	else if (!thin_cmd_error(chan,p[0])) { tmsg("exit_cmd_error\n"); return FALSE;}
	return TRUE;
}
int thin_bounds(int v, int max, char *reason)
{	
	char bf[BFSZ];
	if ((v<0) || (v>=max)) {
		lprintf(bf,BFSZ-1,"Bounds %d,%d, %s",v,max,reason);
		thin_abort(bf);
	}
	return TRUE;
}
int thin_read_line(int chan, char *xrslt,int bfsz)
{
	char *at;
	char serror[BFSZ];
	char *start;
	char *rslt;
	int want;
	int n;
	int len;
	int tlen;
	int nleft;
	char *s;
	
	if ((chan<0) || (chan>=MAX_CHAN)) return TRUE;
again:
	len=chan_len[chan]-chan_upto[chan];
	tlen = bfsz;
	at=chan_bf[chan]+chan_upto[chan];
	start = at;
	rslt = xrslt;
	nleft = CHBFSZ - chan_upto[chan] - 1;
	for (;(len>0) ;tlen--,len--,nleft--) {
		if (nleft<=0) {
			tmsg("thin_read_line failure %d, %d, %d\n",tlen,len,nleft);
			thin_abort("nleft<0");
		}
		if (*at==0) *at = ' ';
		if ((*at=='\n') || (tlen<=1)) {
			*rslt = 0;
			chan_upto[chan] += (at-start+1);
			thin_bounds(chan_upto[chan],CHBFSZ,"upto end of line");
			return TRUE;
		}
		*rslt++ = *at++;
	}
	
	/* Shuffle it and read some more if needed */
	len = chan_len[chan]-chan_upto[chan];
	if (len<0) {
		tmsg("FATAL ERROR Len less zero %d\n",len);
		thin_abort("len less than zero");
	}
	if (chan_upto[chan]>0) {
		if (len>=CHBFSZ) {
			tmsg("FATAL ERROR Len exceeded %d\n",len);
			thin_abort("len exceeded");
		}
		if (len>0) memmove(chan_bf[chan],chan_bf[chan]+chan_upto[chan],len);
		//if (isdebug)  tmsg("Shuffled %d bytes (%.30s) \n",len,chan_bf[chan]);
		chan_upto[chan] = 0;
		chan_len[chan] = len;
	}
	thin_bounds(len,CHBFSZ,"terminate chanbf");

	s = chan_bf[chan]; s[len] = 0;

	want = bfsz-len-100;
	if (want<0) {
		tmsg("Chucking away part of long line\n");
		len = 0;
	}
	want = bfsz-len-100;

	thin_bounds(len+want,CHBFSZ,"terminate chanbf2");

	n = thin_socket_read(chan,s+len,want,120000,serror); thin_stats(chan,n);
	thin_fault_log(chan,s+len,n,FALSE,__LINE__);
	if (n<=0) {
		tmsg("thin: Cannot read command line chan %d %d\n",chan,n);
		if (n<0) {
			thin_perror("thin_cmd_read",serror);
		}
		thin_fault(chan,"cmd_read");
		return FALSE;
	}
	thin_fix_nulls(s+len,n);
	if (isdebug) tmsg("thin: Read %d bytes (%.30s) \n",n,chan_bf[chan]);
	chan_len[chan] += n;
	thin_bounds(chan_len[chan],CHBFSZ,"done_read");
	goto again;
}
int thin_fix_nulls(char *s, int len)
{
	int nfixed=0;
	for (;len>0;s++,len--) {
		if (*s==0) {
			*s = ' ';
			nfixed++;
		}
	}
	if (nfixed>0) tmsg("Removed %d nulls from input stream\n",nfixed);
	return TRUE;
}
int thin_perror(char *s, char *socketerror)
{
	mutex_wait(M_MAIN);
	emsg("thin: socket error (%s) (%s)\n",s,socketerror);
	mutex_release(M_MAIN);
	return TRUE;
}
int thin_abort(char *reason)
{
	tmsg("FATAL ERROR OCCURED %.90s\n",reason);
	mutex_wait(M_MAIN);
	emsg("FATAL ERROR IN THIN THREADING MODULE\n");
	suck_setexit();
	mutex_release(M_MAIN);
	abort();
	return 0;
}
int thin_getrest(int chan, char *out)
{
	int len = chan_len[chan]-chan_upto[chan];

	if ((chan<0) || (chan>=MAX_CHAN)) return TRUE;
	if (len>=maxlen) thin_abort("len>maxlen");
	if (len<0) thin_abort("len<0");
	memcpy(out,chan_bf[chan]+chan_upto[chan],len);
	chan_upto[chan] = 0;
	chan_len[chan] = 0;
	return len;
}
int thin_putrest(int chan, char *out, int len)
{
	if ((chan<0) || (chan>=MAX_CHAN)) return TRUE;
	if ((len >= CHBFSZ) || (len<0)) {
		tmsg("FATAL Thin_putrest failed [%d] %p %d %.20s\n",chan,out,len,out);	
		thin_abort("putrest");
	}
	memcpy(chan_bf[chan],out,len);
	chan_len[chan] = len;
	return TRUE;
}
int thin_cmd_check(int chan, char *mid)
{
	char response[BFSZ];
	int n;
	int xsec;
	if ((chan<0) || (chan>=MAX_CHAN)) return TRUE;
	if ((mid==NULL) || (strlen(mid)>199)) {
		char bf[CHBFSZ];
		/* Immediately reject it */
		if (mid==NULL) mid="";
		lprintf(bf,CHBFSZ-1,"438 %.34000s not wanted, mid too long\r\n",mid);
		if (nntp_socket_write(chan,bf,strlen(bf))<=0) return FALSE;
		if (isdebug) thin_fault_log(chan,bf,strlen(bf),TRUE,__LINE__);
		tmsg("rejecting large message2 id (%.200s)\n",mid);
		return TRUE;
	}
	mutex_wait(M_MAIN);
	vtrace(); /* THIS IS THE LAST CRASH POINT SO FAR. */
	xsec = lib_msec();
	if (isdebug) tmsg("thin: check %.200s\n",mid);
	ncmd_check_result(chan,mid,response);
	chan_setused(chan);
	//nntp_addidle(xsec-lib_msec());
	vtrace();   /* And didn't get to here !!! */
	mutex_release(M_MAIN);
	n = nntp_socket_write(chan,response,strlen(response));
	if (n<=0) {
		tmsg("thin: check socket write returned %d %s\n",n,strerror(errno));
	}
	if (isdebug) thin_fault_log(chan,response,strlen(response),TRUE,__LINE__);
	if (n<=0) return FALSE;
	return TRUE;
}
int thin_cmd_ihave(int chan, char *mid)
{
	char response[BFSZ];
	int n;
	int r;
	int xsec;
	int isok=TRUE;

	if ((chan<0) || (chan>=MAX_CHAN)) return TRUE;
	if ((mid==NULL) || (strlen(mid)>199)) {
		char bf[CHBFSZ];
		/* Immediately reject it */
		lprintf(bf,CHBFSZ-1,"435 article not wanted, mid too long\r\n",mid);
		if (nntp_socket_write(chan,bf,strlen(bf))<=0) return FALSE;
		if (isdebug) thin_fault_log(chan,bf,strlen(bf),TRUE,__LINE__);
		tmsg("rejecting large message id (%.200s)\n",mid);
		return TRUE;
	}
	mutex_wait(M_MAIN);
	vtrace();
	xsec = lib_msec();
	if (isdebug) tmsg("thin: ihave %.200s\n",mid);
	isok = ncmd_ihave_result(chan,mid,response);
	chan_setused(chan);
	//nntp_addidle(xsec-lib_msec());
	vtrace();
	mutex_release(M_MAIN);
	n = nntp_socket_write(chan,response,strlen(response));
	if (n<=0) {
		tmsg("thin: check socket write returned %d %s\n",n,strerror(errno));
	}
	if (isdebug) thin_fault_log(chan,response,strlen(response),TRUE,__LINE__);
	if (n<=0) return FALSE;
	if (isok) {
		r = thin_cmd_takethis(chan,mid,TRUE);
		if (!r) tmsg("thin: cmd_takethis is not happy\n");
		return r;
	}
	return TRUE;
}
int thin_cmd_error(int chan, char *cmd)
{
	char *s;
	char bf[BFSZ];
	int n;
	static int done;
	if ((chan<0) || (chan>=MAX_CHAN)) return TRUE;
	s = "500 Invalid command in thread_input feeding mode\r\n";
	lprintf(bf,BFSZ-1,"thin: [%d] error: Invalid command in thread_input feeding mode (%.90s)\n",chan,cmd);
	thin_fault_log(chan,bf,strlen(bf),TRUE,__LINE__);
	tmsg("thin: [%d] error: Invalid command in thread_input feeding mode (%.90s)\n",chan,cmd);

	if (!done) thin_fault(chan,"command fault");
	done = TRUE;
	
	n = nntp_socket_write(chan,s,strlen(s));
	if (isdebug) thin_fault_log(chan,s,strlen(s),TRUE,__LINE__);
	if (n<=0) {
		tmsg("thin: [%d] error writing to socket - it must have closed\n",chan);
		return FALSE;
	}
	return TRUE;
}
int thin_cmd_quit(int chan)
{
	char *s;
	int n;
	if ((chan<0) || (chan>=MAX_CHAN)) return TRUE;
	s = "205 Closing connection goodbye!\r\n";
	n = nntp_socket_write(chan,s,strlen(s));
	if (n<=0) return FALSE;
	return FALSE;
}
int thin_cmd_takethis(int chan, char *mid, int isihave)
{
	/*	Read in rest of input from buffer
		Read in missing data
		Put left over data into input buffer.
		go read data in.
	*/
	char serror[BFSZ];
	char tmp8[BFSZ];
	static int itimeout;
	char *out = chan_item[chan];
	char *start=out;
	int want = maxlen-1;
	int sz = CHBFSZ-1;
	char *s,*s2;
	int n;
	int w;
	int toobig = FALSE;
	int first=TRUE;
	int start_time,start_clock;
	int sk_time,sk_clock;
	int startt;
	char tmp2[BFSZ];
	char *t;

	if (itimeout==0) itimeout = atoi(config_get(C_IHAVE_TIMEOUT))*1000;
	if (itimeout==0) itimeout = 50000; /* Allow 50 seconds when waiting for ihave .*/
	if ((chan<0) || (chan>=MAX_CHAN)) return TRUE;
	vtrace();

	if (mid==NULL) mid="null";
        start_time = lib_msec();
        start_clock = stats_clock();
	sk_time = sk_clock = 0;
	n = thin_getrest(chan,out);
	vtrace();
	if (n<=0) first = FALSE;
	out += n;
	want -= n;
	for (;TRUE;) {
		if (want<=10) { /* Ran out of space, must be a long item we need to reject */
			tmsg("thin: Input message over-flow, throwing away this item (item_max exceeded)\n");
			toobig = TRUE;
			memmove(chan_item[chan],out-10,10);
			out = chan_item[chan]+10;
			want = maxlen-11;
		}
		if (!first) { 
			w = want;
			if (w>=((CHBFSZ/2)-100)) w = (CHBFSZ/2)-100;
			// if (isdebug)  tmsg("thin: takethis, read more %d %d %d\n",w,want,maxlen);
			if (w<0) {
				emsg("w is %d\n",w);
				thin_abort("w<0");
			}
		        sk_time -= lib_msec();
			sk_clock -= stats_clock();
			vtrace();
			startt = time(NULL);
			/* Read with 10 second timeout */
			n = thin_socket_read(chan,out,w,itimeout,serror); thin_stats(chan,n);
			t = "(cmd_takethis_read)";
			thin_fault_log(chan,t,strlen(t),TRUE,__LINE__);
			thin_fault_log(chan,out,n,FALSE,__LINE__);
			thin_fix_nulls(out,n);
			vtrace();
		        sk_time += lib_msec();
			sk_clock += stats_clock();
			if (n<=0) {
				lprintf(tmp2,200,"(socket_read_failed %d seconds) %d %s",time(NULL)-startt,n,strerror(errno));
				tmsg("thin: Closing link as message body wasn't sent for %d seconds, %s\n",time(NULL)-startt,strerror(errno));
				thin_fault_log(chan,tmp2,strlen(tmp2),TRUE,__LINE__);
				thin_perror("thin_msg_body",serror);
				thin_fault(chan,"body_read");
				vtrace();
				/* See what happens if we prod the line before closing it */
				lprintf(tmp8,BFSZ-1,"437 FATAL PROTOCOL FAULT, GOT %d bytes, waited %d seconds\r\n",n,time(NULL)-startt);
				n = nntp_socket_write(chan,tmp8,strlen(tmp8));
				s = tmp8;
				thin_fault_log(chan,s,strlen(s),TRUE,__LINE__);
				n = thin_socket_read(chan,tmp2,200,1000,serror); 
				if (n>0) {
					tmp2[n] = 0;
					thin_fault_log(chan,tmp2,n,TRUE,__LINE__);
				}
				return FALSE;
			}
			//if (isdebug) tmsg("thin: takethis, read %d bytes (%.30s)\n",n,out);
			want -= n;
			out += n;
			/* Find end of item, if found then process item
			and put rest onto input buffer */
		}
		first = FALSE;
		s = out-n - 4;
		if (s<chan_item[chan]) s = chan_item[chan];
		vtrace();
		s2 = find_eom(s,out-s+1,chan_item[chan],maxlen); /* Points to char after end of line feed */
		vtrace();
		/*if (s2!=NULL) if (isdebug)  tmsg("thin: eom if found len %d %p (%.20s) \n",n,s2,s);
		if (isdebug)  {
			lprintf(tmp2,BFSZ-1,"thin: eom if found len %d %p (%.20s) \n",n,s2,s);
			thin_fault_log(chan,tmp2,strlen(tmp2),TRUE);
		}
		*/
		s = s2;
		if (out-s<0) s = NULL; /* Shouldn't happen but this will stop it*/
		if (s!=NULL) {
			/* Block, copy into string, and send in */
			if (isdebug)  tmsg("Found end of message\n");
			thin_suck(chan,start,s-start,mid,s,out,toobig,start_time,start_clock,sk_time,sk_clock,isihave);
			vtrace();
			return TRUE;
		}
	}
	vtrace();
	return TRUE;
}
int thin_trim_dot(String *xitem)
{
	/* If the end is, lf.lf or   lf.crlf, then strip it */
	int len = str_len(xitem);
	char *s;

	s = str_s(xitem) + len - 3;
	if (strncmp(s,"\n.\n",3)==0) len -=2;
	else {
		s--;
		if (strncmp(s,"\n.\r\n",4)==0) len -=3;
	}
	str_setlen(xitem,len);
	return TRUE;
}
int thin_head_check(char *start, int len)
{
	char *s;
	int longest=0;
	int thislen;
	int xlen=len;
	char *last=start;
	for (s=start; len>0; len--,s++) {
		if (*s=='\n') {
			thislen = s - last;
			if (thislen>longest) longest = thislen;
			last = s;
			if (s[2] == '\n') break; /* end of header */
		}
	}
/*	tmsg("Checked %d of %d bytes \n",len,xlen);*/
	return longest;
}

int thin_suck(int chan, char *start, int len, char *mid, char *s, char *out, int toobig, int start_time, int start_clock, int sk_time, int sk_clock, int isihave)
{
	char response[10000];
	int xsec;
	int n;
	String *xitem;
	int headlen; /* Length of longest header line  - paranoide check looking for bug*/
	
	if ((chan<0) || (chan>=MAX_CHAN)) return TRUE;

	headlen = thin_head_check(start,len);
	mutex_wait(M_MAIN);
	vtrace();
	xsec = lib_msec();

	if (chan_str[chan]==NULL) chan_str[chan] = str_new();
	xitem = chan_str[chan];
/*	db_test_nulls(start,len,"thin.suck",1,"thin_suck"); */
	str_copys(xitem,start,len);
	gstat_xadd("Total In bytes/s",len); /* So the graph makes sense */
	nntp_speedin(chan,len);
	thin_trim_dot(xitem);
#ifndef NORMALTHING
	if (headlen>15000) {
		lprintf(response,10000-1,"439 %.4000s header line too long %d bytes\015\012",mid,headlen);
		tmsg("Rejecting item with long header line %d bytes, %.100s",headlen,mid);
	} else {
		if (strlen(mid)>400) {
			lprintf(response,10000-1,"439 %.4000s mid is too long\015\012",mid);
		} else {
			vtrace();
			ncmd_item_suck(chan,mid,xitem,response,toobig,isihave);
			vtrace();
		}
	}
#else
	lprintf(response,10000-1,"239 %.900s article transferred ok\015\012",mid);
#endif
	chan_setused(chan);
	str_clearbig(xitem);
	//nntp_addidle(xsec-lib_msec());
	vtrace();
	mutex_release(M_MAIN);
	
	/* Send response */
	n = nntp_socket_write(chan,response,strlen(response));
	if (isdebug) thin_fault_log(chan,response,strlen(response),TRUE,__LINE__);
	if (n<=0) return FALSE;
		
	/* Put rest of input buffer into cmd buffer. */
	vtrace();
	thin_putrest(chan, s,out-s);
	vtrace();
	return TRUE;
}
char *find_eom(char *part, int nn, char *start, int len)
{
	register int n=nn;
	register char *s=part;
	int strip=FALSE;

	if (part<start) {
		emsg("find_eom before start of message %p %p %d %d\n",part,start,nn,len);
		dnews_ex("eom_wrong");
	}
	if ((part+nn)>(start+len)) {
		emsg("find_eom len wrong %p %p %d %d\n",part,start,nn,len);
		dnews_ex("eom_wrong_len");
	}

	if (part==start) if (n>2) if (*s=='.') {
		if (s[1]=='\r') if (s[2]=='\n') {
			return s+3;
		}
	}
	for (;n>0;n--,s++) {
		if (*s==0) {*s = 32; strip = TRUE;}
		if (*s=='\n') {
			s++; n--; 
			if (*s==0) {*s = 32; strip = TRUE;}
			if (n<=0) break;
			if (*s!='.') continue;
			s++; n--; 
			if (*s==0) {*s = 32; strip = TRUE;}
			if (n<=0) break;
			if (*s=='\r') {
				s++; n--;
				if (*s==0) {*s = 32; strip = TRUE;}
			}
			if (*s=='\n') {
				/*if (strip) tmsg("Removed null(s) from thread input stream\n");*/
				return s+1; 
			}
		}
	}
	/*if (strip) tmsg("Removed null(s) from thread input stream\n");*/
	return NULL;
}
#endif
/* Thread safe logging */
int thin_test(int chan, char *bf, int len)
{
	/* Generate various test packets */
	static int u;
	static char mid[BFSZ];
	char *o;
	static int upto;
	char big[60000];
	char bigmid[60000];
	int i=0;
	if (chan<0) return TRUE;
	if (u==0) {
		u++;
		lprintf(mid,BFSZ-1,"<a%d@a.com>",time(NULL));
		lprintf(bf,CHBFSZ-1,"takethis %s\r\n",mid);
		return strlen(bf);
	}
	if (u==1) {
		u++;
		o = bf;
		lprintf(o,BFSZ-1,"From: me@here.com\r\n"); o += strlen(o);
		lprintf(o,BFSZ-1,"Subject: this is the subject\r\n"); o += strlen(o);
		lprintf(o,BFSZ-1,"Newsgroups: test.1\r\n"); o += strlen(o);
		lprintf(o,BFSZ-1,"Message-ID: %s\r\n",mid); o += strlen(o);
		lprintf(o,BFSZ-1,"Date: Wed, 10 May 2000 14:43:58 +0200\r\n"); o += strlen(o);
		lprintf(o,BFSZ-1,"\r\n"); o += strlen(o);
		lprintf(o,BFSZ-1,"Test message\r\n"); o += strlen(o);
		lprintf(o,BFSZ-1,".\r\n"); o += strlen(o);
		return strlen(bf);
	}
	if (u==2) {
		u++;
		o = bf;
		for (i=0; i<10; i++) {
			lprintf(mid,BFSZ-1,"<a%d%d@a.com>",time(NULL),i);
			lprintf(o,BFSZ-1,"check %s\r\n",mid); o += strlen(o);
		}
		return strlen(bf);
	}
	if (u==3) {
		u++;
		o = bf;
		lprintf(mid,BFSZ-1,"<eol_ba%d%d@a.com>",time(NULL),i);
		lprintf(o,BFSZ-1,"check %s\r\n",mid); o += strlen(o);
		return strlen(bf)-1;
	}
	if (u==4) {
		strcpy(bf,"\n");
		u++;
		return 1;
	}
	if (u==5) {
		u++;
		o = bf;
		lprintf(o,BFSZ-1,"check <aa@bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb>\r\n",mid); o += strlen(o);
		return strlen(bf);
	}
	if (u==6) {
		memset(bigmid,'x',50000);
		lprintf(big,60000-1,"check <%s>\r\n",bigmid);
		strncpy(bf,big,len);
		upto = len;
		u++;
		return len;
	}
	if (upto<(int) strlen(big)) if (u==7) {
		if (len>((int) strlen(big)-upto)) len = strlen(big) - upto;
		strncpy(bf,big+upto,len);
		upto += len;
		return len;
	}
	lib_msleep(1000*60);
	return -1; /*  Socket close */
}
#endif

int tmsg_do(char *output);
int tmsg(const char * arg_list, ...)
{
        va_list arg_ptr;
        char *format;
        int len;
	char output[4000];
        va_start(arg_ptr, arg_list);
        format = (char *) arg_list;
        len = vsnprintf(output,BFSZ-1, format, arg_ptr);
	tmsg_do(output);
	return 0;
}
int ttmsg(const char * arg_list, ...)
{
        va_list arg_ptr;
        char *format;
        int len;
	char output[4000];
        va_start(arg_ptr, arg_list);
        format = (char *) arg_list;
        len = vsnprintf(output,BFSZ-1, format, arg_ptr);
	ttmsg_do(output);
	return 0;
}
char *thin_time(char *tstr)
{
	char tmpstr[BFSZ];
        time_t t;
	char *ccc;
	int msec=0;
#ifdef WIN32
	SYSTEMTIME systime;
	GetSystemTime(&systime);
	msec = systime.wMilliseconds;
#endif
	time(&t);
	ccc = safe_ctime(&t,tmpstr);
	if (ccc==NULL) ccc = "badtimebadtimebadtime";
        strcpy(tstr,ccc+8);
        tstr[11] = 0;
	lprintf(tstr+11,100,".%.02d",msec/10);

	return tstr;
}

static FILE *ftmsg;
static char tfname[BFSZ];
int tmsg_do(char *output)
{
	char tmp[BFSZ];
	mutex_wait(M_TMSG);
	if (ftmsg==NULL) {
		if (strlen(tfname)==0) {
			mutex_wait(M_MAIN);
			ncpy(tfname,file_log("tmsg.log"),BFSZ-1);
			mutex_release(M_MAIN);
		}
		ftmsg = fopen(tfname,"a");
		if (ftmsg==NULL) goto failed;
	}
	fprintf(ftmsg,"%s %d %s",thin_time(tmp),th_id(),output);
	if (file_length(fileno(ftmsg))>1000000) {
		fclose(ftmsg);
		ftmsg = NULL;
		remove(tfname);
	}
failed:
	mutex_release(M_TMSG);
	return TRUE;
}
static FILE *fttmsg;
static char ttfname[BFSZ];
int ttmsg_do(char *output)
{
	char tmp[BFSZ];
	mutex_wait(M_TMSG);
	if (fttmsg==NULL) {
		if (strlen(ttfname)==0) {
			mutex_wait(M_MAIN);
			ncpy(ttfname,file_log("ttmsg.log"),BFSZ-1);
			mutex_release(M_MAIN);
		}
		fttmsg = fopen(ttfname,"a");
		if (fttmsg==NULL) goto failed;
	}
	fprintf(fttmsg,"%s %d %s",thin_time(tmp),th_id(),output);
	if (file_length(fileno(fttmsg))>2000000) {
		fclose(fttmsg);
		fttmsg = NULL;
		remove(ttfname);
	}
failed:
	mutex_release(M_TMSG);
	return TRUE;
}

static int ntest_done;
#ifndef DOTHREADS
int thin_busy(int chan)
{
	return FALSE;
}
int thin_trim_dot(String *xitem)
{
	return TRUE;
}
int thin_test_threads(int n)
{
	printf("Threads not compiled in this version of dnews, sorry\n");
	printf("This may be an oversight, or it may be due to a problem with the thread library on this system\n");
	printf("Email support@netwinsite.com to find out more details, give exact details of your OS type and version\n");
	return FALSE;
}
#else
int thread_test_main(void *x);
int thin_test_threads(int n)
{
	/* Create 'n' threads, running test function */
	int i;
	Thread *h[30000];

	mutex_init();
	if (n>20000) n = 20000;
	for (i=0; i<n; i++) {
		h[i] = th_new(thread_test_main,(void *) i);
		if (h[i]==NULL) break;
	}
	printf("Successfully created %d threads\n",n);
	for (i=0; i<600; i++) {
		lib_msleep(1000);
		if (ntest_done>=n) break;
	}
	printf("Completed %d/%d tests\n",ntest_done,n);
	for (i=0; i<n; i++) {
		th_free(&h[i]); /* Closes the handle, not the thread itself */
	}
	return TRUE;
}
int test_fail(char *reason)
{
	printf("test failed, Reason (%.90s)\n",reason);
	abort();
	return 0;
}
#ifdef WIN32
int sleep(int x);
#endif
static int glob;
int thread_test_main(void *x)
{
	char bf[BFSZ];
	FILE *f;
	int k;
	static int skip;
	int j;
	char *s;
	int i;
	int n = (int) x;
	/* Open a file, do some strtok's */


	mutex_wait(M_MAIN); 
	mutex_wait(M_MAIN); 
/*	printf("re-entrant mutex's work ok\n");*/
	mutex_release(M_MAIN);
	mutex_release(M_MAIN);


	errno = th_id();
/*	sleep(2); Cannot sleep as it sets errno */
	for (i=0; i<1000000; i++) {
		j = i+2;
		j = i*j;
		j = i+j*i;
	}
	if (errno!=th_id())  {
		printf("errno %d !=%d\n",errno,th_id());
		test_fail("errno"); 
	}
	f = fopen("test.dat","r");
	if (f==NULL) {
		f = fopen("test.dat","a");
	}
	if (f==NULL) {
		printf("Could not open file handle %s\n",strerror(errno));
	}

	for (i=0; i<30; i++) {
#ifdef WIN32
		if (!skip) {
			strcpy(bf,"a b c d e f");
			s = strtok(bf," ");
			lib_msleep(1);
			if (strcmp(s,"a")!=0) {printf("strtok is faulty\n"); skip = TRUE;} 
			s = strtok(NULL," ");
			lib_msleep(1);
			if (strcmp(s,"b")!=0) {printf("strtok is faulty\n"); skip = TRUE;}
		}
#endif
		mutex_wait(M_MAIN); 
		glob = i;
		lib_msleep(1);
		if (glob!=i) {
			mutex_release(M_MAIN);
			test_fail("Mutex fails");
		}
		mutex_release(M_MAIN);
		for (k=0; k<1000; k++) {
			lprintf(bf,100,"%d",99);
			if (strcmp(bf,"99")!=0) {
				test_fail("lprintf not thread safe");
			}
			lprintf(bf,100,"%s","helo");
			if (strcmp(bf,"helo")!=0) {
				test_fail("lprintf not thread safe2");
			}
		}
	}
	if (f!=NULL) fclose(f);	
	printf("Test %d finished\n",n);
	mutex_wait(M_MAIN); 
	ntest_done++;
	mutex_release(M_MAIN);

	return 0;
}
#endif

static int chan_stime[MAX_CHAN];
static double chan_total[MAX_CHAN];
int thin_report(void)
{
	int delay;
	int i;
	for (i=0; i<MAX_CHAN; i++) {
		delay = lib_gmtime() - chan_stime[i];
		if (delay==0) delay = 1;
		if (chan_total[i]>0) tmsg("thin_report chan[%d] bytes %g average speed %9.0fk/s delay=%d\n",i,chan_total[i],(double) (chan_total[i]/1000.0 / (delay)),delay);
	}
	return TRUE;
}
int thin_stats(int chan,int n)
{
	if (chan<0) return 0;
	chan_total[chan] += n;
	if (chan_stime[chan]==0) chan_stime[chan] = lib_gmtime();
	return TRUE;
}
int thin_socket_read(int c, char *bf, int sz, int xdelay,char *reason)
{
	static int delay;
	int r;
	int start=0;
	strcpy(reason,"none");
	if (!config_true(C_THREAD_TIMEOUT)) return nntp_socket_read(c,bf,sz);
again:
	if (delay==0) delay = atoi(config_get(C_THREAD_TIMEOUT_WAIT)) * 1000;
	if (delay==0) delay = 1000 * 60 * 3; 
	if (xdelay==0) xdelay = delay;
	r = nntp_read_timed(c,bf,sz,xdelay,reason);


	if (isdebug) {
		char tmp3[BFSZ];
		int nnulls=0;
		int nn;
		for (nn=0;nn<r;nn++) {
			if (bf[nn]==0) nnulls++;
		}
		if (nnulls>0) tmsg("thin_nulls: Found nulls in input stream %d\n",nnulls);

		lprintf(tmp3,BFSZ-1,"socket_read %d bytes %d nulls",r,nnulls);
		thin_fault_log(c,tmp3,strlen(tmp3),FALSE,__LINE__);
	}

	if (r<0) {
	  if (errno==EAGAIN) {
		if (start==0) start = lib_gmtime();
		if ((lib_gmtime()-start) < (xdelay/1000)) {
			lib_msleep(10);
			goto again;
		}
		tmsg("Timeout, on thin_socket_read, no data for at least %d seconds %s\n",lib_gmtime()-start,reason);
	  } else {
		if (start==0) start = lib_gmtime();
		tmsg("Error, on thin_socket_read, no data for at least %d seconds %s\n",lib_gmtime()-start,reason);
	  }
	} 
   	if (r==0) tmsg("nntp_read - close r,%d %s\n",r,errno,reason);
	return r;
}
static char *chan_fault[MAX_CHAN];
static int chan_faultat[MAX_CHAN];
#define FSZ 500000
int thin_fault(int c, char *reason)
{
	FILE *f;
	char fname[BFSZ];
	if (!isdebug) return TRUE;
	if (chan_fault[c]==NULL) return TRUE;
	mutex_wait(M_MAIN);
	lprintf(fname,100,"thin_%d.log",c);
	if (strstr(reason,"command fault")!=NULL) lprintf(fname,100,"thin_cmd_%d.log",c);


	f = fopen(file_work(fname),"wb");
	fwrite(chan_fault[c],1,chan_faultat[c],f);
	fclose(f);
	mutex_release(M_MAIN);
	return TRUE;
}
static int thin_trace = -1;
int thin_set_trace(int t)
{
	thin_trace = t;
	isdebug = TRUE;
	thin_report();
	return TRUE;
}	
int thin_fault_log(int c, char *in, int n, int response, int line)
{
	int at;
	static FILE *tfile;
	if (in==NULL) {
		tmsg("fault_log passed null line %d\n",line);
		emsg("fault_log passed null line %d\n",line);
		abort();
		return TRUE;
	}
	if (n<=0) return TRUE;
	if (c<0) return TRUE;
	if (c>=MAX_CHAN) return TRUE;
	if (!isdebug) return TRUE;
	if (chan_fault[c]==NULL) {
		mutex_wait(M_MAIN);
		chan_fault[c] = mymalloc(FSZ);
		mutex_release(M_MAIN);
	}

	if (c == thin_trace) {
		if (tfile==NULL) tfile = fopen(file_work("thin_trace.log"),"a");
		if (tfile!=NULL) {
			if (response) fprintf(tfile,"\r\nTHIN< ");
			else fprintf(tfile,"\r\nTHIN> ");
			if (n>0) fwrite(in,1,n,tfile);
		}
	}

	at = chan_faultat[c];
	if ((n + at + 100) > FSZ) at = 0;
	if ((n + at + 100) > FSZ) {
		strcpy(chan_fault[c],"Lost data\r\n");
		chan_faultat[c] = 11;
		return TRUE;
	}
	if (response) {
		strcpy(chan_fault[c]+at,"Sent (");
		at += 6;
	} else  {
		strcpy(chan_fault[c]+at,"Got  (");
		at += 6;
	}
	memcpy(chan_fault[c]+at,in,n);
	at += n;
	chan_faultat[c] = at;
	if (response) {
		strcpy(chan_fault[c]+at,")\r\n");
		at += 3;
		chan_faultat[c] = at;
	} else {
		strcpy(chan_fault[c]+at,")\r\n");
		at += 3;
		chan_faultat[c] = at;
	}
	return TRUE;	
}
#ifndef DOTHREADS
int thin_setdebug(void)
{
	return TRUE;
}
#endif
int thin_release(void)
{
	if (are_threads) {
		stat_start(X_THINRELEASE);
		mutex_release(M_MAIN);
		mutex_wait(M_MAIN);
		stat_end(X_THINRELEASE);
	}
	return TRUE;
}