/*
Tcp library

*/
#ifdef __XCODE__
#include "xcode_quiet.h"
#endif
 
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <time.h>
#include <stdarg.h>
#ifdef unix
#include <unistd.h>
#include <fcntl.h>
#include <signal.h>
#endif
#ifdef WIN32
// this may not be needed at all, since I don't wait on 4000 socekts, possibly could reduce this to 100...
#define FD_SETSIZE 4048
#endif
#include "tcplib.h"
#include "fasttime2.h"

#ifdef USE_POLL
#include <sys/poll.h>
#endif

#ifdef HAS_SSL
#include "myssl.h"
#endif
#define DMID 18
#include "debug.h"
#ifndef FALSE
#define FALSE 0
#endif

#include "defines.h"
#include "lib.h"
#include "lprintf.h"

#define DMID 18
#ifdef USE_DMALLOC
#include "debug.h"
#endif 

#ifndef TRUE
#define TRUE (!FALSE)
#endif

#ifndef INADDR_NONE
#define INADDR_NONE -1
#endif

// CHRISP TESTING.
//#define DO_IPV6 true

#ifdef DO_IPV6
#define mysockaddr_in sockaddr_in6

#else
#define mysockaddr_in sockaddr_in
#endif

struct hostent *mygethostname( struct hostent *hostbuf, const char *host, int *herr, char **tmp);
int  tcp_buffer_flush2(Chan *c, char *s, int len);

int tcp_wouldblock(Chan *c);
void tcp_lib_msleep(int x);

#ifndef NO_HCOUNT
	int hcount_init(void);
	int hcount_open_real(int n, char *filename, int line);
	int hcount_close_real(int n, char *filename, int line);
	int hcount_report(void);
#	define hcount_open(n) hcount_open_real(n,__FILE__,__LINE__)
#	define hcount_close(n) hcount_close_real(n,__FILE__,__LINE__)
#else
#	define hcount_open(n)
#	define hcount_close(n)
#endif
static int log_closes;

static int do_ipv6 = FALSE;
static int all_bigbf;
static int all_sz = 65536;
int tcp_write_ssl(Chan *c, char *bf, int len);
int tcp_setblockraw(SOCKET sk, int t);
void tcp_close_on_exec_raw(int fd);
static int (*emsg)( const char *format ,...);
static int (*dmsg)( const char *format ,...);
int tcp_lookup_ipv6(char *xname, char *ip, int ipv4);
static int isdebug;
#undef strlwr
#define strlwr tcplib_strlwr
#ifdef ncpy
#undef ncpy
#endif 
#define ncpy tcplib_ncpy
#ifndef WIN32
#ifndef SOCKET_ERROR
#define SOCKET_ERROR -1
#endif
#endif



char *ncpy(char *dst, char *src, int len);
char *strlwr(char *s);
int tcp_getip_raw(SOCKET sk, char *ipbf);
int tcp_getmyip_raw(SOCKET sk, char *ipbf);
int tcp_grow_inbf(Chan *c, int len);
int tcp_socket_close(SOCKET sk, char *src, int line);
int tcp_lib_isnt(void);
int tcp_sk_wouldblock(void);
int tcp_bf_free(char **pbf);
char *tcp_bf_get(int len);
/*
#ifdef WIN32
#ifdef WIN64
#define BIGWINDOW 262144
#else

#endif
#else
#define BIGWINDOW 131072
#endif
*/
int tcp_socket_close_hard(SOCKET sk);
int  imsg(const char * arg_list, ...);
int tcp_save_sk(SOCKET sk, int isopen);
static int trace_socket=FALSE;
int tcp_lookup_raw(char *xname, char *ip, int forceip4);

#ifdef WIN32
#define inet_ntop InetNtop
#endif

static int fix_ipv4;
int tcp_remove_prefix(char *ipbf);

int tcp_set_ipv6(int x)
{
	do_ipv6 = x;
	return TRUE;
}
int fake_dmsg( const char *format ,...)
{
	return 0;
}
static int (*open_hist)(IHANDLE handle, int isopen, int logonly, char *filename, int line, char *info);

#ifdef DO_THREADS
#include "thread.h"
static Mutex *m,*m2,*m3;
#endif

#ifndef USE_DMALLOC
void dmalloc_ignore_leak(void){}
#endif

#ifndef MUTEX_TIME
#define MUTEX_TIME 600000
#endif

struct in_addr_win {
  union {
    struct {
      u_char s_b1;
      u_char s_b2;
      u_char s_b3;
      u_char s_b4;
    } S_un_b;
    struct {
      u_short s_w1;
      u_short s_w2;
    } S_un_w;
    unsigned long S_addr;
  } S_un;
};

char * addr_to_string(struct in_addr x, char *bf)
{
#ifdef WIN32
	sprintf(bf,"%d.%d.%d.%d",x.S_un.S_un_b.s_b1,x.S_un.S_un_b.s_b2,x.S_un.S_un_b.s_b3,x.S_un.S_un_b.s_b4);
#else
	struct in_addr_win w;
	memcpy(&w,&x,4);
	sprintf(bf,"%d.%d.%d.%d",w.S_un.S_un_b.s_b1,w.S_un.S_un_b.s_b2,w.S_un.S_un_b.s_b3,w.S_un.S_un_b.s_b4);
#endif
	return bf;
}

static int mutex_i;
static int (*mutex_cb)(int vm, int isget, int seconds);
int tcp_set_mutex(int (*cb)(int vm, int isget, int seconds), int m)
{
		mutex_i = m;
		mutex_cb = cb;
		return TRUE;
}
void mygethostname_free(char **tmp)
{
	if (*tmp!=NULL) free(*tmp);
	*tmp = NULL;
}
// we will implement one for windows.

#ifdef WIN32
#ifndef GUI_INSTALLER  // but not borland...
#define USE_NET_R 1
#define WIN_NETR 1
#endif
#endif

#ifdef WIN32
#ifdef USE_NET_R

int gethostbyname2_r(const char *name, int af, struct hostent *phost, char *buf, size_t buflen, struct hostent **result, int *herrnop)
{
    struct addrinfo hints,*res=NULL,*a;
	int ret;
	int on=-1;
	char sport[BFSZ];
	char *s;
	char *xname=(char *) name;
	char xname2[BFSZ];
	char ip[BFSZ];
	char *rest;

	// setup default return info
	*herrnop = 0;
	*result = NULL;
	memset(buf,0,buflen);
	memset(phost,0,sizeof(struct hostent));
	strcpy(sport,""); // No port information...

	memset(&hints, 0, sizeof (hints));
    hints.ai_family = AF_INET;
    hints.ai_socktype = SOCK_STREAM;
	if (tcp_isip((char *)name)) hints.ai_flags = AI_NUMERICHOST;
	if (*xname=='[') {		
		ncpy(xname2,xname+1,BFSZ-1);
		s = strchr(xname2,']'); 
		if (s!=NULL) *s = 0;
		xname = xname2;
	}
    ret = getaddrinfo(xname, sport, &hints, &res);
	if (ret != 0) {
		if (res!=NULL) freeaddrinfo(res); res = NULL;
		*herrnop = HOST_NOT_FOUND;
		return 1;  // failed
	}
    for (a=res;a!=NULL;a=a->ai_next) {
		int iplen = 100;
		if (a->ai_addrlen<4) continue;
		if (a->ai_family!=AF_INET) continue; 
		*result = phost;
		phost->h_addrtype = AF_INET;
		//a->ai_socktype;  // SOCK_STREAM
		phost->h_length = 4; // a->ai_addrlen;
		
		phost->h_addr_list = (void *) buf;
		rest = buf+sizeof(void *)*2;
		phost->h_addr_list[0] = rest;
		phost->h_addr_list[1] = NULL;

		memcpy(rest,a->ai_addr->sa_data+2, 4); // (int)a->ai_addrlen);

		addr_to_string(*(struct in_addr *)rest,ip);
		imsg("tcp: gethostbyname(%s) - result (%s)\n",name,ip);

//		WSAAddressToString((LPSOCKADDR)rest,4,NULL,ip,&iplen); 

        return 0;
	}
	if (res!=NULL) freeaddrinfo(res); res = NULL;
	*herrnop = HOST_NOT_FOUND;
	return 1;	 // no addresses found.
}

int gethostbyaddr_r(const void *addr, int len, int type, struct hostent *phost, char *buf, size_t buflen, struct hostent **result, int *herrnop)
{
	int ret;
	char AddrName[NI_MAXHOST];

	// setup default return info
	*herrnop = 0;
	*result = NULL;
	memset(buf,0,buflen);
	memset(phost,0,sizeof(struct hostent));


	strcpy(AddrName,"");
	ret = getnameinfo( (struct sockaddr*)&addr, len, AddrName, sizeof (AddrName), NULL, 0, NI_NUMERICHOST);
	if (ret != 0) {
		*herrnop = HOST_NOT_FOUND;
		return 1;  // failed
	}
	
	*result = phost;
	phost->h_name = buf;
	ncpy(buf,AddrName,buflen-1); // Place result in hostent structure...
	return 0;	 // no addresses found.
}

#endif 
#endif

struct hostent *mygethostname( struct hostent *hostbuf, const char *name, int *herr, char **tmp)
{
	struct hostent *hp;
	size_t hstbuflen;
	char *tmphstbuf;
	int res=0;
	*tmp = NULL;
	hstbuflen = 1024;
	tmphstbuf = malloc (hstbuflen);
	*tmp = tmphstbuf; // chrisp added, it was leaking memory...

#ifdef USE_NET_R
#ifdef is_solaris
	hp = gethostbyname_r(name,hostbuf,tmphstbuf,hstbuflen,herr);
	if (hp==NULL) mygethostname_free(tmp);
	return hp;
#else	
	while ((res = gethostbyname2_r( name,AF_INET, hostbuf, tmphstbuf, hstbuflen,&hp, herr)) == ERANGE)
	{
		hstbuflen *= 2;
		tmphstbuf = realloc (tmphstbuf, hstbuflen);
	}
	*tmp = tmphstbuf;
	if (res || hp == NULL) {
		mygethostname_free(tmp);
		return NULL;
	}
	return hp;

#endif
#else
		hp = gethostbyname(name);
		*herr = errno;
		if (hp==NULL) mygethostname_free(tmp);

		return hp;
#endif
}
struct hostent *mygethostaddr(const void *addr,  struct hostent *hostbuf, int *herr, char **tmp)
{
	struct hostent *hp;
	size_t hstbuflen;
	char *tmphstbuf;
	int res=0;
	*tmp = NULL;
	hstbuflen = 2048;
	tmphstbuf = malloc (hstbuflen);

#ifdef USE_NET_R
#ifdef is_solaris
	hp = gethostbyaddr_r( addr,4,AF_INET,hostbuf, tmphstbuf, hstbuflen, herr);
	if (hp==NULL) mygethostname_free(tmp);

	return hp;
#else	
	//int gethostbyaddr_r(const void *addr, socklen_t len, int type, struct hostent *ret, char *buf, size_t buflen, struct hostent **result, int *h_errnop);
	while ((res = gethostbyaddr_r( addr,4,AF_INET,hostbuf, tmphstbuf, hstbuflen,&hp, herr)) == ERANGE)
	{
		hstbuflen *= 2;
		tmphstbuf = realloc (tmphstbuf, hstbuflen);
	}
	*tmp = tmphstbuf;
	if (res || hp == NULL) {
		mygethostname_free(tmp);
		return NULL;
	}
	return hp;

#endif
#else
		hp = gethostbyaddr(addr,4,AF_INET);
		*herr = errno;
		return hp;
#endif
}

int tcp_wait_write_raw(SOCKET sk, int msec, int *failed);

int tcp_mutex_wait(void)
{
	if (mutex_cb) return mutex_cb(mutex_i,TRUE,600);
#ifdef DO_THREADS
	if (m==NULL) return TRUE;
	if (!th_mutex_wait(m,MUTEX_TIME)) { /* 10 minutes */
		FILE *f;

		emsg("Signal Cannot get mutex, fatal problem, dieing tcp\n");
		f = fopen("startstop.log","a");
		if (f!=NULL) {
			fprintf(f,"Signal Cannot get mutex1, fatal problem\n");
			fclose(f);
		}
		lib_abort();
	}
	return TRUE;
#else
	return TRUE;
#endif
}
static int wait2_line;
static char thingtolookup[BFSZ];
void tcp_mutex_waitreal(int line);
void tcp_mutex_inside(void)
{
#ifdef DO_THREADS
	FILE *f;
	if (!th_mutex_wait(m2,MUTEX_TIME)) { 
		emsg("Signal Cannot get mutex2, fatal problem, dieing tcp held by line %d\n",wait2_line);
		f = fopen("startstop.log","a");
		if (f!=NULL) {
			fprintf(f,"Signal Cannot get mutex2, fatal problem, dieing tcp held by line %d %s\n",wait2_line,thingtolookup);
			fclose(f);
		}
		lib_abort();
	}
#endif

}
void tcp_mutex_wait2(int line)
{
#ifdef DO_THREADS
	if (m2==NULL) return;
	tcp_mutex_inside();
	wait2_line = line;
#endif
}

void tcp_mutex_wait3(void)
{
#ifdef DO_THREADS
	if (m3==NULL) return;
	if (!th_mutex_wait(m3,MUTEX_TIME)) { /* 10 minutes */

		emsg("Signal Cannot get mutex3, fatal problem, dieing tcp\n");
		lib_abort();
	}
#endif
}
void tcp_mutex_release(void)
{
	if (mutex_cb) { mutex_cb(mutex_i,FALSE,600); return; }
#ifdef DO_THREADS
	if (m==NULL) return;
	th_mutex_release(m);
#endif
}
void tcp_mutex_release2(void)
{
#ifdef DO_THREADS
	if (m2==NULL) return;
	th_mutex_release(m2);
#endif
}
void tcp_mutex_release3(void)
{
#ifdef DO_THREADS
	if (m3==NULL) return;
	th_mutex_release(m3);
#endif
}

int tcp_set_handle_cb(int (*cb)(IHANDLE handle, int isopen, int logonly, char *filename, int line, char *info))
{
// sockets are not from the same handle pool as file handles on windows!!!
#ifndef WIN32
	open_hist  = cb;
#endif
	return TRUE;
}
int tcp_trim_port(char *hostport, int dfltport);

void tcp_setdebug(int x)
{
	isdebug = x;
}
void tcp_init_dmsg(int (*vemsg)( const char *format ,...))
{
	dmsg = vemsg;
}
static int is_exit;
void tcp_init_wsa(void)
{
	static int done;
	if (done) return;
	done = TRUE;

#ifdef WIN32
	{
		WORD wVersionRequested;
		WSADATA wsaData;
		int err;
		wVersionRequested = MAKEWORD( 1, 1 );
#ifdef DO_IPV6
		wVersionRequested = MAKEWORD( 2, 2 );
#endif

		err = WSAStartup( wVersionRequested, &wsaData );
		if ( err != 0 ) {

			if (emsg) emsg("No winsock found of correct version\n"); return;
		}
#ifndef DO_IPV6
		if ( LOBYTE( wsaData.wVersion ) != 1 || HIBYTE( wsaData.wVersion ) != 1 ) {
				if (emsg) emsg("No winsock found of correct version, trying to continue\n");
		}
#endif

/* wmsg("Socket library found and is version 1.1, Great!\n"); */
/* The Windows Sockets DLL is acceptable. Proceed. */
	}
#endif
}

void tcp_init(int (*vemsg)( const char *format ,...))
{
#ifdef DO_THREADS
	static int done_mutex=FALSE;
#endif 
	emsg = vemsg;
	if (dmsg==NULL) dmsg = fake_dmsg;

#ifdef DO_THREADS
	if (!done_mutex) {
		done_mutex = TRUE;
		m = th_mutex_new("d_tcplib");
		m2 = th_mutex_new("d_tcplib2");
		m3 = th_mutex_new("d_tcplib3");
	}
#endif

#ifdef WIN32
	tcp_init_wsa();
#else
	signal(SIGPIPE,SIG_IGN);
#endif


}
int tcp_getports_raw(SOCKET sk, int *local, int *remote);
void tcp_setrcvbf_raw(SOCKET c, int sz);
void tcp_setsndbf_raw(SOCKET c, int sz);
void tcp_new_do(Chan **c, char *file, int line)
{
	if (*c!=NULL) {
		emsg("tcp_new, not passed null pointer\n");
//		fflush(NULL);
	}
#ifdef USE_DMALLOC
	*c = d_mallocid(DMID,sizeof(Chan),file,line);
#else
	*c = malloc(sizeof(Chan));
#endif
	memset(*c,0,sizeof(Chan));
	(*c)->canclose = FALSE;
	(*c)->start_time = time(NULL); 
	(*c)->id = 88;
	(*c)->sk = INVALID_SOCKET;
	(*c)->block = TRUE;
	tcp_grow_inbf(*c,2000);

}
int tcp_close_on_exec(Chan *c)
{
	if (c->sk==INVALID_SOCKET) return FALSE;
	tcp_close_on_exec_raw((int) c->sk);
	return TRUE;
}

#ifdef SAVE_LOW
static char low_used[4096];
static int low_line[4096];
int tcp_handle_dump(void)
{
	int i;
	FILE *f;
	f = fopen("save_low.log","w");
	for (i=300; i<1024; i++) {
		if (low_used[i]) {
			imsg("Socket Used [%d]\n",i);
			if (f!=NULL) fprintf(f,"Socket Used [%d] line=%d\n",i,low_line[i]);
		}
	}
	if (f!=NULL) fclose(f);
	return TRUE;
}
SOCKET tcp_socket_low(SOCKET sk, int line)
{
	int done=FALSE;
	int nfail=0;
	int i;
	int start=300;
	if (sk==INVALID_SOCKET) return sk;
	if (sk>300) { // Better record that it's used as well.
		if (sk<4000) {
			low_used[sk] = TRUE;
			low_line[sk] = line;
		}
		return sk;
	}
	tcp_mutex_wait();
again:
	for (i=start; i<4000; i++) {
		if ((!low_used[i])) {
			if (dup2(sk,i) == -1) continue;
			// Not recorded as hasn't yet been and is recorded by calling function.
			if (open_hist) open_hist(i,TRUE,TRUE,__FILE__,__LINE__,"a1");  // LOGGED ONLY CLOSE - SEARCHING FOR SOCKET
			if (open_hist) open_hist(sk,FALSE,TRUE,__FILE__,__LINE__,"a2"); // LOGGED ONLY OPEN  - SEARCHING FOR SOCKET  
			socket_close(sk);
			low_used[i] = TRUE;
			low_line[i] = line;
			tcp_mutex_release();
			return (SOCKET) i;
		}
	}
	if (!done) {
		done = TRUE;
		start = 60;
		goto again;
	}
	tcp_mutex_release();
	emsg("No handles free for socket_low, use ulimit -n 1024 before running\n");

	tcp_handle_dump();
	for (i=500; i<4000; i++) {
		if ((low_used[i])) {
			if (dup2(sk,i) != -1) emsg("Duping to %d worked\n",i);
			else emsg("Duping to %d FAILED \n",i);
			break;
		}
	}
	lib_abort();
}
#else
int tcp_handle_dump(void)
{
	return TRUE;
}
#endif

int tcp_socket_low_free(SOCKET xsk)
{
#ifdef SAVE_LOW
	int sk = (int) xsk;
	if (sk<0) return 0;
	if (sk>=4000) return 0;
	tcp_mutex_wait();
	low_used[sk] = FALSE;
	low_line[sk] = 0;
	tcp_mutex_release();
#endif
	return 0;
}

static int inherit_nicer;
int tcp_set_inherit(int t)
{
	inherit_nicer = t;
	return TRUE;
}
#ifdef WIN32
SOCKET tcp_dupaccept(SOCKET s, struct sockaddr FAR *addr, int FAR* addrlen, char *myname, char *theirname, int *lcl, int *rmt, char *whence)
{
	SOCKET sock, sock2;
	HANDLE p;
	int r;
	char info[BFSZ];

	strcpy(myname,"");
	strcpy(theirname,"");
	sock=accept(s, addr, addrlen);
	hcount_open((int)sock);
	if(sock==INVALID_SOCKET) return sock;

	*lcl = 0; *rmt = 0;
	tcp_getports_raw(sock,lcl,rmt);

	tcp_getip_raw(sock,theirname);
	tcp_getmyip_raw(sock,myname);
	sprintf(info,"%.100s:%d:%d:%.100s",theirname,*lcl,*rmt,whence);
	if (open_hist) open_hist((int)sock,TRUE,FALSE,"tcp_dupsocket1",__LINE__,whence); // REAL OPEN (dupaccept)

	if (inherit_nicer) {
		SetHandleInformation((HANDLE) sock,HANDLE_FLAG_INHERIT, 0);
		return sock;
	}

	p=GetCurrentProcess();
	r=DuplicateHandle(p, (HANDLE) sock, p,(HANDLE *) &sock2, 0, FALSE, DUPLICATE_SAME_ACCESS);


	tcp_save_sk(sock,FALSE);
	if (open_hist) open_hist((int)sock,FALSE,FALSE,"dupsocket",__LINE__,whence); // REAL CLOSE  (dupaccept)
	if (tcp_socket_close(sock,__FILE__,__LINE__)!=0) {
		if (tcp_sk_wouldblock()) {
			emsg("tcp_socket_close: Socket close failed will retry - wouldblock\n");
			tcp_socket_close_hard(sock);		
		} else {
			emsg("tcp_socket_close: Socket close failed will retry - notblock\n");
		}
	}
	hcount_close((int)sock);

	if(r==FALSE)
	{
		int i;
		LPVOID lpMsgBuf;

		i=GetLastError();
		FormatMessage( FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL,
		i, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR) &lpMsgBuf, 0, NULL);

		emsg("*** Warning *** Couldn't dup accepted socket: error %d (%s)", i, lpMsgBuf);
		LocalFree(lpMsgBuf);
		return INVALID_SOCKET;
	}
	hcount_open((int)sock2);
	if (open_hist) open_hist((int)sock2,TRUE,FALSE,"dupsocket",__LINE__,whence); // REAL OPEN , SETINHERIT
	return sock2;
}
SOCKET tcp_dupsocket(int af, int type, int protocol, char *whence)
{
	SOCKET sock, sock2;
	HANDLE p;
	int r;

	sock=socket(af, type, protocol);
	if(sock==INVALID_SOCKET)
		return sock;
	hcount_open((int)sock);
	if (open_hist) open_hist((int)sock,TRUE,FALSE,"tcp_dupsocket2",__LINE__,whence); // REAL OPEN DUPSOCKET

	if (lib_isnt351()) {
		if (SetHandleInformation((HANDLE)sock,HANDLE_FLAG_INHERIT,0)==0) {
			imsg("SetHandleInformation failed %d\n",GetLastError());			
		} 
		return sock;
	}

	p=GetCurrentProcess();
	r=DuplicateHandle(p,(HANDLE ) sock, p,(HANDLE *) &sock2, 0, FALSE, DUPLICATE_SAME_ACCESS);
	CloseHandle((HANDLE) sock);
	if(r==FALSE)
	{
		int i;
		LPVOID lpMsgBuf;

		i=GetLastError();
		FormatMessage( FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM, NULL,
		i, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPTSTR) &lpMsgBuf, 0, NULL);

		imsg("*** Warning *** Couldn't dup socket: error %d (%s)", i, lpMsgBuf);
		LocalFree(lpMsgBuf);
		return INVALID_SOCKET;
	}
	hcount_open((int)sock2);
	if (open_hist) open_hist((int)sock2,TRUE,FALSE,"dup_scoket",__LINE__,whence); // REAL OPEN DUPSOCKET
	return sock2;
}
#else
SOCKET tcp_dupaccept(SOCKET s, struct sockaddr *addr, int *addrlen, char *myname, char *theirname, int *lcl, int *rmt, char *whence)
{
	SOCKET sock;

	sock=accept(s, addr, addrlen);
#ifdef SAVE_LOW
	sock = tcp_socket_low(sock,__LINE__);
#endif
	*lcl = 0; *rmt = 0;
	if (sock!=INVALID_SOCKET) {
		char info[BFSZ];
		tcp_close_on_exec_raw(sock);
		hcount_open(sock);
		tcp_save_sk(sock,TRUE);
		tcp_getip_raw(sock,theirname);
		tcp_getmyip_raw(sock,myname);
		tcp_getports_raw(sock,lcl,rmt);

		sprintf(info,"%.100s:%d:%d:%.100s",theirname,*lcl,*rmt,whence);
		if (open_hist) open_hist((int)sock,TRUE,FALSE,"dupacceptd",__LINE__,whence); // RAEL OPEN DUPACCEPT
	}

	return sock;
}
SOCKET tcp_dupsocket(int af, int type, int protocol, char *whence)
{
	SOCKET sock, sock2;

	sock=socket(af, type, protocol);
	if (sock!=INVALID_SOCKET) {
#ifdef SAVE_LOW
		sock = tcp_socket_low(sock,__LINE__);
#endif
		tcp_close_on_exec_raw(sock);
		hcount_open(sock);
		if (open_hist) open_hist((int)sock,TRUE,FALSE,"tcp_dupsocket3",__LINE__,whence); // REAL OPEN DUPSOCKET
		tcp_save_sk(sock,TRUE);
	}
	return sock;
}
#endif


int tcp_accept(Chan *vlisten, Chan **newchan, char *whence)
{
	struct mysockaddr_in xsin;
	int sizesin=sizeof(xsin);
	Chan *c;
	int keepcnt = 6;
	int keepidle = 120; // increase to 120
	int keepintvl = 30;
	int flags =1;

	tcp_new(newchan);
	c = *newchan;

	if (vlisten->sk==INVALID_SOCKET) return FALSE; 

	c->sk = tcp_dupaccept(vlisten->sk,(struct sockaddr *) &xsin, &sizesin, c->myname,c->theirname,&c->lcl_saved,&c->rmt_saved, whence);

	c->never_close = FALSE;
	c->in_close = FALSE;
	tcp_lasterror(c);
	if (c->sk==INVALID_SOCKET)
		return FALSE;	
	tcp_close_on_exec(c);
	if (all_bigbf) {
		tcp_setrcvbf_raw(c->sk,all_sz);
	}
	// Lets set the keepalive settings:
#ifdef WIN32
#ifndef GUI_INSTALLER 
	{
		DWORD bytesreturned;
		struct tcp_keepalive {
			u_long  onoff;
			u_long  keepalivetime;
			u_long  keepaliveinterval;
		};
		struct tcp_keepalive vals;
		vals.onoff = 1;
		vals.keepalivetime = keepidle*1000;
		vals.keepaliveinterval = keepintvl*1000;
		WSAIoctl(
			c->sk,
			SIO_KEEPALIVE_VALS,
			(LPVOID) &vals,    // pointer to tcp_keepalive struct
			(DWORD) sizeof(vals),      // length of input buffer
			NULL,         // output buffer
			0,       // size of output buffer
			(LPDWORD) &bytesreturned,    // number of bytes returned
			NULL,   // OVERLAPPED structure
			NULL);
	}
#endif
#else
	// Enable keepalive, and set the values to something sensible (2 minutes + 6*30seconds,  so 5 minutes before socket released.
	setsockopt(c->sk, SOL_SOCKET, SO_KEEPALIVE, (void *)&flags, sizeof(flags));
#ifdef HASKEEP
	setsockopt(c->sk, IPPROTO_TCP, TCP_KEEPCNT, &keepcnt, sizeof(int));
#ifndef NOKEEPIDLE
	setsockopt(c->sk, IPPROTO_TCP, TCP_KEEPIDLE, &keepidle, sizeof(int));
#endif
	setsockopt(c->sk, IPPROTO_TCP, TCP_KEEPINTVL, &keepintvl, sizeof(int));
#endif 

#endif



	return TRUE;

}
int tcp_128kaccept(Chan *vlisten, Chan **newchan){
	Chan *c;
	int r = tcp_accept(vlisten, newchan,"128k");
	c = *newchan;
	if (r) {
		tcp_setrcvbf_raw(c->sk,all_sz);
		tcp_setsndbf_raw(c->sk,all_sz);
	}
	return r;
}
int tcp_out_bf_128k(Chan *newchan)
{
	Chan *c = newchan;
	tcp_setsndbf_raw(c->sk,all_sz);
	return TRUE;
}
int tcp_out_bf_set(Chan *newchan, int x)
{
	Chan *c = newchan;
	tcp_setsndbf_raw(c->sk,x);
	return TRUE;
}
int tcp_in_bf_set(Chan *newchan, int x)
{
	Chan *c = newchan;
	tcp_setrcvbf_raw(c->sk,x);
	return TRUE;
}
int tcp_in_bf_128k(Chan *newchan)
{
	Chan *c = newchan;
	tcp_setrcvbf_raw(c->sk,all_sz);
	return TRUE;
}
#ifdef WIN32
int tcp_accept_test(Chan *vlisten, Chan **newchan)
{
	struct mysockaddr_in xsin;
	int sizesin=sizeof(xsin);
	Chan *c;




	tcp_new(newchan);
	c = *newchan;
	strcpy(c->myname,"");
	strcpy(c->theirname,"");

	c->sk = accept(vlisten->sk,(struct sockaddr *) &xsin, &sizesin);
	c->never_close = FALSE;
	c->in_close = FALSE;
	hcount_open((int)c->sk);
	if (open_hist) open_hist((int)c->sk,TRUE,FALSE,__FILE__,__LINE__,"accept_test"); // REAL OPEN, ACCEPT_TEST
	tcp_save_sk(c->sk,TRUE);

	if (c->sk==INVALID_SOCKET) {
			tcp_lasterror(c);
		return FALSE;
	}

	tcp_getip_raw(c->sk,c->theirname);
	tcp_getmyip_raw(c->sk,c->myname);
	SetHandleInformation((HANDLE)c->sk,HANDLE_FLAG_INHERIT,0);
		tcp_lasterror(c);
	return TRUE;
}
#else
int tcp_accept_test(Chan *vlisten, Chan **newchan)
{
	return tcp_accept(vlisten,newchan,"nonetest");
}
#endif
#ifdef UCX
#define DOCAST
#endif
#ifdef WIN32
#define DOCAST
#endif
#ifdef NETWARE
#define DOCAST
#endif

#ifdef DOCAST
#define ONCAST (char *)
#else
#define ONCAST
#endif

int tcp_set_tcpbf(int sz)
{
	if (sz==0) return TRUE;
	if (sz<32000) return TRUE; // breaks linux if you set it too small.
	all_sz = sz;
	all_bigbf = TRUE;
	return TRUE;
}
int tcp_set_allbf(void)
{
	all_bigbf = TRUE;
	return TRUE;
}
#ifdef WIN32
static int listen_que_len = 150;
#else
static int listen_que_len = 128; // increased by chrisp, was 25... may get connection refused if exceeded..
#endif
int tcp_listen_que(int x)
{
	if (x==0) return FALSE;
	listen_que_len = x;
	return TRUE;
}
int tcp_listen_ipv4(Chan **cc, int port, char *bindin, int reuse);
int tcp_listen_ipv6(Chan **cc, int port, char *bindin, int reuse, int linkedlist);
int tcp_listen(Chan **cc, int port, char *bindin, int reuse)
{
#ifdef DO_IPV6
	if (do_ipv6) return tcp_listen_ipv6(cc,port,bindin,reuse,FALSE);
	else return tcp_listen_ipv4(cc,port,bindin,reuse);
#else
	return tcp_listen_ipv4(cc,port,bindin,reuse);
#endif
}

int tcp_listen_list(Chan **cc, int port, char *bindin, int reuse) // Identical but it is allowed to return a linked list of channels....
{
#ifdef DO_IPV6
	if (do_ipv6) return tcp_listen_ipv6(cc,port,bindin,reuse,TRUE);
	else return tcp_listen_ipv4(cc,port,bindin,reuse);
#else
	return tcp_listen_ipv4(cc,port,bindin,reuse);
#endif
}
int tcp_listen_ipv4(Chan **cc, int port, char *bindin, int reuse)
{
	struct sockaddr_in xsin;
	Chan *c=NULL;
	int r;
	int on=-1;
	tcp_new(cc);
	c = *cc;
	if (c==NULL) return FALSE;

	c->sk = tcp_dupsocket(AF_INET, SOCK_STREAM, 0,"listen");  // option 1
	c->never_close = FALSE;
	c->in_close = FALSE;
	tcp_lasterror(c);
	if (c->sk == INVALID_SOCKET)
		return FALSE;
	tcp_close_on_exec(c);
	if (reuse) {
		r = setsockopt(c->sk, SOL_SOCKET, SO_REUSEADDR,ONCAST &on, sizeof(int));
		if (r) {
		/* Err... what can you do, eh? */
		}
	}

	xsin.sin_family = AF_INET;
	xsin.sin_addr.s_addr = INADDR_ANY;
	xsin.sin_port = htons((short) port);

	if (bindin!=NULL) if (lib_strlen(bindin)>0) {
		xsin.sin_addr.s_addr = inet_addr(bindin);
	}

	if (bind(c->sk,(struct sockaddr *)  &xsin, sizeof (xsin)) != 0) {
		tcp_lasterror(c);
		return FALSE;
	}
	if (all_bigbf) {
		tcp_setrcvbf_raw(c->sk,all_sz);
		tcp_setsndbf_raw(c->sk,all_sz);
	}
	if (listen(c->sk, listen_que_len) < 0) {
		tcp_lasterror(c);
		return FALSE;
	}
	return TRUE;
}
#ifdef DO_IPV6
int tcp_listen_ipv6(Chan **cc, int port, char *bindin, int reuse, int linkedlist)
{
    struct addrinfo hints;
    int n;
	char sport[BFSZ];
	int on=-1;
	Chan *c=NULL;
	Chan **last=NULL;
	int r;
    struct addrinfo *result=NULL, *rp;
	SOCKET sfd=INVALID_SOCKET;

	*cc = NULL;


    memset(&hints, 0, sizeof(struct addrinfo));

	hints.ai_family = AF_UNSPEC;    /* Allow IPv4 or IPv6 */
	hints.ai_socktype = SOCK_STREAM; /* Stream socket */
	hints.ai_flags = AI_PASSIVE | AI_NUMERICHOST;    /* For wildcard IP address , and only return addresses for interfaces we actually have! */
#ifdef HAVE_ADDRCONFIG
	hints.ai_flags |= AI_ADDRCONFIG;
#endif 
	hints.ai_protocol = 0;          /* Any protocol */
	hints.ai_canonname = NULL;
	hints.ai_addr = NULL;
	hints.ai_next = NULL;
	int_to_str(port,sport);	

	if (lib_strlen(bindin)==0) bindin = NULL;
	if (bindin) if (lib_strlen(bindin)>0) {
		hints.ai_flags |= AI_NUMERICHOST;
	}

    n = getaddrinfo(bindin, sport, &hints, &result);
    if (n <0) {
		goto failed;
    }

	// Iterate through results and  bind to first one we can (should probably bind to all of them, but not critical) 
	for (rp = result; rp != NULL; rp = rp->ai_next) {
		sfd = tcp_dupsocket(rp->ai_family, rp->ai_socktype, rp->ai_protocol,"ipv6"); // option 2
		if (sfd == -1) continue;
		if (bind(sfd, rp->ai_addr, (int) rp->ai_addrlen) == 0)  {
			c = NULL; 
			tcp_new(&c); if (c==NULL) break;
			if (*cc==NULL) *cc = c;
			else *last = c;
			last = &(c->next);
			c->sk = sfd;	
			if (listen(c->sk, listen_que_len) < 0) {
				tcp_lasterror(c);
				// don't care if this fails...
			}
			if (reuse) {
				r = setsockopt(sfd, SOL_SOCKET, SO_REUSEADDR,ONCAST &on, sizeof(int));
			}
			tcp_close_on_exec(c);
			if (!linkedlist) break;
		} else {
			if (open_hist) open_hist((int)sfd,FALSE,FALSE,__FILE__,__LINE__,"ipv6"); // REAL CLOSE, LISTEN IPV6
			tcp_save_sk(sfd,FALSE);
			tcp_socket_close(sfd,__FILE__,__LINE__);
			hcount_close((int)sfd);
			sfd = INVALID_SOCKET;
		}
	}
	if (result!=NULL) freeaddrinfo(result); result = NULL;
	if (c==NULL) goto failed;
	tcp_lasterror(c);
	if (sfd == INVALID_SOCKET) goto failed;
	return TRUE;
failed:
	c = NULL;
	tcp_new(&c);
	if (*cc==NULL) *cc = c;
	tcp_lasterror(c);
	return FALSE;
}
#endif 

int tcp_isip(char *s)
{
	// Also allow [hex:hex:hex:hex]  (ipv6 format address)
	int hexok=FALSE;
	if (strchr(s,':')!=NULL) hexok = TRUE; 
	for (;*s!=0;s++) {
		if (*s=='[') {hexok=TRUE; continue;}
		if (*s=='%') continue; // seems to be on the end of valid addresses, no idea what %12 means...
		if (*s=='/') continue; // seems to be on the end of valid addresses, no idea what %12 means...
		if (*s==']') continue;
		if (*s=='.') continue;
		if (*s==':') continue;
		if (sisdigit(*s)) continue;
		if (hexok) {
			if ((*s >= 'a') && (*s <= 'f')) continue; // hex letters too...
			if ((*s >= 'A') && (*s <= 'F')) continue;
		}
		return FALSE;
	}
	return TRUE;
}
int tcp_notipv4(char *s)
{
	for (;*s!=0;s++) {
		if (*s=='[') return TRUE;
		if (*s=='%') return TRUE;
		if (*s=='/') return TRUE;
		if (*s==']') return TRUE;
	}
	if (lib_strlen(s)>15) return TRUE;
	return FALSE;
}
void tcp_set_closed(Chan *c)
{
	if (c==NULL) return ;
	c->xerrno = -1;
}
int tcp_isopen(Chan *c)
{
	if (c==NULL) return FALSE;
	if (c->sk==INVALID_SOCKET) return FALSE;
	return TRUE;
}
#ifdef is_linux
#define USE_INET_NTOP 1
#endif
// chrisp added next bit to fix ntoa error on osx system, maybe due to threading unsafe with inet_ntoa...
#ifdef ISOSX
#define USE_INET_NTOP 1
#endif


char *inet_ntoa_safe(struct in_addr in, char *tmp)
{
#ifdef USE_INET_NTOP
	inet_ntop(AF_INET,&in,tmp,BFSZ-1);
	return tmp;
#else
	strcpy(tmp,"0.0.0.0");
	if (inet_ntoa(in)!=NULL) ncpy(tmp,inet_ntoa(in),BFSZ-1);
	return tmp;
#endif
}

int tcp_lookup_all(char *xname, Wordlist **res) // THIS IS NOT USED BY ANYTHING IMPORTANT/CRITICAL, SO NO NEED TO CONVERT TO IPV6
{
	time_t start = time(NULL);
	struct hostent hresult;
//	char hbuffer[BFSZ];
	int h_err = 0;
	struct hostent *hp;
	char name[BFSZ];
	char bf[BFSZ];
	char *s;
	int i,x;
	char *tmp=NULL;

	ncpy(name,xname,BFSZ-1);
	s = strchr(name,':');
	if (s) *s = 0;

#ifdef MUTEX_GETHOST
	tcp_mutex_wait2(__LINE__);
#endif
	ncpy(thingtolookup,name,BFSZ-1);

	if (tcp_isip(name)) {
		unsigned int addr = inet_addr(name);
		if (addr == INADDR_NONE) {
#ifdef MUTEX_GETHOST // chrisp added, but shouldn't matter
			tcp_mutex_release2();
#endif
			return 0;
		}
		memset(&hresult,0,sizeof(hresult));
		hp = mygethostaddr((const char*)&addr,&hresult,&h_err,&tmp);
	}
	else {
		memset(&hresult,0,sizeof(hresult));
		hp = mygethostname(&hresult,name,&h_err,&tmp);
	}

#ifdef MUTEX_GETHOST
	tcp_mutex_release2();
	if (time(NULL)-start>4) imsg("gethostbyname1 (%s) took %d seconds, disable g_lookup_names\n",name,time(NULL)-start);
#endif

	if (!hp) {
		mygethostname_free(&tmp);
		return 0;
	}

	for(i = x = 0; hp->h_addr_list[i]; i++) {
		inet_ntoa_safe(*(struct in_addr*)hp->h_addr_list[i],bf);
		word_add(res,bf);
		x++;
	}
	mygethostname_free(&tmp);

	return x;
}
int tcp_trim_port(char *hostport, int dfltport)
{
	char *last=NULL;
	int isok=FALSE;
	int nc=0;
	char *s=hostport;
	if (*s=='[') {
		hostport = strchr(hostport,']');
		if (hostport==NULL) return dfltport;
	} else {
		if (!isok) { // Figure out if this is an ipv6 or ipv4 address, if it's ipv4 then skip to the correct colon... 
			for (s=hostport;*s!=0;s++) if (*s==':') {last=s; nc++;}
			if (nc==7) return dfltport;
			if (nc==8) {
				*last = 0;
				dfltport = atoi(last+1);
				return dfltport;
			}
			// chrisp fix, some addresses seem to not have 7 colons like we expect... 
			if (nc>3) return dfltport; 
		}
	}
	s = strchr(hostport,':');
	if (s!=NULL) {
		*s = 0;
		dfltport = atoi(s+1);
	}
	return dfltport;
}
#ifdef DO_IPV6
int tcp_lookup_ipv6(char *xname, char *ip, int ipv4)
{
	struct addrinfo hints,*result=NULL,*ptr;
	int ret=FALSE;
	int r;
	int len=100-1;

	strcpy(ip,"");
	// Get an ascii reprensentation of the hostname 'xname'....
	memset(&hints, 0, sizeof(hints));
	hints.ai_family = AF_UNSPEC; // IPV4 OR IPV6
	if (ipv4) {
		hints.ai_family = AF_INET; // IPV4
	}
	hints.ai_socktype = SOCK_STREAM;
#ifdef HAVE_ADDRCONFIG
	hints.ai_flags = AI_ADDRCONFIG;
#endif

	r = getaddrinfo(xname, "", &hints, &result);
    if ( r != 0 ) goto cleanup; 
    
    // Retrieve each address and print out the hex bytes
    for(ptr=result; ptr != NULL ;ptr=ptr->ai_next) {
		void *xp=NULL;
		// choose the best option given our 'mode'... 
		len = 99;

		if (ptr->ai_family==AF_INET6 || (ipv4 && (ptr->ai_family==AF_INET))) {
#ifdef WIN32
			WSAAddressToString(ptr->ai_addr,ptr->ai_addrlen,NULL,ip,&len);

#else
	   switch (ptr->ai_family)
		{
		case AF_INET:
		  xp = &((struct sockaddr_in *) ptr->ai_addr)->sin_addr;
		  break;
		case AF_INET6:
		  xp = &((struct sockaddr_in6 *) ptr->ai_addr)->sin6_addr;
		  break;
		}
        inet_ntop (ptr->ai_family, xp, ip, 100);
//		//inet_ntop(ptr->ai_family,ptr->ai_addr->sa_data+2,ip,100-1); // this crazy thing should work I think..
#endif
			break;
		}
    }

cleanup:
    if (result!=NULL) freeaddrinfo(result);   
	if (lib_strlen(ip)>0) ret = TRUE;
	return ret;
}
#endif
int tcp_lookup(char *xname, char *ip)
{
	return tcp_lookup_raw(xname,ip,FALSE);
}
int tcp_lookup_raw(char *xname, char *ip, int forceip4)
{
	struct sockaddr_in xsin;
	struct hostent *hp;
	int dontblock = FALSE;
	char name[1000];

	int dst_port=90;
	const char optval=44;
	struct hostent hresult;
	char hbuffer[BFSZ];
	char itmp[BFSZ];
	int h_err=0;
	char *tmp=NULL;
	time_t start=time(NULL);
	
#ifdef DO_IPV6
	if (do_ipv6 && !forceip4) return tcp_lookup_ipv6(xname,ip,FALSE);
#endif
#ifdef WIN_NETR
	return tcp_lookup_ipv6(xname,ip,TRUE);
#endif
	strcpy(hbuffer,"");
	memset(&hresult,0,sizeof(hresult));
	
	strcpy(ip,"");
	strncpy(name,xname,200);
	memset(&xsin,0,sizeof(xsin));
	xsin.sin_family = AF_INET;
	
	dst_port = tcp_trim_port(name,dst_port);
	
	if (tcp_isip(name)) {
		xsin.sin_addr.s_addr = inet_addr(name);
		xsin.sin_family = AF_INET;
	} else {
		xsin.sin_addr.s_addr = 0;
		imsg("tcp_lookup(%s)\n",name);

#ifdef MUTEX_GETHOST
		tcp_mutex_wait2(__LINE__);
#endif
		ncpy(thingtolookup,name,BFSZ-1);
//		hp = mygethostbyname_r(name,&hresult,hbuffer,sizeof(hbuffer),&h_err); // it's this one that crashes... 
		hp = mygethostname(&hresult,name,&h_err,&tmp);
		if (hp==NULL) imsg("mygethostname %s returned null, err=%d\n",name,h_err);


#ifdef MUTEX_GETHOST
		tcp_mutex_release2();
		if (time(NULL)-start>4) imsg("gethostbyname1 (%s) took %d seconds, disable g_lookup_names\n",name,time(NULL)-start);
#endif
		if (hp==NULL) return FALSE;
		if (hp->h_addr==NULL) {
			imsg("mygethostname->h_addr is null %s %d\n",name,h_err);
			return FALSE;
		}

		safe_memcpy(&(xsin.sin_addr),hp->h_addr,  4,&(xsin.sin_addr),4);
		xsin.sin_family = hp->h_addrtype;
	}
	strcpy(ip,inet_ntoa_safe(xsin.sin_addr,itmp));

	mygethostname_free(&tmp);
	return TRUE;
}

#ifdef DO_IPV6
SOCKET tcp_open_raw_ipv6(Chan *c, char *xname,int dst_port, int reuse, char *bindout, int ipv4only)
{
    // By not setting the AI_PASSIVE flag in the hints to getaddrinfo, we're
    // indicating that we intend to use the resulting address(es) to connect
    struct addrinfo hints,*res=NULL,*a;
	int ret;
	int on=-1;
	char tbf[BFSZ];
	char sport[BFSZ];
	SOCKET sk;
	char *s;
	char xname2[BFSZ];
	char lclname[BFSZ];
	ncpy(lclname,xname,BFSZ-1); xname = lclname; // so we don't modify the input by mistake...

	dst_port = tcp_trim_port(xname,dst_port);
	sprintf(sport,"%d",dst_port);

    memset(&hints, 0, sizeof (hints));
    hints.ai_family = AF_INET6;
	
	if (ipv4only) {
	    hints.ai_family = AF_INET;
		
	}
    hints.ai_socktype = SOCK_STREAM;
	if (tcp_isip(xname)) hints.ai_flags = AI_NUMERICHOST;
	if (*xname=='[') {		
		ncpy(xname2,xname+1,BFSZ-1);
		s = strchr(xname2,']'); 
		if (s!=NULL) *s = 0;
		xname = xname2;
	}
	imsg("tcp: ipv6 open (%s) port %d\n",xname,dst_port);


    ret = getaddrinfo(xname, sport, &hints, &res);
    if (ret != 0) {
		if (!ipv4only) {
			// imsg("tcp: gai_strerror: (%s) (%s)\n",xname,gai_strerror(ret));
		    hints.ai_family = AF_UNSPEC; // try ipv4 too...


		    ret = getaddrinfo(xname, sport, &hints, &res);
		}
		if (ret!=0) {

			imsg("tcp: ipv6 open (%s) port %d FAILED %s\n",xname,dst_port,gai_strerror(ret));
			tcp_lasterror(c);
			return INVALID_SOCKET;
		}
    }
    for (a=res;a!=NULL;a=a->ai_next) {
		int len=BFSZ-1;
//		char ip[BFSZ];
		//printf("Open attempt %x %x\n",a->ai_family,a->ai_flags);
		if (a->ai_family==AF_INET6 || ((a->ai_family==AF_INET))) {
#ifdef WIN32
			//WSAAddressToString(a->ai_addr,a->ai_addrlen,NULL,ip,&len);
			//printf("Open address returned (%s)\n",ip);
#endif
		}
	}
    for (a=res;a!=NULL;a=a->ai_next) {
		int r; 
		//printf("Open attempt %x %x %s\n",a->ai_family,a->ai_flags,c->noblock?"NoBlock":"Block");
        sk = socket(a->ai_family, a->ai_socktype, a->ai_protocol);
		if (sk == INVALID_SOCKET) {
			imsg("tcp: Socket creation failed\n");
			continue;
		}
		// Line added here so log will show different entry
		if (open_hist) open_hist((int)sk,TRUE,FALSE,__FILE__,__LINE__,"ipv6-raw"); // record the handle as open.   -- crashing  due to handle not set as closed? 
		if (reuse || (c->lclport!=0)) {
			r = setsockopt(sk, SOL_SOCKET, SO_REUSEADDR,ONCAST &on, sizeof(int));
			tcp_lasterror(c);
		}
		if (c->noblock) { // chrisp added, so ipv6 is attempted without blocking...
			if (!tcp_setblockraw(sk,FALSE)) {
				tcp_lasterror(c);
			}
		} 

		r = connect(sk, a->ai_addr, (int)a->ai_addrlen);
		if (r == 0) {
			//printf("Connect worked, so far so good\n");
			break;
		}
		tcp_lasterror(c);
		if (tcp_wouldblock(c)) {
			imsg("tcp: woudblock true, so we will use this one \n");
			break;
		}
		imsg("tcp: connect error %s\n",tcp_error(c,tbf));
		if (open_hist) open_hist((int)sk,FALSE,FALSE,"open_ipv6r",__LINE__,"openipv6r"); // REAL CLOSE ipv6 failure
		tcp_socket_close_hard(sk); sk = INVALID_SOCKET;  // close_hard does NOT DO the open_hist call to close the socket 
    }
	if (res!=NULL) freeaddrinfo(res); res = NULL;

	
	if (sk==INVALID_SOCKET) return INVALID_SOCKET;

	tcp_close_on_exec(c);
	if (c->noblock) {
		if (!tcp_setblockraw(sk,FALSE)) {
			tcp_lasterror(c);
		}
	}
	


	return sk;
}
#endif



#ifdef DO_IPV6
#ifndef WIN32
typedef struct in6_addr IN6_ADDR;
#endif
int tcp_match_ipv6(IN6_ADDR *addr, IN6_ADDR *mask, int bits)
{
	int i, m, n = bits / 8;


	for (i = 0; i < n; i++)
		if (addr->s6_addr[i] != mask->s6_addr[i])
			return 0;
	if ((m = bits % 8) == 0)
		return 1;
	if ((addr->s6_addr[n] & ~((1 << (8 - m)) - 1)) == mask->s6_addr[n])
		return 1;
	return 0;
}
int tcp_cidr_ipv6(char *address, char *cidr)
{
	char mask[BFSZ];
	char *s;
	int bits;
	IN6_ADDR ad,m;
	if (!tcp_string_to_addr_ipv6(address, &ad)) {
		return FALSE;
	}
	ncpy(mask,cidr,BFSZ-1);
	s = strchr(mask,'/'); if (s==NULL) {
		return FALSE;
	}
	*s = 0;
	bits = atoi(s+1);
	if (!tcp_string_to_addr_ipv6(mask, &m)) {
		return FALSE;
	}
	return tcp_match_ipv6(&ad,&m,bits);	
}

int tcp_string_to_addr_ipv6(char *xname, struct in6_addr *addr)
{
	struct addrinfo hints,*result=NULL,*ptr;
	int ret=FALSE;
	int r;
	int len=BFSZ-1;
	result = NULL;

	// Get an ascii reprensentation of the hostname 'xname'....
	memset(&hints, 0, sizeof(hints));
	hints.ai_family = AF_UNSPEC; // IPV4 OR IPV6
	hints.ai_socktype = SOCK_STREAM;
	hints.ai_flags = AI_NUMERICHOST;

	r = getaddrinfo(xname, "", &hints, &result);
    if ( r != 0 ) goto cleanup; 
    
    // Retrieve each address and print out the hex bytes
    for(ptr=result; ptr != NULL ;ptr=ptr->ai_next) {
		struct sockaddr_in6 *x;
		if (ptr->ai_family==AF_INET6) {
			x = (struct sockaddr_in6 *) ptr->ai_addr;
			memcpy(addr,&x->sin6_addr,sizeof(IN6_ADDR));
			ret = TRUE;
			break;
		}
    }

cleanup:
    if (result) freeaddrinfo(result);   
	return ret;
}
#else
int tcp_cidr_ipv6(char *address, char *cidr)
{
	return FALSE;
}

#endif





SOCKET tcp_open_raw(Chan *c, char *xname,int dst_port, int reuse, char *bindout, char *whence)
{
	struct sockaddr_in xsin;
	struct sockaddr_in xlcl;
	SOCKET sk;
	char openinfo[BFSZ];
	struct hostent *hp;
	int st;
	int dontblock = FALSE;
	char name[1000];
	char bf[1000];
	char *s;
	int on=-1;
	int didbind=FALSE;
	SOCKET r;
	const char optval=44; //non zero integer
	struct hostent hresult;
	char hbuffer[BFSZ];
	int h_err=0;
	int done_again=FALSE;
	char *tmp=NULL;
	time_t start=time(NULL);


	if (bindout==NULL) bindout = "";
//printf("tcp_open %s\n",xname);

	strncpy(name,xname,200);

	if (lib_strlen(xname)==0) {
		sprintf(c->erradd,"Host Blank");
		return INVALID_SOCKET;
	}
	strlwr(name); /* Seems to be required by NIS */
	s = strstr(name,"/bind=");
	if (s!=NULL) {
		*s = 0;
		bindout = s+6;
		imsg("tcp: open (%s) bind to (%s)\n",name,bindout);
	}  imsg("tcp: open (%s) port=%d binding to (%s)\n",name,dst_port,bindout);



again:


/*	this code doesn't work, so rmeoved it.
if (bindout!=NULL) if (strlen(bindout)>0) if (!tcp_notipv4(bindout)) {
		imsg("tcp: Binding to (%s) which is ipv4 address so not trying ipv6 connection\n",bindout);
		goto not_ipv6;
	}
*/
#ifdef DO_IPV6
	if (do_ipv6) {
		char tmp[BFSZ];
		imsg("tcp: Try ipv6 open(%s)\n",name);
//printf("tcp: Try ipv6 open(%s)\n",name);
		r = tcp_open_raw_ipv6(c,name,dst_port,reuse,bindout,FALSE);
//printf("tcp: raw open result %d\n",r);
		if (r!=INVALID_SOCKET) {
			int x;
			int failed;
			int wait = c->read_timeout;
			if (wait==0) wait = 3*60*1000;
			imsg("tcp: And wait for ipv6 open (%s) event %d\n",name,wait);
//printf("tcp: And wait for ipv6 open (%s) event %d\n",name,wait);

			x = tcp_wait_write_raw(r,wait,&failed);
//if (x) printf("tcp: socket seems to be ready to write to\n");
			if (!failed) if (x) return r;
			if (open_hist) open_hist((int)r,FALSE,FALSE,"open_ipv6",__LINE__,"tcp_open_raw"); // REAL CLOSE ipv6 failure
			tcp_socket_close_hard(r);
			r = INVALID_SOCKET;
		} else {
//printf("tcp: raw open failed \n");
		}
		imsg("tcp: Try ipv4 open(%s) since ipv6 failed -  %s\n",name,tcp_error(c,tmp));
//printf("tcp: Try ipv4 open(%s) since ipv6 failed -  %s\n",name,tcp_error(c,tmp));
		
	} 
#endif
	//not_ipv6:
	strcpy(hbuffer,"");
	memset(&hresult,0,sizeof(hresult));
	strcpy(c->myname,"");
	strcpy(c->theirname,"");
	memset(&xsin,0,sizeof(xsin));
	memset(&xlcl,0,sizeof(xlcl));
	xsin.sin_family = AF_INET;
/*	Moved up above ipv6 so /bind is removed before we try dns lookup/open...
	if (bindot==NULL) bindout = "";
	strlwr(name); 
	s = strstr(name,"/bind=");
	if (s!=NULL) {
		*s = 0;
		bindout = s+6;
		imsg("tcp: open (%s) bind to (%s)\n",name,bindout);
	} else imsg("tcp: open (%s) binding to (%s)\n",name,bindout);

*/	
	dst_port = tcp_trim_port(name,dst_port);


//printf("tcp: Tryin ipv4 open(%s) \n",name);
	if (!tcp_isip(name)) {
		char rawip[BFSZ];


		if (!tcp_lookup_raw(name,rawip,TRUE)) { // do an ipv4 lookup
			tcp_lasterror(c);
			sprintf(c->erradd,"tcp_lookup failed");

			return INVALID_SOCKET;		
		}
		imsg("tcp_open_raw: converted (%s) to (%s)\n",name,rawip);
		ncpy(name,rawip,BFSZ-1);
	}

	if (tcp_isip(name)) { // this code path should 'ALWAYS' happen now... 
		xsin.sin_addr.s_addr = inet_addr(name);
		xsin.sin_family = AF_INET;


	} else {

		imsg("tcp: gethostbyname(%s)\n",name);
		xsin.sin_addr.s_addr = 0;
#ifdef MUTEX_GETHOST
		tcp_mutex_wait2(__LINE__);  // mutex crash here on osx during probe....
#endif
		ncpy(thingtolookup,name,BFSZ-1);
//		hp = mygethostbyname_r(name,&hresult,hbuffer,sizeof(hbuffer),&h_err);
		hp = mygethostname(&hresult,name,&h_err,&tmp);
		c->xerrno = h_err;
		imsg("tcp: gethostbyname(%s) - finished\n",name);
		tcp_lasterror(c);
#ifdef MUTEX_GETHOST
		tcp_mutex_release2();
		if (time(NULL)-start>4) imsg("gethostbyname2 (%s) took %d seconds, disable g_lookup_names\n",name,time(NULL)-start);
#endif		
		sprintf(c->erradd,"gethostbyname failed");

		if (hp==NULL) return INVALID_SOCKET;
		if (hp->h_length<=0) return  INVALID_SOCKET;
		if (hp->h_length<4) return  INVALID_SOCKET; // chrisp addded, the hp address length must be 4 at least...
		if (hp->h_length>sizeof(struct in_addr)) return  INVALID_SOCKET;
		if (hp->h_addr == NULL) return INVALID_SOCKET;
		safe_memcpy(&(xsin.sin_addr),hp->h_addr,  4,&(xsin.sin_addr),4);   /// DID i GET A CRASH HERE?    ?? another crash!!! 
//		safe_memcpy(&(xsin.sin_addr),hp->h_addr,  hp->h_length,&(xsin.sin_addr),4);   /// DID i GET A CRASH HERE? 


#ifdef WIN32
		{
			char ip[BFSZ];
			addr_to_string(xsin.sin_addr,ip);
			imsg("tcp: gethostbyname(%s) - result (%s)\n",name,ip);
		}
#endif


		xsin.sin_family = hp->h_addrtype;
	}

	mygethostname_free(&tmp);
	sprintf(c->erradd,"");
	
	sprintf(openinfo,"%.100s:%d:%.100s",whence,dst_port,name);
	sk = tcp_dupsocket(xsin.sin_family,SOCK_STREAM, 0,openinfo); // option 3
	c->never_close = FALSE;
	c->in_close = FALSE;
	/*	emsg("open, created socket %d\n",sk);*/
	tcp_lasterror(c);
	if (sk==INVALID_SOCKET) {
		imsg("Can't dupsocket connect to {%s} \n",name);
		sprintf(c->erradd,"dupsocket failed");
		return INVALID_SOCKET;
	}
	tcp_close_on_exec(c);
	if (c->noblock) {
		if (!tcp_setblockraw(sk,FALSE)) {
			tcp_lasterror(c);

		}
	}
	
	/*		r = setsockopt(c->sk, SOL_SOCKET, SO_REUSEADDR,ONCAST &on, sizeof(int));*/
	if (reuse || (c->lclport!=0)) {
		r = setsockopt(sk, SOL_SOCKET, SO_REUSEADDR,ONCAST &on, sizeof(int));
		tcp_lasterror(c);
		if (r!=0) imsg("set reuseaddress failed  %.200s \n",tcp_error(c,bf));
	}
	if (bindout==NULL) bindout = "";
	if ((c->lclport!=0) || (lib_strlen(bindout)>0) ) {
		dmsg("Binding outgoing connection to port %s:%d\n",bindout,c->lclport);
		
		xlcl.sin_family = AF_INET;
		xlcl.sin_addr.s_addr = INADDR_ANY;
		if (lib_strlen(bindout)>0)  xlcl.sin_addr.s_addr =  inet_addr(bindout);
		xlcl.sin_port = htons((short) c->lclport);
		didbind = TRUE;
		if (bind(sk,(struct sockaddr *)  &xlcl, sizeof (xlcl)) < 0) {
			tcp_lasterror(c);
			imsg("LCL Bind failed - %.200s - will bind to free port instead\n",tcp_error(c,bf));
			/*			tcp_socket_close(sk);
			return INVALID_SOCKET;*/
		}
	}
	if (all_bigbf) {
		tcp_setrcvbf_raw(sk,all_sz);
		tcp_setsndbf_raw(sk,all_sz);
	}
	
	xsin.sin_port = htons((unsigned short) dst_port);
	start = time(NULL);
	st = connect(sk,(struct sockaddr *)   &xsin, sizeof(xsin));
	tcp_lasterror(c);
	if (st!=0) {
		char tmp[BFSZ];
		if (tcp_wouldblock(c)) return sk;
		imsg("Connect failed %dsec (%s:%d) bind=(%s) %s\n",(int) (time(NULL)-start),name,dst_port,bindout,tcp_error(c,tmp));
		if (open_hist) open_hist((int)sk,FALSE,FALSE,__FILE__,__LINE__,"tcp_open_raws");  //NORMAL CLOSE 
		tcp_save_sk(sk,FALSE);
		if (tcp_socket_close(sk,__FILE__,__LINE__)!=0) {
			if (tcp_sk_wouldblock()) {
				emsg("tcp_socket_close: Socket close failed will retry - wouldblock\n");
				tcp_socket_close_hard(sk);
			} else {
				emsg("tcp_socket_close: Socket close failed will retry - notblock\n");
			}
		} else {
			hcount_close((int)sk);
		}
		sprintf(c->erradd,"%d sec's",(int) (time(NULL)-start));
		if (didbind && !done_again) {
			//retry without bind:
			bindout="";
			done_again = TRUE; // double  check to prevent loops
			goto again;
		}
		return INVALID_SOCKET;
	}
	return sk;
}


int tcp_udp_write(Chan *c, char *bf, int len)
{
	return sendto(c->sk,bf,len,0,(struct sockaddr *) &c->udp_sin,sizeof(struct sockaddr));	
}
int tcp_udp_read(Chan *c, char *bf, int len)
{
	struct sockaddr rsin;
	int rlen;
	rlen = sizeof(struct sockaddr);
	return recvfrom(c->sk,bf,len,0,(struct sockaddr *) &rsin,&rlen);	
}
#ifdef DO_IPV6
int tcp_open_udp_ipv6(Chan *c, char *xname,int dst_port)
{
    // By not setting the AI_PASSIVE flag in the hints to getaddrinfo, we're
    // indicating that we intend to use the resulting address(es) to connect
    struct addrinfo hints,*res=NULL,*a;
	int ret;
	int on=-1;
	char sport[BFSZ];
	SOCKET sk;
	char *s;
	char xname2[BFSZ];
	char lclname[BFSZ];
	ncpy(lclname,xname,BFSZ-1); xname = lclname; // so we don't modify the input by mistake...

	dst_port = tcp_trim_port(xname,dst_port);
	sprintf(sport,"%d",dst_port);

    memset(&hints, 0, sizeof (hints));
    hints.ai_family = AF_INET6;
    hints.ai_socktype = SOCK_DGRAM;

	if (tcp_isip(xname)) hints.ai_flags = AI_NUMERICHOST;
	if (*xname=='[') {		
		ncpy(xname2,xname+1,BFSZ-1);
		s = strchr(xname2,']'); 
		if (s!=NULL) *s = 0;
		xname = xname2;
	}
	imsg("tcp: ipv6 open (%s) port %d\n",xname,dst_port);

    ret = getaddrinfo(xname, sport, &hints, &res);
    if (ret != 0) {
		//imsg("tcp: gai_strerror: (%s) (%s)\n",xname,gai_strerror(ret));
	    hints.ai_family = AF_UNSPEC; // try ipv4 too...
	    ret = getaddrinfo(xname, sport, &hints, &res);
		if (ret!=0) {
			imsg("tcp: ipv6 open (%s) port %d FAILED %s\n",xname,dst_port,gai_strerror(ret));
			tcp_lasterror(c);
			return FALSE;
		}
		return FALSE;
    }
    for (a=res;a!=NULL;a=a->ai_next) {
        sk = socket(a->ai_family, a->ai_socktype, a->ai_protocol);
        if (sk == INVALID_SOCKET) continue;
		// Line added here so log will show different entry
		if (open_hist) open_hist((int)sk,TRUE,FALSE,__FILE__,__LINE__,"ipv6-udp"); // record the handle as open.   -- crashing  due to handle not set as closed? 
		if (connect(sk, a->ai_addr,(int) a->ai_addrlen) != SOCKET_ERROR) break;
		tcp_lasterror(c);
		if (open_hist) open_hist((int)sk,FALSE,FALSE,"open_ipv6",__LINE__,"openipv6"); // REAL CLOSE ipv6 failure
		tcp_socket_close_hard(sk); sk = INVALID_SOCKET;  // close_hard does NOT DO the open_hist call to close the socket 
    }
	if (res!=NULL) freeaddrinfo(res); res = NULL;


	if (sk==INVALID_SOCKET) return TRUE;
	c->sk = sk;
	c->never_close = FALSE;
	c->in_close = FALSE;
	tcp_lasterror(c);

	tcp_close_on_exec(c);
	if (c->noblock) {
		if (!tcp_setblockraw(sk,FALSE)) {
			tcp_lasterror(c);
		}
	}

	return (int)sk;
}


#endif

int tcp_open_udp(Chan *c, char *xname,int dst_port)
{
	struct sockaddr_in xsin;
	SOCKET sk;
	struct hostent *hp;
	int dontblock = FALSE;
	char name[1000];
	int on=-1;
	char *tmp=NULL;
	const char optval=44; //non zero integer
	struct hostent hresult;
	char hbuffer[BFSZ];
	int h_err=0;
	time_t start=time(NULL);

//	imsg("tcp_open_udp: open (%s) %d\n",xname,dst_port);
	if (xname==NULL) {
		imsg("tcp_open_udp: with null parameter\n");
		return FALSE; 
	}
	tcp_init_wsa();
#ifdef DO_IPV6
	if (do_ipv6) {
		int r;
		r = tcp_open_udp_ipv6(c,xname,dst_port);
		if (r) {
			return r; 
		}
	}
#endif
	strcpy(hbuffer,"");
	memset(&hresult,0,sizeof(hresult));
	strcpy(c->myname,"");
	strcpy(c->theirname,"");
	if (lib_strlen(xname)==0) return FALSE;
	strncpy(name,xname,200);
	memset(&xsin,0,sizeof(xsin));
	xsin.sin_family = AF_INET;
	
	strlwr(name); /* Seems to be required by NIS */
	dst_port = tcp_trim_port(name,dst_port);
	
	if (tcp_isip(name)) {
		xsin.sin_addr.s_addr = inet_addr(name);
		xsin.sin_family = AF_INET;
	} else {
		xsin.sin_addr.s_addr = 0;
#ifdef MUTEX_GETHOST
		tcp_mutex_wait2(__LINE__);
#endif
		ncpy(thingtolookup,name,BFSZ-1);
//		hp = mygethostbyname_r(name,&hresult,hbuffer,sizeof(hbuffer),&h_err);
		hp = mygethostname(&hresult,name,&h_err,&tmp);
		c->xerrno = h_err;
		tcp_lasterror(c);
#ifdef MUTEX_GETHOST
		tcp_mutex_release2();
		if (time(NULL)-start>4) imsg("gethostbyname2 (%s) took %d seconds, disable g_lookup_names\n",name,time(NULL)-start);
#endif		
		if (hp==NULL) return FALSE;
		if (hp->h_length<=0) goto failed3;
		if (hp->h_length>500) goto failed3;
		if (hp->h_addr == NULL) {
failed3:
			mygethostname_free(&tmp);
			return FALSE;
		}
		safe_memcpy(&(xsin.sin_addr),hp->h_addr,  4,&(xsin.sin_addr),4);
//		safe_memcpy(&(xsin.sin_addr),hp->h_addr,  hp->h_length,&(xsin.sin_addr),4);
		xsin.sin_family = hp->h_addrtype;
		mygethostname_free(&tmp);
	}
	
	sk = tcp_dupsocket(xsin.sin_family,SOCK_DGRAM, 0,"udp"); // option 4
	c->sk = sk;
	c->never_close = FALSE;
	c->in_close = FALSE;
	tcp_lasterror(c);
	if (sk==INVALID_SOCKET) {
		imsg("Can't dupsocket connect to {%s} \n",name);
		return FALSE;
	}
//	if (open_hist) open_hist(sk,TRUE,__FILE__,__LINE__);
	tcp_close_on_exec(c);
	if (c->noblock) {
		if (!tcp_setblockraw(sk,FALSE)) {
			tcp_lasterror(c);
		}
	}
	
	xsin.sin_port = htons((unsigned short) dst_port);

	memcpy(&c->udp_sin,&xsin,sizeof(xsin));
	return TRUE;
}

int tcp_sk_wouldblock(void)
{
	int err = errno;
#ifdef WIN32
	err = WSAGetLastError();
	if (err==WSAEWOULDBLOCK) return TRUE;
	if (err==WSAEINPROGRESS) return TRUE;
#else
	// removed because we suspect on unix the socket should not be  closed again.
//	if (err==EWOULDBLOCK) return TRUE;
//	if (err==EINPROGRESS) return TRUE;
//	if (err==EAGAIN) return TRUE;
#endif
	return FALSE;
}
int tcp_wouldblock(Chan *c)
{
		int err = c->xerrno;;
#ifdef HAS_SSL
		if (c->ssl!=NULL) return myssl_wouldblock(c->ssl);
#endif
#ifdef WIN32
		if (err==WSAEWOULDBLOCK) return TRUE;
		if (err==WSAEINPROGRESS) return TRUE;
		if (err==WSAENOTCONN) return TRUE;
#else
		if (err==EWOULDBLOCK) return TRUE;
		if (err==EINPROGRESS) return TRUE;
		if (err==ENOTCONN) return TRUE;
		if (err==EAGAIN) return TRUE;
#endif
		return FALSE;
}
int tcp_check_chan(Chan *c)
{
	if (c->notfree!=0) {
		emsg("Aborting because channel structure has been freed I think %d %d\n",c->sk,c->notfree);
		lib_abort();
	}
	return TRUE;
}
int tcp_open_reuse_nice(Chan *cc, char *hostname,  int port, char *bindout)
{
	int r=TRUE;
	int wait = cc->read_timeout;
	if (wait==0) wait = 3*60*1000;
    if ((cc->sk != 0) && (cc->sk != INVALID_SOCKET) ) emsg("tcp: tcp_open_reuse_nice called with sk member set to %d\n",cc->sk);
	tcp_check_chan(cc);
	cc->never_close = FALSE;
	cc->in_close = FALSE;
	cc->block = FALSE;
	cc->noblock = TRUE;
	cc->close_now = FALSE; // clear this flag
	cc->sk = tcp_open_raw(cc,hostname,port,TRUE,bindout,"open_reuse_nice");
	cc->noblock = FALSE;
	cc->block = TRUE;
	if (cc->sk==INVALID_SOCKET) return FALSE;
	// This does not open a non blocking socket, it opens a blocking socket, but the open
	// itself doesn't block .
	tcp_setblockraw(cc->sk,TRUE);
	r = tcp_wait_open(cc,wait);
	if (!r) tcp_close_nice(cc);
	return r;
}
int tcp_open_nice_x(Chan *cc, char *hostname,  int port, char *bindout, char *src, int line)
{
	int r;
	time_t start=time(NULL);
	time_t start2=start;
	if (cc == NULL) { imsg("tcp_open_nice called with NULL channel - crashing...\n"); lib_abort();}
	// This is must like tcp_open but it will abort if we shutdown surgemail.

	r = tcp_open_noblock_x(cc,hostname,port,bindout,src,line);
	start2 = time(NULL);
	if (r) {
		int wait = cc->read_timeout;
		if (wait==0) wait = 3*60*1000;
		r = tcp_wait_open(cc,wait);
		if (!r) tcp_close_nice(cc);
	}
	if (!r)	sprintf(cc->erradd,"%d sec, %d wsec",(int) (time(NULL)-start),(int) (time(NULL)-start2));
	return r;
}
int tcp_open_noblock_x(Chan *cc, char *hostname,  int port, char *bindout, char *src, int line)
{
    if ((cc->sk != 0) && (cc->sk != INVALID_SOCKET) ) emsg("tcp: tcp_open_noblock called with sk member set to %d %s:%d\n",cc->sk,src,line);
	tcp_check_chan(cc);
	cc->block = FALSE;
	cc->noblock = TRUE;
	cc->close_now = FALSE; // clear this flag
	cc->never_close = FALSE;
	cc->in_close = FALSE;
	cc->sk = tcp_open_raw(cc,hostname,port,FALSE,bindout,"open_noblock"); // this is where it comes from...
	cc->noblock = FALSE;
	cc->block = TRUE;
	if (cc->sk==INVALID_SOCKET) return FALSE;
	// This does not open a non blocking socket, it opens a blocking socket, but the open
	// itself doesn't block .
	tcp_setblockraw(cc->sk,TRUE);
	return TRUE;
}
int tcp_open_x(Chan *cc, char *hostname,  int port, char *bindout, char *src, int line)
{
	tcp_check_chan(cc);
    if ((cc->sk != 0) && (cc->sk != INVALID_SOCKET) ) emsg("tcp: tcp_opena called with sk member set to %d %s:%d\n",cc->sk,src,line);
	cc->in_close = FALSE;
	cc->never_close = FALSE;
	cc->close_now = FALSE; // clear this flag
	cc->sk = tcp_open_raw(cc,hostname,port,FALSE,bindout,src);
	if (cc->sk==INVALID_SOCKET) return FALSE;
	return TRUE;
}
int tcp_open_reuse(Chan *cc, char *hostname,  int port, char *bindout)
{
	tcp_check_chan(cc);
	cc->in_close = FALSE;
	cc->close_now = FALSE; // clear this flag
	cc->never_close = FALSE;
	cc->sk = tcp_open_raw(cc,hostname,port,TRUE,bindout,"open_reuse");
	if (cc->sk==INVALID_SOCKET) return FALSE;
	return TRUE;
}
int tcp_not_connected(Chan *c)
{
#ifdef WIN32
	if (c->xerrno==WSAENOTCONN) return TRUE;
#else
		if (c->xerrno==ENOTCONN) return TRUE;
#endif
	return FALSE;
}

int tcp_lasterror(Chan *c)
{
	if (c==NULL) {
		char *s=NULL;
		lib_abort();
	}
#ifdef WIN32
	c->xerrno = WSAGetLastError();
#else
	c->xerrno = errno;
#endif
	return c->xerrno;
}
int tcp_release(Chan *c)
{
	c->canclose = TRUE;
	return TRUE;
}
int tcp_close_mean(Chan *c)
{
	if (c->sk==INVALID_SOCKET) return TRUE;
#ifdef HAS_SSL
	if (c->ssl!=NULL) {
		return tcp_close_nice(c);
	}
#endif
	tcp_setblock(c,FALSE);
	tcp_save_sk(c->sk,FALSE);
	if (open_hist) open_hist((int)c->sk,FALSE,FALSE,__FILE__,__LINE__,"close_mean"); // NORMAL CLOSE 
	if (tcp_socket_close(c->sk,__FILE__,__LINE__)!=0) {
		if (tcp_sk_wouldblock()) {
			tcp_socket_close_hard(c->sk);
			emsg("tcp_socket_close failed - wouldblock\n");
		} else {
			emsg("tcp_socket_close: Socket close failed will retry - wouldnotblock\n");
		}
	}
	hcount_close((int) c->sk);
	tcp_check_chan(c);
	c->sk = INVALID_SOCKET;
	return TRUE;
}
int tcp_close_ssl(Chan *c)
{
	if (c->ssl!=NULL) {
		tcp_setblock(c,FALSE);
#ifdef HAS_SSL
		myssl_close(c,c->ssl);
#endif
		c->ssl = NULL;
		if (c->sk!=INVALID_SOCKET) {
			tcp_socket_low_free(c->sk); // This should be valid, needed as myssl closes the socket.
			c->sk = INVALID_SOCKET;
		}
		return TRUE;
	}
	return FALSE;
}
int tcp_close_nice(Chan *c)
{
	int is98 = FALSE;
	if (c==NULL) return TRUE; // obviously it's not worth cosing null! 
	if (c->in_close) return TRUE;
	c->in_close = TRUE;
	tcp_buffer_flush(c);
#ifdef HAS_SSL
	if (tcp_close_ssl(c)) return TRUE;

	if (c->never_close) {
		tcp_check_chan(c);
		c->sk = INVALID_SOCKET;
		return TRUE;
	}
#endif
	if (c->sk==INVALID_SOCKET) return TRUE;
#ifdef WIN32

#endif
#ifdef WIN32
	is98 = !tcp_lib_isnt();
	if (is98) { // seems to fail to close so trying simple close  - yep this fixed it, should probably do this all the time. 
		tcp_setblockraw(c->sk,TRUE);
		if (open_hist) open_hist((int)c->sk,FALSE,FALSE,__FILE__,__LINE__,"close_nicea"); // NORMAL CLOSE 
		tcp_save_sk(c->sk,FALSE);
		tcp_socket_close(c->sk,__FILE__,__LINE__);
		hcount_close((int) c->sk);
		tcp_check_chan(c);
		c->sk = INVALID_SOCKET;
		return TRUE;
	}
#endif
	// Chrisp moved socket_shutdown 27 april 2021, so it is linux and windows, instead of just windows.
	socket_shutdown(c->sk,1);
	tcp_setlinger(c,1,10);/* 60*100*3);  this old value made tcp_socket_close lock up for the timeout on rare occasions. */
						 /* UNITS ARE SECONDS, NOT 100THS OF SECONDS AND tcp_socket_close MAY BLOCK EVEN IF NON BLOCKING ON UNIX */
	tcp_setblockraw(c->sk,FALSE);
	if (open_hist) open_hist((int)c->sk,FALSE,FALSE,__FILE__,__LINE__,"close_nice"); // NORMAL CLOSE 
	tcp_save_sk(c->sk,FALSE);
	if (tcp_socket_close(c->sk,__FILE__,__LINE__)!=0) {
		int err = errno;
		if (tcp_sk_wouldblock()) {
	  		tcp_lasterror(c);
			emsg("tcp_socket_close: Socket close failed will retry-wouldblock %s\n",strerror(err));
			tcp_setblockraw(c->sk,TRUE);	
#ifdef WIN32
			if (tcp_socket_close_hard(c->sk)!=0) {
				emsg("tcp_socket_close: failed again\n");
				if (open_hist) open_hist((int)c->sk,FALSE,TRUE,__FILE__,__LINE__,"close_hard_failed"); // LOGGED ONLY, TIS ALREADY CLOSED
			} else {
				hcount_close((int) c->sk);
			}
#else
				if (open_hist) open_hist((int)c->sk,FALSE,TRUE,__FILE__,__LINE__,"tcp_socket_close_failed3"); // logged only, tis already closed. 
#endif
		} else {
			emsg("tcp_socket_close: Socket close failed wil try and force anyway %s\n",strerror(err));
//			if (tcp_socket_close_hard(c->sk)!=0) { // retrying the close is not valid unless it's windows, and sk_wouldblock was returned.
				if (open_hist) open_hist((int)c->sk,FALSE,TRUE,__FILE__,__LINE__,"tcp_socket_close_failed"); // logged only, tis already closed. 
//			}
			hcount_close((int) c->sk);
		}
	} else {
		hcount_close((int) c->sk);
	}
	tcp_check_chan(c);
	c->sk = INVALID_SOCKET;
	return TRUE;
}
int tcp_free(Chan **cc)
{
	Chan *c = *cc;
	if (c==NULL) return FALSE;
	if (c->sk!=INVALID_SOCKET) {
		tcp_close_nice(c);
		tcp_check_chan(c);
		c->sk = INVALID_SOCKET;
	}
	tcp_close_ssl(c); // make sure ssl structure is freed
	if (c->inbf!=NULL) tcp_bf_free(&c->inbf); 
	if (c->ssl_clientcert!=NULL) free(c->ssl_clientcert);  // CRASH COMING FROM HERE???
	*cc = NULL;
	free(c);
	return TRUE;
}

int tcp_clean(char *data, int len, char *tmp)
{
	char *out=tmp;
	unsigned char c;
	int i;
	for (i=0; i<len; i++) 
	{
		c = (unsigned char) *data++;
		if (out-tmp-10>BFSZ) break;
		if (c<32) {
			*out++ = '\\';
			if (c==0) *out++ = '_'; 
			else *out++ = c + 'a' - 1;
		} else {
			*out++ = c;
		}
	}
	*out ++ = 0;
	return TRUE;
}
char *tcp_emsg_time(char *tstr)
{
	time_t t;
	char tmpstr[BFSZ];
	char *ccc;
	int msec=0;

#	ifdef WIN32
	SYSTEMTIME systime;
	GetSystemTime(&systime);
	msec = systime.wMilliseconds;
#	endif

	time(&t);
	ccc = safe_ctime(&t,tmpstr);
	if (ccc==NULL) ccc = "badtimebadtimebadtime";
	strcpy(tstr,ccc+8);
	tstr[11] = 0;
	sprintf(tstr+11,".%.02d",msec/10);

	return tstr;
}

int tcp_log_data(char *mode, char *data, int len)
{
	int chunk;
	char tmp[BFSZ];
	static FILE *f;

	tcp_mutex_wait();
	if (f==NULL) f = fopen("tcp.log","ab"); // chrisp added b
	if (f!=NULL) {
		if (lib_file_osize(f)>5000000) {
			fclose(f); f = NULL;
			remove("tcp2.log");
			rename("tcp.log","tcp2.log");
			f = fopen("tcp.log","wb"); // chrisp added b
		}
	}
	// Chrisp moved mutex down below where it should be, hope this doesn't cause dead lock.
	if (len<=0) {
//		imsg("tcp_%s: %d bytes\n",mode,len);
//		fprintf(f,"tcp_%s: %d bytes\n",mode,len);
		fflush(f);
		tcp_mutex_release();
		return TRUE;
	}
	for (;len>0;) {
		char tmp3[BFSZ];
		chunk = 100;
		if (chunk>len) chunk=len;
		len -= chunk;
		tcp_clean(data,chunk,tmp);
		imsg("tcp_%s: [%d] %s\n",mode,chunk,tmp);
		fprintf(f,"%s tcp_%s: [%d] %s\n",tcp_emsg_time(tmp3),mode,chunk,tmp);
		data += chunk;
	}
	tcp_mutex_release();
	return TRUE;
}
int tcp_read_log(char *s, int got, int wanted)
{
	FILE *f;
	f = fopen("tcp.log","ab");
	if (f==NULL) return TRUE;
	fprintf(f,"\r\nGot %d bytes, wanted %d bytes\r\n",got,wanted);
	if (got>0) {
		if (got<100) fwrite(s,1,got,f);
	}
	fclose(f);
	return TRUE;
}
int tcp_read_complete(Chan *c, char *s, int len)
{
	// Keep reading until we get the entire block, or an error...
	int r=10;
	int got=0;
	for (;got<len;) {
		r = tcp_read_timed(c,s+got,len-got,10000);
		if (r<=0) {
			break;
		}
		got += r;
	}
	return got;
}
int tcp_read(Chan *c, char *s, int len)
{
	int r;
	errno = 0;
	if (c->sk==INVALID_SOCKET) {
		c->xerrno = -1;
		return 0;
	}
#ifdef HAS_SSL
	if (c->ssl!=NULL){
		r = myssl_read(c, c->ssl, s, len);
		if (r>0) c->total_read += r;
		if (r<=0) tcp_lasterror(c);
		if (c->log_read) {
			tcp_log_data("read",s,r);
		}
		return r;
	}
#endif
	r = socket_read(c->sk,s,len);
	if (r>0) c->total_read += r;


	if (r == 0) {
		tcp_set_closed(c);
		c->xerrno = -1; // So any error sais 'connection was closed' 
	}
	else if (r < 0) tcp_lasterror(c);
	else if (trace_socket) {
		FILE *f;
		f = fopen("socket.log","a");
		if (f!=NULL) {
			fprintf(f,"\nPacket %d bytes\n",r);
			fwrite(s,1,r,f);
			if (lib_file_osize(f) > 100000000) {
				fclose(f); f = NULL;
				remove("socket.log");
			}
			if (f!=NULL) fclose(f); f = NULL;
		}
	}
	if (c->log_read) tcp_log_data("read",s,r);
	return r;
}

int tcp_socket_write_acc(Chan *c, char *s, int len);
int tcp_socket_write(Chan *c, char *s, int len)
{
	int t;
	t = tcp_socket_write_acc(c,s,len);
	if (t>0) c->total_sent += t;

	if (c->log_write) {
		tcp_log_data("write",s,t);
	}
	return t;
}
int tcp_socket_write_raw(Chan *c, char *s, int len)
{
	return socket_write(c->sk,s,len);
}
int tcp_socket_write_acc(Chan *c, char *s, int len)
{
	if (c->out_fn!=NULL) return c->out_fn(c->out_p1,s,len);
	if (c->isbuffer) return tcp_buffer_add(c,s,len);
	if (c->ssl!=NULL) return tcp_write_ssl(c,s,len);

	return tcp_socket_write_raw(c,s,len);
}
int tcp_pop_response(Chan *c, char *mode, char *reason)
{
	char bf[BFSZ];
	char tmp[BFSZ];
	if (!tcp_readline(c,bf,BFSZ-1)) {
		sprintf(reason,"pop_response (%.100s) %.100s",mode,tcp_error(c,tmp));
		return FALSE;
	}
	if (*bf!='+') {
		sprintf(reason,"pop_response (%.100s) %.100s",mode,bf);
		return FALSE;
	}
	ncpy(reason,bf,BFSZ-1);
	return TRUE;
}

int tcp_write_smooth(Chan *c, char *s, int len)
{
	int sent;
	int olen=len;


	// Going to send in 'perfect' 64k chunks, see if that works better. 
	for (;(len + c->buffer_len)>127000;) {
		sent = tcp_buffer_flush2(c,s,len);
//		imsg("tcp_write_smooth flush2, len=%d, sent=%d, rest=%d\n",len,sent,len-sent);
		if (sent<0) return olen;
		len -= sent;
		s+= sent;
	}
//	imsg("tcp_write_smooth add, len=%d\n",len);
	if (len>0) tcp_buffer_add(c,s,len);
	return olen;
}
int tcp_write(Chan *c, char *s, int len)
{
	int r;
	if (c->sk==INVALID_SOCKET) return 0;
	if (len==0) return 0;
	if (c->isbuffer) return tcp_buffer_add(c,s,len);
	r = tcp_socket_write(c,s,len);
	if (r<=0) tcp_lasterror(c);
	return r;
}

int tcp_write_tout(Chan *c, char *s, int len)
{
	int tout = 1000*60*10;
	if (c==NULL) return -1;
	if (c->write_timeout>0) tout = c->write_timeout;
	return tcp_write_timed(c,s,len,tout);
}
int tcp_wait_write_nice(Chan *c, int msec);
int tcp_write_timed(Chan *c, char *s, int len, int msecs)
{
	int r;
	time_t start=time(NULL);
	time_t original=start;
	char tmp[BFSZ];
	int upto=0;
	c->xerrno = 0;
	if (c->sk==INVALID_SOCKET) {
		c->xerrno = -1;
		return 0;
	}

	if (c->close_now) return 0;
	if (c->isbuffer) {
		int ret;
		int was;
		was = c->write_timeout;
		c->write_timeout = msecs;
		ret = tcp_buffer_add(c,s,len);
		c->write_timeout = was;
		return ret;
	}

	tcp_setblockraw(c->sk,FALSE);
	for (;upto<len;) {
		errno = 0; // Just to be paranoid.
		r = tcp_socket_write(c,s+upto,len-upto);

		if (r<=0) tcp_lasterror(c);
		if (r>0) {
			start = time(NULL);
			upto+=r;
		}
		if (r<=0) {
			if (c->mean) break;
			if (!tcp_wouldblock(c)) {
				imsg("Closing as error on socket_write %d %s %p %d %d\n",c->xerrno,tcp_error(c,tmp),s,upto,len-upto);
				sprintf(c->erradd,"write_timed1 %d %s r=%d",c->xerrno,tcp_error(c,tmp),r);

				if (tcp_wait_read(c,1)) {
						char *s;
						char tmp4[BFSZ];
						strcpy(tmp4,"");
						tcp_readline(c,tmp4,BFSZ-1);
						s = strchr(tmp4,'\n'); if (s!=NULL) *s = 0;
						s = strchr(tmp4,'\r'); if (s!=NULL) *s = 0;
						sprintf(c->erradd,"write_timed3.1 %dsec (%.100s) %d %s r=%d ",(int) (time(NULL)-start),tmp4,(int) c->xerrno,tcp_error(c,tmp),r);
						imsg("tcp Write error, read back (%s)\n",tmp4);
				}

				tcp_close_nice(c);
				break;
			}
			tcp_lib_msleep(1); /* Stop race condition on solaris etc */
			if (c->spinwrite) {
				tcp_lib_msleep(10); /* testing on solaris avoiding the poll call */
			} else {
			  if (!tcp_wait_write_nice(c,msecs)) {
				tcp_lasterror(c);

				if (!tcp_wouldblock(c)) {
					imsg("Closing as error on tcp_wait_write %d\n",c->xerrno);
					sprintf(c->erradd,"write_timed2 %d %s r=%d",c->xerrno,tcp_error(c,tmp),r);
					if (tcp_wait_read(c,1)) {
						char *s;
						char tmp4[BFSZ];
						strcpy(tmp4,"");
						tcp_readline(c,tmp4,BFSZ-1);
						s = strchr(tmp4,'\n'); if (s!=NULL) *s = 0;
						s = strchr(tmp4,'\r'); if (s!=NULL) *s = 0;
						sprintf(c->erradd,"write_timed2.1 %dsec (%.100s) %d %s r=%d ",(int) (time(NULL)-start),tmp4,(int) c->xerrno,tcp_error(c,tmp),r);
					}
					

					tcp_close_nice(c);
				}
				break;
			  }
			}
			// next 4 lines were commented out 6 months ago for no reason I can figure. it was
			// done by jesse, possibly for performance reasons.  
			if (time(NULL) - start > (msecs/1000)) {
				imsg("Closing as timeout on write %d %d seconds\n",(int) (time(NULL)-start),msecs/1000);
				sprintf(c->erradd,"write_timed3 %d %s r=%d",c->xerrno,tcp_error(c,tmp),r);

				if (tcp_wait_read(c,1)) {
						char *s;
						char tmp4[BFSZ];
						strcpy(tmp4,"");
						tcp_readline(c,tmp4,BFSZ-1);
						s = strchr(tmp4,'\n'); if (s!=NULL) *s = 0;
						s = strchr(tmp4,'\r'); if (s!=NULL) *s = 0;
						sprintf(c->erradd,"write_timed3.1 %dsec (%.100s) %d %s r=%d ",(int) (time(NULL)-start),tmp4,(int) c->xerrno,tcp_error(c,tmp),r);
				}
			
				
				break;
			}
			if (is_exit) {
				imsg("tcp: Closing channel on write due to shutdown\n");
				sprintf(c->erradd,"write_timed4 %d %s r=%d",c->xerrno,tcp_error(c,tmp),r);
				break;
			}
			if (c->close_now) {

				imsg("tcp: Closing channel on write due to close_now setting\n");
				sprintf(c->erradd,"write_timed5 %d %s r=%d",c->xerrno,tcp_error(c,tmp),r);
				break;
			}
			if (!tcp_check_open(c)) {
				sprintf(c->erradd,"writeopencheck");
				tcp_close_nice(c);
				break;
			}	
		}
	}
	tcp_setblockraw(c->sk,TRUE);
	if (upto==0)
		// imsg("Unable to send any data upto=0 len=%d, waited %d (%d) seconds\n",len,(int) (time(NULL)-start),time(NULL)-original);
	if (upto<len) if (time(NULL)-start>10) imsg("tcp: Slow write %d (%d) seconds, %d/%d bytes sent\n",(int) (time(NULL)-start),time(NULL)-original,upto,len);	
	return upto;
}

int tcp_read_timed_raw(Chan *c, char *s, int len, int msecs)
{
	// Returns -2 if timeout occurs, returns 0 on closed socket, otherwise returns data
	int ndone = 0;
	int r = -2;
	time_t start;
	c->xerrno = 0;
	if (c->sk==INVALID_SOCKET) {
		sprintf(c->erradd,"socket invalid");
		return 0;
	}

#ifndef xis_freebsd /* causes lockups on freebsd due to freebsd being broken in some way */
	tcp_setblock(c,FALSE);
	r = tcp_read(c,s,len);
//if (c->log_read) imsg("tcp_read() got %d bytes\n",r);
	/*RMH: done in tcp_read
	tcp_lasterror(c);*/
	if (r<=0) {
		if (!tcp_wouldblock(c)) {
			sprintf(c->erradd,"CloseOnRead");
			if (c->log_read) imsg("tcp_read() !wouldblock so closing r=%d \n",r);
			tcp_close_nice(c);
			return 0;
		}
		if (!tcp_check_open(c)) {
				if (c->log_read) imsg("tcp_read() check_open failed \n",r);
				sprintf(c->erradd,"open check2");
				tcp_close_nice(c);
				return 0;
		}	
	}
	if (r>=0) goto done;
#endif
	start = time(NULL);
	if (tcp_waitfor(c,msecs)) {
		while(TRUE){
/*			if (!tcp_check_open(c)) {
				sprintf(c->erradd,"open check2");
				return 0;
			}
*/
			r = tcp_read(c,s,len);
			/*RMH:done in tcp_read
			tcp_lasterror(c);*/
			if (r<=0) {
				if (tcp_wouldblock(c)) {
					/* This shouldn't really happen, lets try again for half a second */
					tcp_lib_msleep(100);
					ndone++;
					if (ndone<5) continue;
					r = -2;
					sprintf(c->erradd,"ODD after %d seconds",(int) (time(NULL)-start));
				}
				else {tcp_close_nice(c);
				}
			}
			break;
		}
	} else {
		r = -2;
		sprintf(c->erradd,"after %d seconds",(int) (time(NULL)-start));
	}
done:	
	tcp_setblock(c,TRUE);
	if (r==0) {
		tcp_close_nice(c);
	}
	return r;
}

static int lf_fix;

char *tcp_find_dot(char *bf, char *end, int *isnulls, int isstart)
{
	register char *s=bf;
	register char c;
	char *eom = NULL;

	if (isstart) if (strncmp(bf,".\r\n",3)==0) {
		if (end-bf==3) {
			return bf;
		}

	}
	if (lf_fix) { 
		for (;s<end;s++) {
			c = *s;
			if (c==0) *isnulls = TRUE;
			if (c=='\n') {
				if (s[1]=='.') {
					if (s[2]=='\r') if (s[3]=='\n') {
						eom = s + 1;
						break;
					}
					if (s[2]=='\n') {
						eom = s + 1;
						break;
					}
				}
			}
		}
	} else {
		for (;s<end;s++) {
			c = *s;
			if (c==0) *isnulls = TRUE;
			if (c=='\r') {
				if (s[1] == '\n') if (s[2]=='.') {
					if (s[3]=='\r') if (s[4]=='\n') {
						eom = s + 2;
						break;
					}
				}
			}
		}
	}
	return eom;
}
int tcp_dot_lf(int fix)
{
	lf_fix = fix;
	return TRUE;
}
char *tcp_showbf(Chan *c, char *tmp)
{
	char *s;
	strcpy(tmp,"");
	for (s=c->inbf + c->inupto;s<c->inbf+c->inlen;s++) {
		if (sisalnum(*s)) {
			lpcat(tmp,BFSZ-1,"%c",*s);
		} else {
			lpcat(tmp,BFSZ-1,"/%02x",*s);
		}
	}
	return tmp;
}


char *tcp_read_dot(Chan *c, int *len, int *isend, int *isnulls, int secs, int isstart)
{
	char *start = c->inbf + c->inupto;
	char *end = c->inbf + c->inlen;
	int r=TRUE;
	char *eom;
	char tmp3[BFSZ];
	int doneread = FALSE;

	if (c->issaved) {
		start[0] = c->savedc;
		c->issaved = FALSE;
	}

	if (end-start<=2) {
do_read:
		r = tcp_readbf(c,secs*1000,2000);
		doneread = TRUE;
	}


	if (!r) {
		imsg("tcp_read_dot: Last bytes received (%s)\n",tcp_showbf(c,tmp3));

		lprintf(c->erradd,200-1,"LastBytes (%s)",tcp_showbf(c,tmp3));
		return NULL;
	}
	start = c->inbf + c->inupto;
	end = c->inbf + c->inlen;
	*end = 0; /* Null terminate the whole thing */


	eom = tcp_find_dot(start,end,isnulls,isstart);
	if (!doneread) if (eom==NULL) goto do_read;
	if (eom!=NULL) {
		*len = (int)(eom - start);
		c->inupto += *len;
		*isend = TRUE;
		/* Now swallow the .crlf */
		*eom = 0; /* Because some code in surgenews does rely on a terminated string*/
		c->inupto++;
		if (c->inbf[c->inupto] == '\r') c->inupto++;
		if (c->inbf[c->inupto] == '\n') c->inupto++;
		return start;
	}
	*len = (int)(end - start + 1 - 5);  /* Leave 4 incase start of 'eom' */
	if (*len<0) *len = 0;
	c->inupto += *len;
	c->savedc = start[*len];
	c->issaved = TRUE;
	start[*len] = 0;
	return start;
}

int tcp_read_timed_nice(Chan *c, char *s, int len, int msecs)
{	// this version obeys the shutdown flag faster...
	int i,r=0;
	if (msecs<=1000) return tcp_read_timed(c,s,len,msecs);
	for (i=0; i<msecs/1000; i++) {
		r = tcp_read_timed(c,s,len,1000);
		if (r>0) break;
		if (r!= -2) break;
		if (is_exit) break;
		if (c->close_now) break; 
	}
	return r;
}

int tcp_read_all(Chan *c, char *s, int len, int msecs) // Keep reading until we have the entire packet we want, or the timeout occurs.
{
	time_t start = time(NULL);
	int want=len;
	int upto=0;
	int r;
	for (;time(NULL)-start<msecs/1000;) { // given time limit
		if (want<=0) break;
		r = tcp_read_timed(c,s+upto,want,msecs);
		if (r<=0) return r;
		upto += r;
		want -= r;
	}
	return upto;
}
int tcp_read_timed(Chan *c, char *s, int len, int msecs)
{
	int x;
	int doneagain=FALSE;
	/*wasblock breaks things cos c->block is not actually always telling the truth. */
	/*int wasblock;*/
	char tmp2[BFSZ];
	if (len==BFSZ) { // Note: it must be passed a string longer than len!  
		//emsg("TCP_READ_TIMED: WAS CALLED WITH len==BFSZ!!! %s:%d (probably not serious)\n",file,line);
		//lib_abort();
	}
	/* If there is stuff in buffer and s != the internal buffer get data from internal buffer */
	if ( (c->inupto < c->inlen) && ((s < c->inbf) || (s > (c->inbf + c->maxinbf))) ) {

		for ( x = 0; c->inupto < c->inlen && (len--); x++ ) {
			s[x] = c->inbf[c->inupto++];
		}

		if (x>0) return x;
	}
	if (c->sk==INVALID_SOCKET) {if (dmsg!=NULL) dmsg("tcp_read_timed invalid socket");return -1;}
#ifndef xis_freebsd /* causes lockups on freebsd due to freebsd being broken in some way */
	tcp_setblock(c,FALSE);
again:	

	x = tcp_read(c,s,len); /*  this op can fail with WSAEWOULDBLOCK  */
	if ( x > 0 ) {
		/*if(wasblock)*/
		tcp_setblock(c,TRUE);
		return x;
	}
	if (x==0) {
		/* Normal close occurred */
		tcp_close_nice(c);
		return 0;
	}
	/*RMH: done in tcp_read
	tcp_lasterror(c);*/
	if (!tcp_wouldblock(c)) {
		if (x!=0) imsg("tcp_read_timed: Not a blocking error so something really wrong %s\n",tcp_error(c,tmp2));
		tcp_close_nice(c);
		return 0;
	}
	/* If data is arriving flat out, avoid a select call, just do a tiny sleep and re-read */
	if (!doneagain) if (x<0) {
		doneagain = TRUE;
		tcp_lib_msleep(1);
		goto again;
	}
	if (!tcp_check_open(c)){
		imsg("tcp_read_timed, socket is not open so returning zero\n");
		tcp_close_nice(c);
		return 0;
	}
#endif
	tcp_setblock(c,TRUE); // why is this here???? 

	while (TRUE) {
		tcp_setblock(c,TRUE); // chrisp ensure blocking is off for the read
		x = tcp_waitfor(c, msecs);
		if (x == TRUE) {
			tcp_setblock(c,FALSE);   // chrisp ensure blocking is disabled during the read, so this function cannot block.  
			x = tcp_read(c,s,len);
			if ( x > 0 ) {
				tcp_setblock(c,TRUE); // chrispensuring blocking is ehanbed when we return.
				return x;
			}
			if (x==0) {
				/*RMH: done in tcp_read tcp_lasterror(c);*/
				imsg("tcp_read_timed: socket closed normally by remote end\n");
				tcp_close_nice(c);
				return 0;
			}
			if (!tcp_wouldblock(c)) {
				if (x!=0) imsg("tcp_read_timed2: Not a blocking error so something really wrong %s\n",tcp_error(c,tmp2));
				tcp_close_nice(c);
				return 0;
			}
			continue;
		}
		// tcp_waitfor returned 'false' goes to -2... 
		break;
	}
	// what does -2 mean... (it means timeout) 
	tcp_setblock(c,TRUE); // chrispensuring blocking is ehanbed when we return.
	return -2;
}

int tcp_peek_data_timed(Chan *c, int msecs)
/*
Returns TRUE if data waiting to be read on this channel. Waits for
up to msecs for new data.
*/
{
	if (c->inupto<c->inlen) return TRUE;
	//if (msecs==0) return FALSE;
	tcp_readbf(c, msecs,10); /* Read up to 10 bytes */
	if (c->inupto<c->inlen) return TRUE;
	return FALSE;
}

int tcp_peek_line_timed(Chan *c, char *s, int len, int msecs)
{
	int x, y;
	if ( (c->inupto < c->inlen) && ((s < c->inbf) || (s > (c->inbf + c->maxinbf))) ) {
		for ( x = 0, y = c->inupto; y < c->inlen && --len && c->inbf[y] != '\n'; x++ ) {
			s[x] = c->inbf[y++];
		}
		s[x] = 0;
		if (x>0)
			return x;
	}
	tcp_readbf(c, msecs,len+1000); /* this is ok but odd, */
	/* If there is stuff in buffer and s != the internal buffer get data from internal buffer */
	if ( (c->inupto < c->inlen) && ((s < c->inbf) || (s > (c->inbf + c->maxinbf))) ) {
		for ( x = 0, y = c->inupto; y < c->inlen && --len && c->inbf[y] != '\n'; x++ ) {
			s[x] = c->inbf[y++];
		}
		s[x] = 0;
		return x;
	}
	if (c->sk==INVALID_SOCKET) return FALSE;
	return -2;
}





/*
	char inbf[CHAN_INSIZE];
	int inupto;
	int inlen;
*/
int tcp_clear_buffer(Chan *c)
{
	c->inlen = 0;
	c->inupto = 0;
	return TRUE;
}
int tcp_readbf(Chan *c, int msec, int max)
{
	int want;
	int loop=0;
	int r;
	if (c->inupto>0) {
		safe_memmove(c->inbf,c->inbf + c->inupto,c->inlen - c->inupto+1,c->inbf,c->maxinbf);
		c->inlen -= c->inupto;
		c->inupto = 0;
	}
	want = c->maxinbf-c->inlen-2;
	if (want<=0) if (c->maxinbf <= max) {
		tcp_grow_inbf(c,c->maxinbf*2);
		want = c->maxinbf-c->inlen-2;
	}
	if (want<=0) {
		/* Now lets scan for nulls and if we find some turn into spaces and log error and continue */
		int fixed=FALSE;
		/* chrisp removed as this seems evil.... but probably creates infinite input loop of some sort...
		for (i=0; i<c->inlen; i++) {
			if (c->inbf[i] == 0) {c->inbf[i] = ' '; fixed = TRUE;}
		}
		*/
		if (fixed) {
			imsg("Found nulls in input buffer and replaced them with spaces as input buffer was about to over flow, this should be valid as only used with readline which doesn't support nulls anyway\n");
			return TRUE;
		}

		imsg("No end of line in input buffer inupto=%d inlen=%d maxinbf=%d max=%d\n",c->inupto,c->inlen,c->maxinbf,max);
		c->inupto = 0;
		c->inlen = 0;
		want = c->maxinbf -10 ;
	}

	if (msec>1000) {
		time_t tick,start = time(NULL);
		int i;

		for (i=0;i<msec/1000;i++) {
			time(&tick);
			r = tcp_read_timed_raw(c,c->inbf+c->inlen,want,1000);
			if (r>0) break;
			/*sprintf(c->erradd,"after %d seconds",time(NULL)-start);*/
			if (r>=0) break;
			if ((r<0) && tcp_wouldblock(c) && (time(NULL)-tick==0)) {
				lib_msleep(100); /* Tiny delay to stop cpu loops on stupid resource errors*/
			}
			if (r != -2 && !tcp_wouldblock(c)) break;
			if (is_exit || c->close_now) {
				imsg("Fake channel close for tcp_set_exit\n");
				r = 0; break;
			} 
		}
		if (r<=0) {
			sprintf(c->erradd,"after %d seconds",(int) (time(NULL)-start));
		}
	} else {
		r = tcp_read_timed_raw(c,c->inbf+c->inlen,want,msec);
	}

	if (r<=0) {
		return FALSE;
	}
	if (c->fixnulls) {
		int i;
		char *s = c->inbf+c->inlen;
		for (i=0; i<r; i++) {
			if (s[i]==0) s[i] = 'N';
		}
	}
	c->inlen += r;
	c->inbf[c->inlen] = 0;

	return TRUE;
}
int tcp_set_read_timeout(Chan *c, int msec)
{
	c->read_timeout = msec;
	return TRUE;
}

int tcp_set_write_timeout(Chan *c, int msec)
{
	c->write_timeout = msec;
	return TRUE;
}


int tcp_grow_inbf_safe(Chan *c, int len)
{
	if (c->maxinbf<len) tcp_grow_inbf(c,2000+len);
	return TRUE;
}
int tcp_grow_inbf(Chan *c, int len)
{
	char *newbf;
	newbf = tcp_bf_get(len);
	if (len<c->maxinbf) lib_abort();  // doh! 
	if (c->inbf!=NULL) memcpy(newbf,c->inbf,c->maxinbf);
	tcp_bf_free(&c->inbf); c->inbf = newbf;
	c->maxinbf = len;
	return TRUE;
}
void tcp_set_exit(void)
{
	is_exit = TRUE;
}
int tcp_inbf_show(char *whence, Chan *c)
{
	
	return TRUE;
}
int tcp_insert_buffer(Chan *c, char *bf, int len)
{
	int want = c->inlen + len + 1;
	char *s;
	int oldlen = c->inlen - c->inupto;
	if (len<=0) return TRUE;
	tcp_grow_inbf_safe(c,want);
	s = c->inbf + c->inupto;
	safe_memmove(c->inbf+len,s,oldlen,c->inbf,c->maxinbf);
	safe_memcpy(c->inbf,bf,len,c->inbf,c->maxinbf);
	c->inupto = 0;
	c->inlen = oldlen + len;
	return TRUE;
}
int tcp_readline(Chan *c, char *out, int len)
{
	char *s2;
	char *s;
	time_t start=0;
	int blen=len;
	int tout = c->read_timeout;
	if (blen<2000) blen = 2000;
//	if (tout==0) tout = 3 * 60 * 1000; 
	if (tout==0) tout = 9 * 60 * 1000; /* As bufferring occurs on writes up to 1mb at least
	then when downloading a 1mb file at 5k per second the next 'read' from the client cannot
	occur in less than 3.3 minutes, so surgemail closes the link, this upsets outlook which
	then downlaods the message again - forever */
	/* n = bytes read, 0 = socket closed, -1 = socket closed, -2 = timeout */
again:
	if (c->maxinbf<=blen) tcp_grow_inbf(c,blen);
	s = c->inbf + c->inupto;
	c->inbf[c->inlen] = 0; /* Make sure it's null terminated */
	s2 = strchr(s,'\n');
	
	if (s2==NULL) {
		if (!tcp_readbf(c,tout,len+100)) return FALSE;
		if (start==0) start = time(NULL);
		if ((time(NULL)-start) > (tout/1000)) {
			sprintf(c->erradd,"after %d seconds",(int) (time(NULL)-start));
			return FALSE;
		}
		goto again;
	}
	*s2 = 0;
	if (s2>s) if (s2[-1]=='\r') s2[-1] = 0;

	ncpy(out,s,len-1);
	c->inupto += (int)(s2-s+1);

	return TRUE;
}
int tcp_readline_free(char *out)
{
	if (out!=NULL) free(out);
	return TRUE;
}
int tcp_readline_big(Chan *c, char **vout, int max,int tout)
{
	char *s2;
	char *s;
	tout = tout * 1000; // sec to msec conversion
	if (tout==0) tout = c->read_timeout;
	if (tout==0) tout = 3 * 60 * 1000;
	*vout = NULL;
	//tout = 60*1000; // adjust for testing  REMOVE THIS LINE


	/* n = bytes read, 0 = socket closed, -1 = socket closed, -2 = timeout */
again:
	s = c->inbf + c->inupto;
	c->inbf[c->inlen] = 0; /* Make sure it's null terminated */
	s2 = strchr(s,'\n');
	if (s2==NULL) {
		if (!tcp_readbf(c,tout,max)) return FALSE;
		goto again;
	}
	*s2 = 0;
	if (s2>s) if (s2[-1]=='\r') s2[-1] = 0;

	*vout = strdup(s);
	c->inupto += (int)(s2-s+1);
	return TRUE;
}
int tcp_noblock(Chan *c)
{
	c->noblock = TRUE;
	c->block = FALSE;
	return TRUE;
}
/*
This is really a 'wait until open' function, it should be modified for wait_write
*/
static int slow_getsock=FALSE;
int tcp_set_getsock(int t)
{
	slow_getsock = t;
	return TRUE;
}
unsigned int tcp_clock(void)
{
#ifdef WIN32
	unsigned int lo,hi,tot,r;
	FILETIME crtime,extime,ktime,utime;

	GetThreadTimes(GetCurrentThread(),&crtime,&extime,&ktime,&utime);
	lo = ktime.dwLowDateTime >> 16;
	hi = ktime.dwHighDateTime << 16;
	tot = lo + hi;
	lo = utime.dwLowDateTime >> 16;
	hi = utime.dwHighDateTime << 16;
	tot += lo + hi;

	r = (int) ((double) tot * .0065536 * CLOCKS_PER_SEC);
	r = r & 0x07fffffff;
	return (int) r;
#else
//	return clock(); NEVER USE CLOCK
	return 0;
#endif
}
int tcp_wait_open_real(Chan *c, int msec);

int tcp_wait_open(Chan *c, int msec)
{
	int r;
	int cpu = tcp_clock();
	r = tcp_wait_open_real(c,msec);
	//imsg("tcp_wait_open %d cpu msec\n",1000*(tcp_clock()-cpu) / CLOCKS_PER_SEC);
	return r;
}
int tcp_wait_open_real(Chan *c, int msec)
{
	int i;
	int xisunix=FALSE;
	static int slow_getsock;
#ifndef WINDOWS
	xisunix = TRUE;
#endif
	if (c==NULL) {
		imsg("TCP_WAIT_OPEN_REAL CALLED WITH NULL CHANNEL\n");
		emsg("TCP_WAIT_OPEN_REAL CALLED WITH NULL CHANNEL\n");
		lib_abort();
	}

	c->done_open = FALSE;
	if (msec<1000) msec = 1000;
	imsg("tcp: wait open %d msec\n",msec);
	for (i=0; i<msec/1000; i++) {
		if (is_exit) break;
		if (c->close_now) break;
		if (tcp_wait_write(c,1000)) {
			imsg("tcp: wait open %d msec - yay we can write\n",msec);
			return TRUE;
		} else {
			if (c->done_open) {
				if (tcp_wouldblock(c) && i<10 && xisunix) {
					lib_msleep(1000); // Lets just wait a bit
				} else {
					imsg("tcp: wait open %d msec - failed\n",msec);
					return FALSE;	
				}
			}
			lib_msleep(100);
		}
	}
	imsg("tcp: wait open %d msec - failed took too long\n",msec);
	return FALSE; // Faile to open in time.
}
int tcp_wait_write_nice(Chan *c, int msec)
{
	time_t start=time(NULL);
	int r = 0;
	errno = 0;
	if (msec<1000) return tcp_wait_write(c,msec);
	for (;time(NULL)-start < msec/1000;) {
		r = tcp_wait_write(c,1000);
		if (!tcp_isopen(c)) break;
		if (r) break;
		if (c->close_now) break;
		if (is_exit) break;
	}
	return r;
}
int tcp_wait_write(Chan *c, int msec)
{
	int failed;
	int r = tcp_wait_write_raw(c->sk,msec,&failed);

	if (failed) c->close_now = TRUE;
	if (r) c->done_open = TRUE;
	return r;
}

int tcp_wait_write_raw(SOCKET sk, int msec, int *failed)
{
#ifdef USE_POLL
	int nc;
	struct pollfd ufds[10];
	*failed=FALSE;
	memset(&ufds[0],0,sizeof(struct pollfd));
	if (sk==INVALID_SOCKET) return FALSE; /* giveup immediately */
	ufds[0].fd = sk;
	ufds[0].events = POLLOUT;
	ufds[0].revents = 0;
	errno = 0;
	nc = poll(ufds,1,msec);
#else
	fd_set readfd,writefd,exceptfd;
	int nc;
	struct timeval timeout;
	*failed=FALSE;

	if (sk==INVALID_SOCKET) return FALSE; /* giveup immediately */
	stack_ok("tcplib");
	timeout.tv_sec = msec/1000; /*Wake up once a second*/
	timeout.tv_usec = (msec%1000) * 1000 ;
	FD_ZERO(&readfd);
	FD_ZERO(&writefd);
	FD_ZERO(&exceptfd);
	FD_SET(sk,&writefd);
	FD_SET(sk,&exceptfd);
//FD_SET(sk,&readfd);
	errno = 0;

	nc = select(FD_SETSIZE,&readfd,&writefd,&exceptfd,&timeout);

#endif
	if (nc>0) {
		int was = 0;
		int xlen=sizeof(was);

#ifndef USE_POLL
		if (FD_ISSET(sk,&exceptfd)) {
			getsockopt(sk, SOL_SOCKET, SO_ERROR, (char *)&was, &xlen);
			
			if (was!=0) {
				
				*failed = TRUE;
				return FALSE;
			}
		}
#endif
			

		return TRUE;
	}
			
	return FALSE;
}

int tcp_wait_read(Chan *c, int msec)
{
#ifdef USE_POLL
	int nc;
	struct pollfd ufds[10];
	if (c->sk==INVALID_SOCKET) return FALSE; /* giveup immediately */
	ufds[0].fd = c->sk;
	ufds[0].events = POLLIN;
	ufds[0].revents = 0;
	errno = 0;
	nc = poll(ufds,1,msec);
#else
	fd_set readfd,writefd,exceptfd;
	int nc;
	struct timeval timeout;

	stack_ok("tcplib");
	if (c->sk==INVALID_SOCKET) return FALSE; /* giveup immediately */
	timeout.tv_sec = msec/1000; /*Wake up once a second*/
	timeout.tv_usec = (msec%1000) * 1000 ;
	FD_ZERO(&readfd);
	FD_ZERO(&writefd);
	FD_ZERO(&exceptfd);
	FD_SET(c->sk,&readfd);
	FD_SET(c->sk,&exceptfd);
	/*FD_SET(c->sk,&readfd);*/
	errno = 0;
	nc = select(FD_SETSIZE,&readfd,&writefd,&exceptfd,&timeout);
#endif
	if (nc>0) {
		int was = 0;
		int xlen=sizeof(was);

		c->done_open = TRUE;
#ifndef USE_POLL
		if (FD_ISSET(c->sk,&exceptfd)) {
			getsockopt(c->sk, SOL_SOCKET, SO_ERROR, (char *)&was, &xlen);
			if (was!=0) {
				c->xerrno = was;
				return FALSE;
			}
		}
#endif
		return TRUE;
	}
	c->done_open = FALSE;
	return FALSE;

}
int tcp_check_open(Chan *c)
{
	return tcp_check_open_real(c,FALSE);
}
int tcp_check_open_real(Chan *c, int noclose)
{
	fd_set readfd,writefd,exceptfd;
	int nc;
	struct timeval timeout;
	char tmp[BFSZ];

	stack_ok("tcplib");
	if (c->sk==INVALID_SOCKET) return FALSE;
#ifdef USE_POLL
	{
	struct pollfd ufds[10];
	if (c->sk==INVALID_SOCKET) return FALSE; /* giveup immediately */
	ufds[0].fd = c->sk;
	ufds[0].events = POLLIN | POLLPRI;
	ufds[0].revents = 0;
	errno = 0;
	errno = 0;
	nc = poll(ufds,1,0);
	tcp_lasterror(c);
	if (nc<0) {
		imsg("tcp_check_open got POLL failure  %d (%s)\n",errno,tcp_error(c,tmp));
		return FALSE;
	}
	if (nc>0) { /* Should get an except error if socket closes I think */
		/* !!!!!!!!!!!except error comes in during normal data transmission!!!!!!!!!!!!!!!!!!!!!!! */
		if (ufds[0].revents & POLLHUP) {
		  emsg("tcp_check_open got pullhup event on socket (%s)\n",tcp_error(c,tmp));
		  if (!noclose) tcp_close_nice(c);
		  return FALSE;
		}
	}
	return TRUE;
	}
#else
	{
	timeout.tv_sec = 0;
	timeout.tv_usec = 10;
	FD_ZERO(&readfd);
	FD_ZERO(&writefd);
	FD_ZERO(&exceptfd);
	FD_SET(c->sk,&readfd);
	FD_SET(c->sk,&exceptfd);
	nc = select(FD_SETSIZE,&readfd,&writefd,&exceptfd,&timeout);
	if (nc>0) { /* Should get an except error if socket closes I think */
		/* !!!!!!!!!!!except error comes in during normal data transmission!!!!!!!!!!!!!!!!!!!!!!! */
		if (FD_ISSET(c->sk, &readfd)) return TRUE;
		tcp_lasterror(c);
		if (tcp_wouldblock(c)) return TRUE;
		emsg("tcp_check_open got except event on socket (%s)\n",tcp_error(c,tmp));
		if (!noclose) tcp_close_nice(c);
		return FALSE;
	}
	return TRUE;
	}
#endif
}
int tcp_check_open_socket(SOCKET sk) // WARNING, NOT SAFE, AS IT DOESN'T USE POLL, BUT NOT CALLED BY ANYTHING IN PUSHIMAP...
{
	fd_set readfd,writefd,exceptfd;
	int nc;
	struct timeval timeout;

	stack_ok("tcplib");
	if (sk==INVALID_SOCKET) return FALSE;
		timeout.tv_sec = 0;
		timeout.tv_usec = 1;
		FD_ZERO(&readfd);
		FD_ZERO(&writefd);
		FD_ZERO(&exceptfd);
	FD_SET(sk,&exceptfd);
	nc = select(FD_SETSIZE,&readfd,&writefd,&exceptfd,&timeout);
	if (nc>0) { /* Should get an except error if socket closes I think */
		return FALSE;
	}
	return TRUE;
}
int tcp_waitforbuf(Chan *c, int msec){

	int r;
	if (c->sk==INVALID_SOCKET) return TRUE;

	// First if not at start of buffer, shift it!!!
	if (c->inupto>0) {
		safe_memmove(c->inbf,c->inbf + c->inupto,c->inlen - c->inupto+1,c->inbf,c->maxinbf);
		c->inlen -= c->inupto;
		c->inupto = 0;
	}

	if (c->inlen) {
		return TRUE;
	}

	r = tcp_waitfor(c,msec);
	return r;

}
int tcp_wait_read_buf(Chan *c, int msec)
{
	char *s;
	char *s2;
	if (c->sk==INVALID_SOCKET) return TRUE;

	if (c->inbf!=NULL) {
		s = c->inbf + c->inupto;
		c->inbf[c->inlen] = 0; /* Make sure it's null terminated */
		s2 = strchr(s,'\n');
		if (s2!=NULL) return TRUE;
	}

	return tcp_wait_read(c,msec);

}

int tcp_waitfor(Chan *c, int msec)
{
	return tcp_waitfor2(c,msec,FALSE);
}
int tcp_waitfor2(Chan *c, int msec, int israw)
{
	fd_set readfd,writefd,exceptfd;
	int nc;
	int nloop=0;
	struct timeval timeout;
	stack_ok("tcplib");

	if (c->sk==INVALID_SOCKET) return TRUE;
#ifdef HAS_SSL
	/*Data may be in buffer already */
	if (!israw) if (c->ssl!=NULL)
		if (myssl_pending(c->ssl)==1) return TRUE;
#endif
#ifdef USE_POLL
{
	int nc;
	struct pollfd ufds[10];
	if (c->sk==INVALID_SOCKET) return FALSE; /* giveup immediately */
again:
	ufds[0].fd = c->sk;
	ufds[0].events = POLLIN | POLLPRI;
	ufds[0].revents = 0;
	errno = 0;
	nc = poll(ufds,1,msec);

	if (nc>0) {
		if (ufds[0].revents & POLLIN) return TRUE;
		if (ufds[0].revents & POLLHUP) return TRUE;
		if (ufds[0].revents & POLLERR) return TRUE;
		if (ufds[0].revents & POLLNVAL) return TRUE;
		if (ufds[0].revents & POLLPRI) {
			int optval=0;
			int len=sizeof(optval);
			getsockopt(c->sk,SOL_SOCKET,SO_ERROR,(char *) &optval,&len);
			if (nloop<4) if (optval==0) {
				emsg("tcp: got except event with error value zero, so trying select again sk=%d\n",c->sk);
				goto again;
			}
		}
		return TRUE;
	}
	return FALSE;
}
#else 
again:
	nloop++;
	timeout.tv_sec = msec/1000; /*Wake up once a second*/
	timeout.tv_usec = (msec%1000) * 1000 ;
	FD_ZERO(&readfd);
	FD_ZERO(&writefd);
	FD_ZERO(&exceptfd);
	FD_SET(c->sk,&readfd);
	/* Experiment, next line might help with IE problem */
	/* Anothe experiment, I think maybe we need exceptfd to detect socket closing */
	FD_SET(c->sk,&exceptfd);
	nc = select(FD_SETSIZE,&readfd,&writefd,&exceptfd,&timeout);
	if (nc>0) {
		if (FD_ISSET(c->sk,&readfd)) return TRUE;
		if (FD_ISSET(c->sk,&writefd)) return TRUE;
		if (FD_ISSET(c->sk,&exceptfd)) {
			int optval=0;
			int len=sizeof(optval);
			getsockopt(c->sk,SOL_SOCKET,SO_ERROR,(char *) &optval,&len);
			if (nloop<4 && optval==0) {
				emsg("tcp: got except event with error value zero, so trying select again sk=%d\n",c->sk);
				goto again;
			}
		}
		return TRUE;
	} else {
		char tmp[BFSZ];
		if (nc<0) {
			tcp_lasterror(c);
			imsg("tcp_waitfor error so closing socket %d %s\n",nc,tcp_error(c,tmp));
			tcp_close_nice(c);
			// I think I should close the socket here.  
		}
	}
	return FALSE;
#endif
}
//very naughty of me
//int dfopen_die(char *reason, char *src, int line, int n);
#ifdef USE_POLL
int tcp_waitfor_poll(Chan *first, int msec)
{
	Chan *ch;
	struct pollfd ufds[200];
	char tmp2[BFSZ];
	int nf=0;
	int nc;
	for (ch = first; ch!=NULL; ch = ch->next) {
		ufds[nf].fd = ch->sk;
		ufds[nf].events = POLLIN | POLLPRI ;
		ufds[nf].revents = 0;
		nf++;
	}
	nc = poll(ufds,nf,msec);

	if (nc<0) {
		tcp_lasterror(first);
		emsg("poll error %s\n",tcp_error(first,tmp2));
#ifndef WIN32
		if (errno==EBADF) {
			emsg("Something fairly serious wrong, aborting as listen socket is dead Signal 99 ERROR\n");
#ifdef USE_DFOPEN
			dfopen_die("select_fault" ,(char *) __FILE__,__LINE__,0);
#endif
			lib_abort();
		}
#endif
	} 
	if (nc>0) return TRUE;
	return FALSE;
}
#endif
int tcp_waitfor_list(Chan *first, int msec)
{
	Chan *ch;
	fd_set readfd,writefd,exceptfd;
	int nc;
	static int nfail;
	struct timeval timeout;
	char tmp2[BFSZ];
	
	stack_ok("tcplib");
	if (first==NULL) { // was not waiting, and returning true, bad bad bad....
		lib_msleep(msec);
		return FALSE;
//		return TRUE;
	}
	for (ch = first; ch!=NULL; ch = ch->next) {
		if (ch->sk==INVALID_SOCKET) return TRUE;
	}
#ifdef HAS_SSL
	/*Data may be in buffer already */
	for (ch = first; ch!=NULL; ch = ch->next) {
		if (ch->ssl!=NULL) if (myssl_pending(ch->ssl)==1) return TRUE;
	}
#endif
#ifdef USE_POLL
	return tcp_waitfor_poll(first,msec);
#endif
	timeout.tv_sec = msec/1000; /*Wake up once a second*/
	timeout.tv_usec = (msec%1000) * 1000 ;
	FD_ZERO(&readfd);
	FD_ZERO(&writefd);
	FD_ZERO(&exceptfd);
	for (ch = first; ch!=NULL; ch = ch->next) {
		FD_SET(ch->sk,&readfd);
		FD_SET(ch->sk,&exceptfd);
	}
	errno = 0;
	nc = select(FD_SETSIZE,&readfd,&writefd,&exceptfd,&timeout);
	if (nc<0) {
		tcp_lasterror(first);
		emsg("select error %s\n",tcp_error(first,tmp2));
#ifndef WIN32
		if (errno==EBADF) {
			emsg("Something fairly serious wrong, aborting as listen socket is dead Signal 99 ERROR\n");
#ifdef USE_DFOPEN
			dfopen_die("select_fault" ,(char *) __FILE__,__LINE__,0);
#endif
			lib_abort();
		}
#endif
	} else nfail = 0;
	if (nc>0) return TRUE;
	return FALSE;
}
#include "vsnprintf.h"
int tcp_printf(Chan *c, char * arg_list, ...)
{
	va_list arg_ptr;
	char *format;
	int len, n;
	char output[5000];
	int r;
	char tmp2[BFSZ];
	int tout = 1000*60*10;

	if (c==NULL) return TRUE;
	if (c->sk==INVALID_SOCKET) return FALSE;
	if (c->write_timeout>0) tout = c->write_timeout;

	va_start(arg_ptr, arg_list);
	format = arg_list;
	output[0] = 0;
	len = vsnprintf(output,4000, format, arg_ptr); output[4000-1] = 0;
	n = lib_strlen(output);
	if (n==0) return TRUE; /* We dont want to send an empty string */
	c->sent += n;
	r = tcp_write_timed(c,output,n,tout);
	if (r<(int) lib_strlen(output)) {
  		/*tcp_lasterror(c);*/
		imsg("tcp_printf: failed only %d sent %s\n",r,tcp_error(c,tmp2));
		/* Next line added by chrisp so madmin doesn't timeout */
		tcp_close_nice(c);
		return FALSE;
	}
	if (isdebug) {
		emsg("tcp_printf --> %.90s",output);
	}
	if (r<=0) return FALSE;
	return TRUE;
}
int req_printf(char *req, int maxreq, char * arg_list, ...)
{
	va_list arg_ptr;
	char *format;
	int len, n;
	char output[5000];
	int tout = 1000*60*10;


	va_start(arg_ptr, arg_list);
	format = arg_list;
	output[0] = 0;
	len = vsnprintf(output,4000, format, arg_ptr); output[4000-1] = 0;
	n = lib_strlen(output);
	if (n==0) return TRUE; /* We dont want to send an empty string */
	ncat(req,output,maxreq-1);
	return TRUE;
}
int tcp_setlclport(Chan *c, int x)
{
	c->lclport = x;
	return TRUE;
}
int tcp_printf_nolog(Chan *c, char * arg_list, ...)
{
		va_list arg_ptr;
		char *format;
		int len;
	char tmp2[BFSZ];
	char output[5000];
	int r;

		va_start(arg_ptr, arg_list);
		format = arg_list;
	output[0] = 0;
		len = vsnprintf(output,4000, format, arg_ptr); output[4000-1] = 0;
	r = tcp_write(c,output,lib_strlen(output));
	if (r<(int) lib_strlen(output)) {
  		tcp_lasterror(c);
		imsg("tcp_printf2: failed only %d sent %s\n",r,tcp_error(c,tmp2));
		return FALSE;
	}
	if (r<=0) return FALSE;
	return TRUE;
}
int tcp_getip(Chan *c, char *ipbf)
{
	if (lib_strlen(c->theirname)>0) {
		ncpy(ipbf,c->theirname,60);
		return TRUE;
	}
	strcpy(ipbf,"");
	if (c->sk==INVALID_SOCKET) return FALSE;

	return tcp_getip_raw(c->sk,ipbf);
}
#ifdef DO_IPV6
int tcp_getip_raw_ipv6(SOCKET sk, char *ipbf)
{
    struct sockaddr_storage addr;
	int len;
	char AddrName[NI_MAXHOST];

	strcpy(AddrName,"");
    len = sizeof (addr);
    if (getpeername(sk, (struct sockaddr*)& addr, &len) == SOCKET_ERROR) 	return FALSE;
    if (getnameinfo( (struct sockaddr*)&addr, len, AddrName, sizeof (AddrName), NULL, 0, NI_NUMERICHOST) !=0) return FALSE;
	ncpy(ipbf,AddrName,100-1);
	return TRUE;
}
#endif
int tcp_getip_raw2(SOCKET sk, char *ipbf);
int tcp_getip_raw(SOCKET sk, char *ipbf)
{
	int r = tcp_getip_raw2(sk,ipbf);
	if (fix_ipv4) {
		tcp_remove_prefix(ipbf);
	} 	
	return r;
}
int tcp_getip_raw2(SOCKET sk, char *ipbf)
{
	char itmp[BFSZ];
		int len;
		struct sockaddr_in usoc, *us;

#ifdef DO_IPV6
		if (do_ipv6) {
			strcpy(ipbf,""); // chrisp added 11 may 2022
			return tcp_getip_raw_ipv6(sk,ipbf);
		}
#endif

	strcpy(ipbf,"");
	if (sk==INVALID_SOCKET) return FALSE;
		len=sizeof(struct sockaddr_in);
		us = &usoc;
		if (getpeername(sk,(struct sockaddr *) us,&len) < 0 ){
		  return FALSE;
		}

		strcpy(ipbf,inet_ntoa_safe(us->sin_addr,itmp));
		return TRUE;
}
int tcp_remove_prefix(char *ipbf)
{
	char tmp[BFSZ];
	char *s = strstrnc(ipbf,"::ffff:");
	if (s==NULL) return FALSE;
	ncpy(tmp,s+7,BFSZ-1);
	s = strchr(tmp,']'); 
	if (s!=NULL) *s = 0;
	ncpy(ipbf,tmp,60);
	return TRUE;
}
int tcp_getmyip_raw_ipv6(SOCKET sk, char *myip);
int tcp_fix_ipv4(int x)
{
	fix_ipv4 = x;
	//imsg("tcp_fix_ipv4 %d\n",fix_ipv4);
	return TRUE;
}
int tcp_getmyip(Chan *c, char *ipbf)
{
	if (lib_strlen(c->myname)>0) {
		ncpy(ipbf,c->myname,60);
		return TRUE;
	}
	strcpy(ipbf,"");
	if (!tcp_getmyip_raw(c->sk,ipbf)) {
		tcp_lasterror(c);
		return FALSE;
	}
	return TRUE;
}
int tcp_getmyip_raw2(SOCKET sk, char *ipbf);

int tcp_getmyip_raw(SOCKET sk, char *ipbf)
{
	int r = tcp_getmyip_raw2(sk,ipbf);
	//imsg("getmyip_raw before trimmed (%s)\n",ipbf);
	if (fix_ipv4) {
		tcp_remove_prefix(ipbf);
		//imsg("getmyip_ipv6 trimmed (%s)\n",ipbf);
	} 	
	return r;
}
int tcp_getmyip_raw2(SOCKET sk, char *ipbf)
{
		int len;
	char itmp[BFSZ];
		struct sockaddr_in usoc, *us;

	strcpy(ipbf,"");
	if (sk==INVALID_SOCKET) return FALSE;
#ifdef DO_IPV6
	if (do_ipv6) {
		if (tcp_getmyip_raw_ipv6(sk,ipbf)) {
			imsg("getmyip_ipv6 found (%s)\n",ipbf);
			return TRUE;
		} else {
			imsg("getmyip_ipv6 failed so trying ipv4 (%s)\n",ipbf);
		}
	}
#endif

		len=sizeof(struct sockaddr_in);
		us = &usoc;
		if (getsockname(sk,(struct sockaddr *) us,&len) < 0 ){
		  return FALSE;
		}

		strcpy(ipbf,inet_ntoa_safe(us->sin_addr,itmp));
		if (do_ipv6) {
			imsg("getmyip_ipv4 found (%s)\n",ipbf);
		}
		return TRUE;
}
#ifndef SS_PORT
#define SS_PORT(ssp) (((struct sockaddr_in*)(ssp))->sin_port)
#endif

int tcp_getports(Chan *c, int *local, int *remote)
{
	if (c->lcl_saved!=0) {
		*local = c->lcl_saved;
		*remote = c->rmt_saved;
		return TRUE;
	}
	return tcp_getports_raw(c->sk,local,remote);
}
#ifdef DO_IPV6
int tcp_getports_raw_ipv6(SOCKET sk, int *local, int *remote)
{
    struct sockaddr_storage addr;
	int len;
	char AddrName[BFSZ];

    len = sizeof (addr);
    if (getpeername(sk, (struct sockaddr*)& addr, &len) == SOCKET_ERROR) 	return FALSE;
    if (getnameinfo((struct sockaddr*)& addr, len, AddrName, sizeof (AddrName), NULL, 0, NI_NUMERICHOST) !=0) return FALSE;
	*remote = ntohs(SS_PORT(&addr));

    len = sizeof (addr);
	if (getsockname(sk, (struct sockaddr*)& addr, &len) == SOCKET_ERROR) return FALSE;
    if (getnameinfo((struct sockaddr*)& addr, len, AddrName, sizeof (AddrName), NULL, 0, NI_NUMERICHOST) !=0) return FALSE;
	*local = ntohs(SS_PORT(&addr));
	return TRUE;

}
int tcp_getmyip_raw_ipv6(SOCKET sk, char *xmyip)
{
    struct sockaddr_in6 addr;
	int len;
	int r;
	char myip[BFSZ];
	strcpy(myip,"");
	strcpy(xmyip,"");
	len = sizeof(addr);
	if (getsockname(sk, (struct sockaddr*)& addr, &len) == SOCKET_ERROR) {
		imsg("getmyip getsockname failed %s\n",strerror(errno));
		return FALSE;
	}
    if ((r = getnameinfo((struct sockaddr*)& addr, len, myip, BFSZ-1, NULL, 0, NI_NUMERICHOST)) !=0) {
		imsg("getmyip getnameinfo failed %d %s\n",r,strerror(errno));
		return FALSE;
	}
	ncpy(xmyip,myip,100-1);
	if (lib_strlen(myip)==0) return FALSE;
	if (strcmp(myip,"0.0.0.0")==0) return FALSE;
	return TRUE;
}
char *tcp_ipv6_to_string(struct in6_addr *addr, char *ip) // thisfunction... 
{
	int len=BFSZ-1;
	struct sockaddr_in6 ad;
	int len2 = sizeof(ad.sin6_addr);
	strcpy(ip,"");
	memset(&ad,0,sizeof(ad));
	ad.sin6_family = AF_INET6;
	ad.sin6_addr = *addr;

#ifdef WIN32
			WSAAddressToString((struct sockaddr *) &ad,sizeof(ad),NULL,ip,&len);
#else
			inet_ntop(AF_INET6,addr,ip,BFSZ-1);
#endif
			return ip;
}

#endif
int tcp_getports_raw(SOCKET sk, int *local, int *remote)
{
		int len;
		struct sockaddr_in hsoc, usoc, *him, *us;

	if (sk==INVALID_SOCKET) return FALSE;

#ifdef DO_IPV6
	if (do_ipv6) {
		if (tcp_getports_raw_ipv6(sk,local,remote)) return TRUE; // Otherwise try for ipv4 result...
	}
#endif


		/* get the port on our machine he is connecting to */
		len=sizeof(struct sockaddr_in);
		us = &usoc;
		if (getsockname(sk,(struct sockaddr *) us,&len) < 0 ){
		  imsg("getports getsockname() failed \n");
		  return FALSE;
		}

		*local = (int) ntohs(us->sin_port);
	
	
		him = &hsoc;
		len=sizeof(struct sockaddr_in);
		if ( getpeername(sk, (struct sockaddr *) him, &len) < 0 ) {
		  imsg("getpeername failed\n");
		  return FALSE;
		}

		*remote = (int) ntohs(him->sin_port);
		return TRUE;
}

int tcp_getport(Chan *c, int *local)
{
		int remote;

	if (c->sk==INVALID_SOCKET) return FALSE;
	return 	tcp_getports(c,local,&remote);
}
#ifdef WIN32
#define HAS_NODELAY TRUE
#else
#ifdef TCP_NODELAY
#define HAS_NODELAY
#endif
#endif
int tcp_nodelay(Chan *c, int x)
{
	int on = x;
	int r;

	if (c->sk==INVALID_SOCKET) return FALSE;
#ifdef HAS_NODELAY
	r = setsockopt(c->sk, IPPROTO_TCP, TCP_NODELAY,(char *) &on, sizeof(int));
#else
	r = 1;
#endif
		tcp_lasterror(c);
	if (r!=0) return FALSE;
	return TRUE;

}
int tcp_lib_isnt(void)
{
#ifdef WIN32
	int r;
	OSVERSIONINFO info;
	info.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
	r = GetVersionEx(&info);
	if (!r) {

		return FALSE;
	}
	
	if (info.dwPlatformId==VER_PLATFORM_WIN32_NT) return TRUE;
	return FALSE;
#else
	return FALSE;
#endif

}

/*tcp_setrcbf_raw: sets minimum buffer size for receives on a socket
  Arguements: c: the socket
			  sz: minimum buffer size in bytes				 */
void tcp_setrcvbf_raw(SOCKET c, int sz)
{
	int xsz=sz;
	int was;
	int xlen=sizeof(xsz);
	char *s;

#ifdef WIN32
	/* Windows 98 is short of buffers so must not increase the usage */
	if (!tcp_lib_isnt()) {
		imsg("Windows 9x so not setting rcv buffer sizes\n");
		return ;
	}
#endif
	
	
	s = (char *) &was;
		getsockopt(c, SOL_SOCKET, SO_RCVBUF, s, &xlen);
	if (was>=sz) return;
	s = (char *) &xsz;
		setsockopt(c, SOL_SOCKET, SO_RCVBUF, s, sizeof(xsz));
	s = (char *) &was;
		getsockopt(c, SOL_SOCKET, SO_RCVBUF, s, &xlen);

	if (was<sz) imsg("setsockopt: setrcvbf failed got %d/%d bytes\n",was,sz);
}
char *tcp_hostname(char *host)
{
	static char bf[BFSZ];
	char *s;
	if (lib_strlen(bf)>0) {
		ncpy(host,bf,100);
		return host;
	}
	tcp_init_wsa();
	if (gethostname(host,190)!=0) strcpy(host,"127.0.0.1");
	ncpy(bf,host,BFSZ-1);
	// On macosx you can specify a host name that contains spaces
	// this then breaks things for unique file names that use the name...
	s = strchr(host,' '); if (s!=NULL) *s = 0;
	return host;
}
char *tcp_hostname_full(char *host)
{
	char *s;
	char dom[BFSZ];
	imsg("tcp_hostname_full called\n");
	strcpy(dom,"");
	tcp_init_wsa();
	if (gethostname(host,190)!=0) strcpy(host,"127.0.0.1");
	imsg("hostname_full called host (%s)\n",host);
#ifndef WIN32
	if (strchr(host,'.')==NULL) {
		if (getdomainname(dom,190)!=0) {
			ncat(host,".",BFSZ-1);
			ncat(host,dom,BFSZ-1);
			imsg("tcp_hostname_full: (%s)\n",dom);
		} else {
			imsg("tcp_hostname_failed: (%s)\n",strerror(errno));
		}

	}
#endif
	// On macosx you can specify a host name that contains spaces
	// this then breaks things for unique file names that use the name...
	s = strchr(host,' '); if (s!=NULL) *s = 0;
	return host;
}

#include <fcntl.h>

#ifdef xx_is_freebsd
#define USE_O_NONBLOCK 1
#endif
int tcp_setblockraw(SOCKET fd, int t)
{
	int w;
	int flags=0;
	if (t) w=0; else w=1;

	if (fd==INVALID_SOCKET) return FALSE;
#ifdef USE_O_NONBLOCK
	flags = fcntl(fd, F_GETFL, 0);
	if (flags == -1) {
		if (emsg!=NULL) emsg("Unable to set non blocking %s\n",strerror(errno));
		return FALSE;
	}
	if (t) flags |= O_NONBLOCK;
	else flags &= ~O_NONBLOCK;
	if (fcntl(fd, F_SETFL, flags)!=0) return FALSE;
#else
	if (socket_ioctl(fd,FIONBIO,&w)!=0) {dmsg("socket_ioctl failed (%s)", strerror(errno));return FALSE;}
#endif
	return TRUE;
}

int tcp_setblock(Chan *c, int t)
{
	if (!tcp_isopen(c)) return FALSE;
	c->block = t;
	if (tcp_setblockraw(c->sk,t)) return TRUE;
	if (imsg != NULL) imsg("Cant set blocking to %d for socket %d %s\n", t,c->sk,strerror(errno));
	return FALSE;
}

void tcp_setsndbf_raw(SOCKET c, int sz)
{
	int xsz=sz;
	int was;
	int xlen=sizeof(xsz);
	char *s;

#ifdef WIN32
	/* Windows 98 is short of buffers so must not increase the usage */
		if (!tcp_lib_isnt()) return ;
#endif
	
	s = (char *) &was;
		getsockopt(c, SOL_SOCKET, SO_SNDBUF, s, &xlen);

	if (was>=sz) return;
	s = (char *) &xsz;
		setsockopt(c, SOL_SOCKET, SO_SNDBUF, s, sizeof(xsz));
	s = (char *) &was;
		getsockopt(c, SOL_SOCKET, SO_SNDBUF, s, &xlen);

	if (was<sz) imsg("setsockopt: setsndbf failed got %d/%d bytes\n",was,sz);
}
#ifdef WIN32
char *tcp_error(Chan *c, char *bf)
{
	int err;
	char *s="";

	strcpy(bf,"Error not defined");
	if (c==NULL) {
		sprintf(bf,"Channel structure is NULL");
		return bf;
	}
	err = c->xerrno;
#ifdef HAS_SSL
	if (c->ssl!=NULL) {
		return myssl_lastmsg(c->ssl);
	}
#endif
	if (err == 0) strcpy(bf,"Err Code Zero");
	else if (err == -1) strcpy(bf,"Connection was closed.");
#ifdef WIN32
	else if (err == WSAEWOULDBLOCK) strcpy(bf,"Socket Timeout");
#else
	else if (err == EWOULDBLOCK) strcpy(bf,"Socket Timeout");
#endif
	else system_error(err,bf,BFSZ);
	/*
		if (err==WSANOTINITIALISED)		sprintf(bf,"A successful WSAStartup must occur before using this API.");
		else if (err==WSAENETDOWN)	  sprintf(bf,"The Windows Sockets implementation has detected that the network subsystem has failed.");
		else if (err==WSAEACCES)		sprintf(bf,"The requested address is a broadcast address, but the appropriate flag was not set.");
		else if (err==WSAEINTR)			sprintf(bf,"The (blocking) call was canceled via WSACancelBlockingCall.");
		else if (err==WSAEINPROGRESS)   sprintf(bf,"A blocking Windows Sockets operation is in progress.");
		else if (err==WSAEFAULT)		sprintf(bf,"The buf argument is not in a valid part of the user address space.");
		else if (err==WSAENETRESET)	 sprintf(bf,"The connection must be reset because the Windows Sockets implementation dropped it.");
		else if (err==WSAENOBUFS)	   sprintf(bf,"The Windows Sockets implementation reports a buffer deadlock.");
		else if (err==WSAENOTCONN)	  sprintf(bf,"The socket is not connected.");
		else if (err==WSAENOTSOCK)	  sprintf(bf,"The descriptor is not a socket.");
		else if (err==WSAEOPNOTSUPP)	sprintf(bf,"MSG_OOB was specified, but the socket is not of type SOCK_STREAM.");
		else if (err==WSAESHUTDOWN)	 sprintf(bf,"The socket has been shutdown; it is not possible to sendln on a socket after shutdown has been invoked with how set to 1 or 2.");
		else if (err==WSAEWOULDBLOCK)   sprintf(bf,"The socket is marked as non-blocking and the requested operation would block.");
		else if (err==WSAEMSGSIZE)	  sprintf(bf,"The socket is of type SOCK_DGRAM, and the datagram is larger than the maximum supported by the Windows Sockets implementation.");
		else if (err==WSAEINVAL)		sprintf(bf,"The socket has not been bound with bind.");
		else if (err==WSAECONNABORTED)  sprintf(bf,"The virtual circuit was aborted due to timeout or other failure.");
		else if (err==WSAECONNRESET)	sprintf(bf,"The virtual circuit was reset by the remote side.");
		else if (err==WSAETOOMANYREFS)  sprintf(bf,"Too many refs.");
		else if (err==WSAETIMEDOUT)	 sprintf(bf,"Timedout");
		else if (err==WSAECONNREFUSED)  sprintf(bf,"Connection refused");
		else if (err==WSAEDESTADDRREQ)  sprintf(bf,"DestAddrReq");
		else if (err==WSAEMSGSIZE)	  sprintf(bf,"Message Size");
		else if (err==WSAEADDRINUSE)	sprintf(bf,"Address in use");
		else if (err==WSAEADDRNOTAVAIL) sprintf(bf,"Address not avail");
		else if (err==WSAENETUNREACH)   sprintf(bf,"Network Unreachable");
		else if (err==WSAENETRESET)	 sprintf(bf,"Network Reset");
		else if (err==WSAENOBUFS)	   sprintf(bf,"No bufs");
		else if (err==WSAESHUTDOWN)	 sprintf(bf,"Shutdown");
		else if (err==WSAENAMETOOLONG)  sprintf(bf,"Name too long");
		else if (err==WSAEHOSTDOWN)	 sprintf(bf,"Host down");
		else if (err==WSAEHOSTUNREACH)  sprintf(bf,"Host Unreachable");
		else if (err==WSAENOTEMPTY)	 sprintf(bf,"Not Empty");
		else if (err==WSAHOST_NOT_FOUND)	 sprintf(bf,"Host Not Found");
		else if (err==WSATRY_AGAIN)	 sprintf(bf,"Host Not Found Try Again");
		else if (err==WSANO_RECOVERY)	 sprintf(bf,"Host Lookup, no_recovery");
		else if (err==WSANO_DATA)	 sprintf(bf,"Host Lookup, no_data");
		else if (err==WSANO_ADDRESS)	 sprintf(bf,"Host Lookup, no_address");
		else sprintf(bf,"Socket Unkown error code %d ",err);
	*/
	if (lib_strlen(c->erradd)>0) {
		ncat(bf," ",BFSZ-100); // must be -100 because some code passes buffers smaller than BFSZ! 
		ncat(bf,c->erradd,BFSZ-100);
	}
	return bf;
}
#else
char *tcp_error(Chan *c, char *bf)
{
	int err;
	char *serr;
	if (c==NULL) {
		sprintf(bf,"Channel structure is NULL");
		return bf;
	}
	err = c->xerrno;
	if (err == 0) {strcpy(bf,"Err Code Zero"); goto ret;}
	if (err == -1) {
		strcpy(bf,"Connection was closed.");
		goto ret;
	}
#ifdef HAS_SSL
	if (c->ssl!=NULL) {
		ncpy(bf,myssl_lastmsg(c->ssl),BFSZ-1);
		goto ret;
	}
#endif
	serr=strerror(c->xerrno);
	if (serr)
		ncpy(bf,serr,100);
	else 
		ncpy(bf,"Invalid error code",100);
ret:
	if (lib_strlen(c->erradd)>0) {
		ncat(bf," ",BFSZ-1);
		ncat(bf,c->erradd,BFSZ-1);
	}

	return bf;
}
#endif

#ifdef DO_IPV6
int tcp_getname_ipv6(char *ipnum, char *ipnamex)
{
	struct addrinfo hints,*res=NULL,*a;
	int ret=FALSE;
	int on=-1;
	char ipname[BFSZ];

	if (*ipnum!='[') return FALSE;
	strcpy(ipname,"");
	strcpy(ipnamex,"");
    memset(&hints, 0, sizeof (hints));
    hints.ai_family = AF_INET6;
    hints.ai_socktype = SOCK_STREAM;
    ret = getaddrinfo(ipnum, "", &hints, &res);
    if (ret != 0) {
		return FALSE;
    }
    for (a=res;a!=NULL;a=a->ai_next) {
		if (getnameinfo(a->ai_addr,(int) a->ai_addrlen,ipname,BFSZ-1,NULL,0,0)==0) {ret=TRUE; break;}
	}
	if (res!=NULL) freeaddrinfo(res);  res = NULL;
	ncpy(ipnamex,ipname,100-1);
	return ret;
}
#endif

int tcp_getname(char *ipnumber, char *ipname)
{
	struct hostent *hp;
	struct sockaddr_in sin;
	struct hostent hresult;
	char hbuffer[BFSZ];
	int h_err=0;
	time_t start=0;
	char *tmp=NULL;
	static int isbusy;

	strcpy(hbuffer,"");
	memset(&hresult,0,sizeof(hresult));
#ifdef DO_IPV6
	if (do_ipv6) {
		if (tcp_getname_ipv6(ipnumber,ipname)) return TRUE;
	}
#endif
	strncpy(ipname,ipnumber,200); ipname[200] = 0;
	memset(&sin,0,sizeof(struct sockaddr_in));
	sin.sin_addr.s_addr = inet_addr(ipnumber);
#ifdef MUTEX_GETHOST
	start=time(NULL);
	tcp_mutex_wait3();
	if (isbusy) {
		tcp_mutex_release3();
		emsg("gethostbyaddr failed as it was busy (%s)\n",ipnumber);
		return FALSE;
	}
	isbusy = TRUE;
	tcp_mutex_release3();
#endif

//	hp = mygethostbyaddr_r( (char *) &(sin.sin_addr), sizeof(sin.sin_addr), AF_INET,&hresult,hbuffer,sizeof(hbuffer),&h_err);
	hp = mygethostaddr((const char*)(char *) &(sin.sin_addr),&hresult,&h_err,&tmp);


#ifdef MUTEX_GETHOST
	tcp_mutex_wait3();
	isbusy = FALSE;
	tcp_mutex_release3();
	if (time(NULL)-start>4) emsg("gethostbyaddr (%s) took %d seconds, disable g_lookup_names\n",ipnumber,time(NULL)-start);
#endif
	if (hp==NULL) return FALSE;
	if (hp->h_name==NULL) {
		mygethostname_free(&tmp);
		return FALSE;
	}
	strncpy(ipname,hp->h_name,200); ipname[200] = 0;
	mygethostname_free(&tmp);
	return TRUE;
}
void tcp_close_on_exec_raw(int fd)
{
	// FD_CLOEXEC this is really setting this .
#ifndef WIN32
	if (fd== -1) return;
		if (fcntl(fd,F_SETFD,1)==-1){
				/*imsg("Error doing close on exec fcntl %d %s\n",fd,strerror(errno));*/
		}
#endif
}
void tcp_open_on_exec_raw(int fd)
{
	// FD_CLOEXEC this is really setting this .
#ifndef WIN32
	if (fd== -1) return;
		if (fcntl(fd,F_SETFD,0)==-1){
				/*imsg("Error doing close on exec fcntl %d %s\n",fd,strerror(errno));*/
		}
#endif
}
char * tcp_buffer_get(Chan *c, int *len)
{
	*len = c->buffer_len;
	return c->buffer;
}
void tcp_buffer_clear(Chan *c)
{
	free(c->buffer); c->buffer = NULL;
	c->buffer_len = 0;
	c->buffer_len_malloced = 0;
	c->isbuffer = FALSE;
}
void tcp_out_fn(Chan *c,int (*fn)(void *x, char *bf, int len), void *p1)
{
	c->out_fn = fn;
	c->out_p1 = p1;
}
void tcp_buffer_set(Chan *c, int x)
{
	c->isbuffer = x;
	if (!c->isbuffer) {
		tcp_buffer_flush(c);
	}
}
int tcp_buffer_flush2(Chan *c, char *s, int len)
{
	int want;
again:
	want = 127000 - c->buffer_len;
	if (want<=0) {tcp_buffer_flush(c); goto again;}
	if (want>len) want = len;
	tcp_buffer_add(c,s,want);
	if (c->buffer_len>126000) tcp_buffer_flush(c);
	return want;
}
int tcp_buffer_flush(Chan *c)
{
	int was;
	int sent=0;
	int sending;
	int tout = 1000 * 60 * 10;
	if (c->write_timeout) tout = c->write_timeout;
	if (c->buffer_len==0) return TRUE;
	was = c->isbuffer;
	c->isbuffer = FALSE;

	sending = c->buffer_len;
	sent = tcp_write_timed(c,c->buffer,c->buffer_len,tout);
	free(c->buffer); c->buffer = NULL;
	c->buffer_len = 0;
	c->buffer_len_malloced = 0;
	c->isbuffer = was;
	if (sent!=sending) {
		return FALSE;
	}
	return TRUE;
}
int tcp_buffer_len(Chan *c)
{
	return c->buffer_len;
}

#define TCP_BUFFER_ADD_FAST 1

int tcp_buffer_add(Chan *c, char *bf, int len)
{
	int newlen;
	if (len<=0) return 0;
	newlen = len + c->buffer_len;

#ifdef TCP_BUFFER_ADD_FAST
	if (c->buffer_len_malloced<newlen) {
		c->buffer_len_malloced=newlen*2+500;
		if (c->buffer_len_malloced<32000) c->buffer_len_malloced = 32000;
		if (c->buffer_len_malloced<newlen) c->buffer_len_malloced=newlen;
		c->buffer = realloc(c->buffer,c->buffer_len_malloced);
		
	}
#else
		c->buffer = realloc(c->buffer,newlen);
		
#endif
	if (c->buffer==NULL) return 0; // failed badly
	safe_memcpy(c->buffer + c->buffer_len,bf,len,c->buffer,newlen);
	c->buffer_len = newlen;

	if (newlen>10000) {
		if (!tcp_buffer_flush(c)) return 0; 
	} 
	return len;
}
void tcp_setlinger(Chan *c, int xon, int hunds)
{
//	char tmp[1000];
	struct linger ling;
	if (xon) xon = 1;
	else xon = 0;
	ling.l_onoff = xon;
	ling.l_linger = hunds;
	if (setsockopt(c->sk, SOL_SOCKET, SO_LINGER, (char *) &ling, sizeof(ling))!=0) {
		//tcp_lasterror(c);
		//imsg("Unable to set socket to linger, not serious\n",tcp_error(c,tmp));
	}
}
/*
typedef struct CLOSELIST {SOCKET sk; time_t whence; struct CLOSELIST *next;} Clist;
int tcp_close_later(SOCKET sk)
{


}
int tcp_idle(void)
{

}
*/

int tcp_handle_count(void)
{
	int i,n=0;
	SOCKET list[700];
	for (i=0; i<700; i++) {
		list[i] = INVALID_SOCKET;
	}
	for (i=0; i<700; i++) {
		list[i] = socket(AF_INET, SOCK_STREAM, 0);
		hcount_open((int)list[i]);
		if (list[i]==INVALID_SOCKET) break;
		n = i+1;
	}
	for (i=0; i<n; i++) {
		if (list[i]!=INVALID_SOCKET) {
			hcount_close((int)list[i]);
			tcp_socket_close(list[i],__FILE__,__LINE__);
		}
		list[i] = INVALID_SOCKET;
	}
	return n;
}

static char kfile[1000];
static char pfile[1000];
int tcp_ssl_setkey_done = FALSE;
int tcp_ssl_setkey(char *pkeyfile, char *keyfile)
{
	strncpy(kfile,keyfile,200);
	strncpy(pfile,pkeyfile,200);
	//imsg("tcp_ssl_setkey (%s) (%s)\n",kfile,pfile);
	tcp_ssl_setkey_done = TRUE;
	return TRUE;
}
int tcp_ssl_setdir(char *dir) {
#ifdef HAS_SSL
	return myssl_setdir(dir);
#else
	return TRUE;
#endif
}
int tcp_ssl_clientcert(Chan *c, char *cert)
{
	char bf[BFSZ];
	ncpy(bf,cert,BFSZ-1);
	c->ssl_clientcert=strdup(strlwr(bf));
	return TRUE;
}



static Wordlist *ssl_domains;
Wordlist *tcp_ssl_setdomains(Wordlist *domains) 
{
	Wordlist *old;

	old = ssl_domains;
	ssl_domains = domains;
#ifdef HAS_SSL
	myssl_set_domains(ssl_domains);
#endif
	return old;
}
int tcp_ssl_cmp(Chan *a, Chan *b)
{
#ifdef HAS_SSL
//	return myssl_cmp(a->ssl,b->ssl);
	return 0;
#else
	return 0;
#endif

}
char *tcp_myssl_get_protocol_name_used(Chan *c){
#ifdef HAS_SSL
	if (c->ssl != NULL) return myssl_get_protocol_name_used(c->ssl);
#endif
	return "No SSL, plain text mode";
}
int tcp_ssl_ciphers_web(char *xciphers)
{
#ifdef HAS_SSL
	myssl_set_ciphers_web(xciphers);
#endif
	return TRUE;
}
int tcp_ssl_ciphers(char *xciphers)
{
#ifdef HAS_SSL
	myssl_set_ciphers(xciphers);
#endif
	return TRUE;
}
int g_xdebug;
int tcp_ssl_init_real(Chan *c, int require_cert, int isserver, char *domain, Chan *inherit, char *reason,int isweb,char *mstate)
{
#ifdef HAS_SSL
	char lpfile[BFSZ];
	char lkfile[BFSZ];
	char xmstate[BFSZ];
	static int done = FALSE;
	ncpy(lpfile,pfile,BFSZ-1);
	ncpy(lkfile,kfile,BFSZ-1);
	if (mstate==NULL) mstate = xmstate; // not wanted so point to buffer to write instead. 

	/*if (!tcp_ssl_setkey_done){
		if (dmsg != NULL) dmsg("tcp_ssl_init You have not specified the pfile or kfile with tcp_ssl_setkey call.\n");
		emsg("tcp_ssl_init You have not specified the pfile or kfile with tcp_ssl_setkey call.\n");
		strcpy(reason, "tcp_ssl_init You have not specified the pfile or kfile with tcp_ssl_setkey call.\n");
		return FALSE;
	}*/
	//imsg("tcp_ssl init_domain-a(s) %d (%s) (%s)\n",isserver,kfile,pfile);
	if (g_xdebug) printf("tcp_ssl_init_real. step1\n");
	if (!isserver) strcpy(lpfile,"");
	if (!isserver) strcpy(lkfile,"");

	if (!done) {
		strcpy(mstate,"ssl-initdom");
		if (g_xdebug) printf("tcp_ssl_init_real. step2\n");
		tcp_mutex_wait();
		if (g_xdebug) printf("tcp_ssl_init_real. step3\n");
		if (!done) {
			done = TRUE;
			tcp_mutex_release();
			imsg("tcp_ssl init_domain(s) (%s) (%s)\n",lkfile,lpfile);

			if (ssl_domains) {
				if (g_xdebug) printf("tcp_ssl_init_real. step4\n");
				if (!myssl_init_domains(lpfile,lkfile,ssl_domains,reason,(int (*)(char *,...))imsg,(int (*)(char *,...))emsg))
					{if (dmsg!=NULL) dmsg("tcp_ssl_init failed, myssl init (%s)\n", reason); strcpy(reason,"myssl_init failed.");return FALSE;}
			}
			else {
				if (g_xdebug) printf("tcp_ssl_init_real. step5\n");
				if (!myssl_init(lpfile,lkfile,reason,(int (*)(char *,...))imsg,(int (*)(char *,...))emsg))
					{if (dmsg!=NULL) dmsg("tcp_ssl_init failed, myssl init (%s)\n", reason); strcpy(reason,"myssl_init3 failed.");return FALSE;}
			}
		}
		else {
			tcp_mutex_release();
		}
	}
	if (c==NULL) return TRUE;

	if (c->ssl!=NULL) {
		strcpy(reason,"ssl is already enabled");
		return FALSE;
	}
	strcpy(mstate,"ssl-init2");

	if (!c->never_close) if (open_hist) open_hist((int)c->sk,FALSE,FALSE,__FILE__,__LINE__,"ssl_init"); // NORMAL CLOSE
	if (g_xdebug) printf("tcp_ssl_init_real. step6\n");
	hcount_close((int) c->sk);
	tcp_save_sk(c->sk,FALSE);
	c->never_close = TRUE; // so it won't record it as closed again.

	if (g_xdebug) printf("tcp_ssl_init_real. step7\n");
	strcpy(mstate,"ssl-init3");
	c->ssl = myssl_takeover_real(c, c->sk, require_cert, isserver,domain,inherit==NULL ? NULL : inherit->ssl,reason,isweb,mstate);
	strcpy(mstate,"ssl-init4");
	if (dmsg!=NULL) dmsg("tcp_ssl_init (%s)\n", reason);
	if (c->ssl==NULL) {
		c->sk = INVALID_SOCKET;  // if myssl_takeover fails, then it's closed, added by chrisp, 23/3/06
		// we should really close it I think...
		if (imsg!=NULL) imsg("tcp_ssl_init failed, cant takeover chanel. (%s)\n", reason);
		return FALSE;
	}
	if (dmsg!=NULL) dmsg("tcp_ssl_init succeded.\n");
	if (g_xdebug) printf("tcp_ssl_init_real. step8\n");
	return TRUE;
#else
	strcpy(reason, "SSL not compiled in this version.");
	imsg("SSL not compiled in this version.");
	if (g_xdebug) printf("tcp_ssl_init_real. step9\n");
	return FALSE;
#endif
}
int tcp_ssl_init(Chan *c, int isserver, Chan *inherit, char *reason)
{
	ncpy(c->rhosts,"",BFSZ-1);
	return tcp_ssl_init_real(c, -TRUE, isserver, NULL, inherit, reason,FALSE,NULL);
	/* -TRUE means use default */
}
int tcp_ssl_init_rhosts(Chan *c, int isserver, Chan *inherit, char *reason, char *rhosts, char *mstate)
{
	ncpy(c->rhosts,rhosts,BFSZ-1);
	return tcp_ssl_init_real(c, -TRUE, isserver, NULL, inherit, reason,FALSE,NULL);
	//return tcp_ssl_init_real(c, TRUE, isserver, NULL, inherit, reason,FALSE);
	/* -TRUE means use default */
}
int tcp_ssl_init_domain(Chan *c, int isserver, char *domain, Chan *inherit, char *reason, int isweb, char *mstate)
{
	return tcp_ssl_init_real(c, -TRUE, isserver, domain, inherit, reason,isweb,mstate);
	/* -TRUE means use default */
}
int tcp_write_ssl(Chan *c, char *bf, int len)
{
#ifdef HAS_SSL
		int r;
		r = myssl_write(c, c->ssl,bf,len);
	if (r<=0) tcp_lasterror(c);
		return r;
#else
	return -1;
#endif
}

#ifndef WIN32
void tcp_lib_usleep(int x)
{
#ifndef NONANO
	struct timespec rqtp;
	rqtp.tv_sec = x/1000000;
	rqtp.tv_nsec = (x%1000000)*1000;
	nanosleep(&rqtp,NULL);
#else
	usleep(x);
#endif
}
#endif

void tcp_lib_msleep(int x)
{
#ifdef WIN32
	Sleep(x);
#else
	tcp_lib_usleep(x*1000);
#endif
}


#define xtolower(_c)	( (_c)-'A'+'a' )
#define xtoupper(_c)	( (_c)-'a'+'A' )
char *strlwr(char *s)
{
		char *ss=s;
		for (;*s!=0;s++) if (sisupper(*s)) if (*s>0) *s = xtolower(*s);
		return ss;
}
char *ncpy(char *dst, char *src, int len)
{
		char *xdst = dst;
		int xlen = len;

		for (;(*src !=0) && (len>0); ) {*dst++  = *src++; len--;}
		*dst++ = 0;
		xdst[xlen] = 0;
		return xdst;
}
static int rawsk[FD_SETSIZE];
int tcp_socket_close_hard2(SOCKET sk, char *src, int line);
int tcp_close_all(void)
{
	/* Used by lib_spawn to close all sockets between fork and exec on freebsd4 */
	int i;
	for (i=0; i<FD_SETSIZE; i++) {
		if (rawsk[i]) tcp_socket_close_hard2(i,__FILE__,__LINE__);
	}
	return TRUE;
}
int tcp_save_sk(SOCKET sk, int isopen)
{
	if (sk<0) return FALSE;
	if (sk>=FD_SETSIZE) return FALSE;
	rawsk[sk] = isopen;
	return TRUE;
}
int tcp_log_closes(int x)
{
	log_closes = x;
	return TRUE;
}
int tcp_socket_close_hard(SOCKET sk)
{
	struct linger ling;
	int r;
	ling.l_onoff = 0;
	ling.l_linger = 0;
	setsockopt(sk, SOL_SOCKET, SO_LINGER, (char *) &ling, sizeof(ling));
	if (open_hist) open_hist((int)sk,FALSE,TRUE,__FILE__,__LINE__,"tcp_socket_close");  // Close log entry here, because alreayd logged in function that calls this! .  
	r = socket_close(sk);
	if (r!=0) {
		if (tcp_sk_wouldblock()) {
			emsg("tcp_socket_close: Socket close failed will retry - wouldblock2\n");
			tcp_setblockraw(sk,TRUE);
			if (open_hist) open_hist((int)sk,FALSE,TRUE,__FILE__,__LINE__,"tcp_slocket_close"); // LOGGED ONLY 
#ifdef WIN32
			r = socket_close(sk); // INVALID on linux but required  on windows if wouldblock returned.
#endif
		} else {
			emsg("tcp_socket_close: Socket close failed will retry - notblock2\n");
		}
	}
	tcp_socket_low_free(sk);
	return r;
}
int tcp_socket_close_hard2(SOCKET sk, char *src, int line)
{
	struct linger ling;
	int r;

	if (open_hist) open_hist((int)sk,FALSE,TRUE,src,line,"socket_close"); // LOGGED ONLY

	// Chrisp change so it will not block on closes
	ling.l_onoff = 0;
	ling.l_linger = 0;
	setsockopt(sk, SOL_SOCKET, SO_LINGER, (char *) &ling, sizeof(ling));

	r = socket_close(sk);
	tcp_socket_low_free(sk);
	return r;
}
int tcp_socket_close(SOCKET sk, char *src, int line)
{
	int r;

//emsg("tcp_socket_close\n");
	if (log_closes) imsg("tcp_socket_close fd=%d\n",sk);
	if (open_hist) open_hist((int)sk,FALSE,TRUE,src,line,"socket_close"); // LOGGED ONLY
	r = socket_close(sk);
	tcp_socket_low_free(sk);
	return r;
}
static char alphabet[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
                  "0123456789+/";
int tcp_mime_encode_base64(char *xin, int inlen, char *xout, int outlen)
/*
Warning - inlen must be one more than the length of the input.
e.g. mime_encode_base64_bad(txt,lib_strlen(txt)+1,bf,999)
*/

{
   int  index, shift, save_shift=0;
   int help_flag = 0, replace_flag = 0, perm_replace_flag = 0, quit = 0;
   int cycle_flag = 0;
   unsigned char blivit;
   unsigned long accum, value;
   char buf[280];
   int firsttime = 1;
   int skipflag = 0;
   int printmsg = 1;


      shift = 0;
      accum = 0;
      index = 0;
      while ( (inlen>0) || (shift != 0) )  {
         if ( (inlen!=0) && ( quit == 0 ) ) {
		blivit = *xin++; inlen--;
		if ( inlen<=0 ) {
		       quit = 1;
		       save_shift = shift;
		       blivit = 0;
		}
         } else {
            quit = 1;
            save_shift = shift;
            blivit = 0;
         }

         if ( (quit == 0) || (shift != 0) )
         {
            value = (unsigned long)blivit;
            accum <<= 8;
            shift += 8;
            accum |= value;
         } /* ENDIF */

         while ( shift >= 6 )
         {
            shift -= 6;
            value = (accum >> shift) & 0x3Fl;
            blivit = alphabet[value];
			if (index>270) break;
            buf[index++] = blivit;
            if ( quit != 0 )  shift = 0;
         }
      }

      if ( save_shift == 2 ) {
         buf[index++] = '=';
         buf[index++] = '=';
      } else if ( save_shift == 4 ) {
         buf[index++] = '=';
      }

      buf[index] = 0;
      ncpy(xout,buf,index);
	return TRUE;
}


#ifndef NO_TCP_DOWNLOAD_FILE
static int download_tout=120;
int tcp_http_get(char *url, char *reason)
{
	time_t modified = 0;
	int is_modified = FALSE;
	int binmode=FALSE;
	return tcp_download_file_do(url,"","",modified,&is_modified,reason,binmode,NULL,NULL,NULL,NULL);
}
int tcp_download_file(char *url, char *proxy, char *localname, time_t modified, int *is_modified, char *reason, int binmode)
{
	return tcp_download_file_do(url,proxy,localname,modified,is_modified,reason,binmode,NULL,NULL,NULL,NULL);
}
int tcp_download_timeout(int x)
{
	download_tout = x;
	return TRUE;
}
static int  v11;
int tcp_set_v11(int x)
{
	v11 = x;
	return TRUE;
}
static int do_gzip;
int tcp_set_do_gzip(int x)
{
	do_gzip = x;
	return TRUE;
}
int tcp_download_file_do(char *xurl, char *proxy, char *localname, time_t modified, int *is_modified, char *reason, int binmode, int (*progress_fn)(void *param, int so_far, int total),void *progress_param, char *user, char *pass)
{
	return tcp_download_file_real(xurl, proxy, localname, modified, is_modified, reason, binmode, progress_fn, progress_param, user, pass, "",NULL);
}
static int readclose;
int tcp_set_readclose(int x)
{
	readclose = x;
	return TRUE;
}
/*
POST / HTTP/1.1
HOST: host.example.com
Cookie: some_cookies...
Connection: Keep-Alive
Content-Type: multipart/form-data; boundary=12345

--12345
Content-Disposition: form-data; name="sometext"

some text that you wrote in your html form ...
--12345
Content-Disposition: form-data; name="name_of_post_request" filename="filename.xyz"

content of filename.xyz that you upload in your form with input[type=file]
--12345
Content-Disposition: form-data; name="image" filename="picture_of_sunset.jpg"

content of picture_of_sunset.jpg ...
--12345--

*/
int tcp_download_file_real(char *xurl, char *proxy, char *localname, time_t modified, int *is_modified, char *reason, int binmode, int (*progress_fn)(void *param, int so_far, int total),void *progress_param, char *user, char *pass, char *cookies, int *vcode)
/*
Use progress_fn for obtaining download progress.
If progress_fn returns false, then the download is aborted.
*/
{
	/* Use date on file so we only fetch file if it's been modified */
	// Returns 'true' if file is successfully fetched, or if file has not been modified since date specified */
	char host[BFSZ];
	char url[BFSZ];
	Chan *c=NULL;
	int sz=0;
	char rest[BFSZ];
	int count;
	char *u=url;
	time_t server_copy_last_modified_at=0;
	char tmp3[BFSZ];
	char bf[BFSZ];
	int gotsize = FALSE;
	char fname[BFSZ];
	int downloadsize = 0;
	char *s;
	char tmp[BFSZ];
	int ishead = TRUE;
	FILE *fnew=NULL;
	int gotfile = FALSE;
	char bf2[BFSZ];
	int ntry=0;
	int tout = download_tout;
	int chunked=FALSE;
	int do_ssl=FALSE;
	int port=80;
#define REQSZ 4000
	char req[REQSZ];

	if (vcode) *vcode = 0;
	ncpy(url,xurl,BFSZ-1);
try_again:
	imsg("tcp: Downloading (%s) from (%s) via (%s), timeout %d seconds\n",localname,url,proxy,download_tout);

	tcp_new(&c);
	*is_modified = TRUE;
	ncpy(rest,url,BFSZ-1);
	ncpy(fname,localname,BFSZ-1);
	ncat(fname,"_tmp",BFSZ-1);
	/* url = http://host.name:port/restofit */
	u = url;
	if (strncmp(u,"http:",5)==0) u += 5;
	if (strncmp(u,"https:",6)==0) {u += 6; do_ssl=TRUE; port=443;}
	if (strncmp(u,"//",2)==0) u += 2;
	ncpy(host,u,BFSZ-1);
	s = strstr(host,"/"); 
	if (s!=NULL) ncpy(rest,s,BFSZ-1);
	if (s!=NULL) *s = 0;
	if (lib_strlen(proxy)==0) proxy = host; 

	sprintf(reason,"Get Worked");

	imsg("tcp: Open connection to (%s) and fetch (%s) %s ssl=%d, port=%d\n",proxy,rest,v11 ? "1.1" : "1.0" ,do_ssl,port);
	if (is_exit) return FALSE; 
	if (!tcp_open(c,proxy,port,NULL)) {
		sprintf(reason,"tcp: open (%.80s) %d failed (%.80s)\n",proxy,port,tcp_error(c,tmp));
		tcp_free(&c);
		return FALSE;;
	}

	if(do_ssl){
		imsg("starting ssl connection on tcp_download_file_do");
		if (!tcp_ssl_init_rhosts(c,FALSE,NULL,reason,host,NULL)) {
			imsg("ssl init failed %s\n",reason);
			tcp_free(&c);
			return FALSE;	
		}
	}
	if (!c->ssl_verified) {
		if (vcode) *vcode = c->ssl_code;		
	}
	strcpy(req,"");

	if (v11) req_printf(req,REQSZ,"GET %s HTTP/1.1\r\n",rest);
	else req_printf(req,REQSZ,"GET %s HTTP/1.0\r\n",rest);

	if (!req_printf(req,REQSZ,"Host: %s\r\n",host)) imsg("tcp_printf failed host request\n"); 
	if (v11) {
		if (!req_printf(req,REQSZ,"Connection: close\r\n")) imsg("tcp_printf failed connection\n"); 
	}
				//tcp_printf(c,"Connection: Keep-Alive\r\n"); 1.1
	if (user && lib_strlen(user)>0) {
		sprintf(bf,"%.100s:%.100s",user,pass);
		tcp_mime_encode_base64(bf,lib_strlen(bf)+1,bf2,BFSZ-1);
		req_printf(req,REQSZ,"Authorization: Basic %s\r\n",bf2);
	}
	imsg("tcp: Downloading If-Modified-Since: %s\n",lib_web_date2(modified,bf));
	

	if (do_gzip) {
		req_printf(req,REQSZ,"Accept-Encoding: gzip\r\n");		
	}

	if (cookies) if(lib_strlen(cookies)>0){
		req_printf(req,REQSZ,"Cookie: %s\r\n",cookies);		
	}

	if (modified!=0) req_printf(req,REQSZ,"If-Modified-Since: %s\r\n",lib_web_date2(modified,bf));
	if (!req_printf(req,REQSZ,"\r\n")) imsg("tcp_printf failed during request\n");

	imsg("tcp_request_sending: %s\n",req);

	if (tcp_write(c,req,lib_strlen(req))!=lib_strlen(req)) {
		char tmp[BFSZ];
		imsg("tcp_write of request failed %s\n",tcp_error(c,tmp));
	}
	tcp_nodelay(c,TRUE); // try and make it flush the buffer... 

	if (lib_strlen(localname)>0) {
		fnew=fopen(fname,binmode ? "wb" : "w");
		if(fnew==NULL) {
			sprintf(reason,"tcp: open (%.80s) failed (%.90s)\n",fname,strerror(errno));
			tcp_free(&c);
			return FALSE;;
		}
		gotfile = TRUE;
	}
	tcp_set_read_timeout(c,tout*1000);
//c->log_read	= TRUE;

	//	if (!web_printf_now(l,"HTTP/1.0 304 Not modified\r\n")) goto failed;
	while( ishead ) {
		if (!tcp_readline(c,bf,BFSZ-1)) {
			char tmp[BFSZ];
			sprintf(reason,"Error reading headers %.200s",tcp_error(c,tmp));
			tcp_free(&c);
			if (fnew!=NULL) fclose(fnew); fnew = NULL;
			return FALSE;
		}

		//imsg("tcp_fetch_header: %s\n",bf);
		if (strncmp(bf,"HTTP",4)==0) {
			if (atoi(bf+9)==304) *is_modified = FALSE;
			else if (atoi(bf+9)!=200) { /* Something wrong didn't get file we wanted*/
				ncpy(reason,bf+9,BFSZ-1);
				if (fnew!=NULL) fclose(fnew); fnew = NULL;
				if (atoi(reason)==302 && ntry<3) { // redirect, not used by surgemail, used by chris...
					for (;tcp_readline(c,bf,BFSZ-1);) {
						if (strncmp(bf,"Location: ",10)==0) {
							ncpy(url,bf+10,BFSZ-1);
							ntry++;
							imsg("Processing redirect to (%s)\n",url);
							
							tcp_free(&c); c = NULL;
							goto try_again;
						}
					}
				}
				tcp_free(&c); c = NULL;
				return FALSE;
			}
		}

		if (strncmp(bf, "Transfer-Encoding: chunked",26)==0) chunked = TRUE;

		if (strncmp(bf, "Content-Length: ",16)==0)  {
			downloadsize = atoi( bf+16 );
			gotsize = TRUE;
			
		}
		if (strncmp(bf, "Last-Modified: ",15)==0) {
			server_copy_last_modified_at = web_date_get_correct(bf+15);

		}
		if (lib_strlen(bf)==0) ishead=0;
	}
	if (chunked) { // added by chrisp, alternate http protocol used by some http servers.
		int chunk;
		imsg("Trying to read in 'chunked' mode... \n");

		for (;TRUE;) {
			if (!tcp_readline(c,bf,BFSZ-1)) break;
			chunk = hex_to_ulong(bf);
			imsg("tcp: Reading chunk of %d bytes (%s)\n",chunk,bf);
			if (chunk==0) break; 
			for (;chunk>0;) {
				int w = chunk; 
				if (w>BFSZ-1) w = BFSZ - 1;
				count = tcp_read_timed(c,bf,w, tout*1000);
				if (count<=0) goto failed;
				if (fnew!=NULL) sz+=(int)fwrite(bf,1,count,fnew);
				chunk -= count; 
			}
			count = tcp_read_timed(c,bf,2, tout*1000); // read trailing crlf 
			imsg("tcp: sz is now %d - read trailing crlf %d\n",sz,count);
		}
	} else {
	  while(count = tcp_read_timed(c,bf,BFSZ-1, tout*1000)) {


		if (count<0) {
			sprintf(reason,"Connection closed c=%d",count);
			imsg("tcp: Download failed %d/%d - connection closed %d %s %s\n",sz,downloadsize,count,tcp_error(c,tmp3),localname);

			tcp_free(&c);
			if (fnew!=NULL) fclose(fnew); fnew = NULL;
			return FALSE;
		}
		if (fnew!=NULL) sz+=(int)fwrite(bf,1,count,fnew); else sz+= count;
		if (is_exit) {
			sprintf(reason,"Server shutting down");
			imsg("tcp: Download failed - Server shutting down\n");
			tcp_free(&c);
			if (fnew!=NULL) fclose(fnew); fnew = NULL;
			return FALSE;
		}
		if (progress_fn) {
			//imsg("Calling progress function\n");
			if (!progress_fn(progress_param,sz,downloadsize)) {
				sprintf(reason,"Canceled");
				imsg("tcp: Download failed - Canceled\n");
				tcp_free(&c);
				if (fnew!=NULL) fclose(fnew); fnew = NULL;
				return FALSE;
			}
		}
		// added by chrisp so it will stop when it's got the whole file

		if (gotsize && (sz>=downloadsize)) break;
	  }

	}

failed:

	if (fnew!=NULL) fclose(fnew); fnew=NULL;
	tcp_free(&c);
	if (! *is_modified) {
		imsg("tcp: Not downloading. (%s) has not changed since it was last downloaded.\n",localname);
		return TRUE;
	}
	
	imsg("tcp: Downloaded %d/%d bytes %s\n",sz,downloadsize,localname);
	if (((gotsize && (downloadsize==sz)) || ((sz>0) && (downloadsize==0))) ) {
		if (!gotfile) {
			strcpy(reason,"all good");
			return TRUE;
		}
		if (remove(localname)!=0) imsg("Remove of (%.80s) failed %s\n",localname,strerror(errno));
		if (rename(fname,localname)!=0) {
			sprintf(reason,"Rename (%.80s) failed %.80s",localname,strerror(errno));
			return FALSE;
		}
		// This is very bad due to daylight saving it is not reliable and screws up the date.
		// to fix I will apply fiddle factor - make it 3 hours newer here. 
		if (server_copy_last_modified_at>0) {
			lib_set_mtime(localname,server_copy_last_modified_at + 60*60*25);
		}
	}else{
		sprintf(reason,"Error downloading %.80s %d want=%d (%d)\n",localname,sz,downloadsize,gotsize);
		if (downloadsize==0) if (sz>0) return TRUE;
		return FALSE;
	}
	return TRUE;
}



// Alternative function where we use a callback to process the data on the fly...
int tcp_download_cb(char *xurl, char *user_agent, char *prefix, int tout, char *reason, int (*cb)(int inhead, void *param, char *bf, int len),void *param)
{
	Chan *c=NULL;
	int chunked=FALSE;
	int sz=0;
	int downloadsize=0;
	int gotsize=0;
	int count=0;
	char tmp[BFSZ];
	int ishead = TRUE;
	char bf[BFSZ];
	int failed=FALSE;
	char host[BFSZ];
	char url[BFSZ];
	char fname[BFSZ];
	int moved=FALSE;
	int ntry=0;
	
	ncpy(url,xurl,BFSZ-1);
try_again:	
	failed = moved=FALSE;
	ncpy(prefix,url,BFSZ-1);
	if (!lib_url_split_host(url,host,fname)) return FALSE;
	if (strcmp(fname,"")==0) strcpy(fname,"/");

	
	tcp_new(&c);



	if (!tcp_open(c,host,80,NULL)) {
		sprintf(reason,"tcp: open (%.80s) failed (%.80s)\n",host,tcp_error(c,tmp));
		tcp_free(&c);
		return FALSE;;
	}

	tcp_printf(c,"GET %s HTTP/1.1\r\n",fname);
	imsg("tcp: GET %s HTTP/1.1\r\n",fname);
	tcp_printf(c,"Host: %s\r\n",host);
	tcp_printf(c,"User-Agent: %s\r\n",user_agent);
	tcp_printf(c,"Connection: close\r\n\r\n"); // Make sure it closes nicely
	tcp_set_read_timeout(c,tout*1000);
	while( ishead ) {
		if (!tcp_readline(c,bf,BFSZ-1)) {
			tcp_free(&c);
			imsg("Reading header failed\n",tcp_error(c,bf));
			ncpy(reason,"Error reading http headers",BFSZ-1);
			return FALSE;
		}
		imsg("Header (%s)\n",bf);
		ncat(bf,"\r\n",BFSZ-1);
		if (strncmp(bf,"HTTP",4)==0) {
			if (lib_strlen(bf)>9) if (atoi(bf+9)!=200) { /* Something wrong didn't get file we wanted*/
				failed = TRUE;
				if (atoi(bf+9)==302 || atoi(bf+9)==301) {
					moved=TRUE;
					imsg("Moved so looking for location header\n");
				}
			}
		}
		if (ntry<3) if (moved) if (strncmp(bf,"Location: ",10)==0) {
			char tmp3[BFSZ];
			char *s;
			ncpy(url,bf+10,BFSZ-1);
			s = strchr(url,'\r'); if (s!=NULL) *s = 0;
			s = strchr(url,'\n'); if (s!=NULL) *s = 0;
			lib_trim_spaces(url,tmp3); ncpy(url,tmp3,BFSZ-1);
			if (strncmpnc(url,"http",4)!=0) {
				ncpy(tmp3,"http://",BFSZ-1); 
				ncat(tmp3,host,BFSZ-1);
				ncat(tmp3,url,BFSZ-1);
				ncpy(url,tmp3,BFSZ-1);
			}
			ntry++;
			tcp_free(&c); c = NULL;
			goto try_again;
		}
		if (moved) continue; 
		cb(ishead,param,bf,lib_strlen(bf));

		if (strncmp(bf, "Transfer-Encoding: chunked",26)==0) chunked = TRUE;
		if (strncmp(bf, "Content-Length: ",16)==0)  {
			downloadsize = atoi( bf+16 );
			gotsize = TRUE;
		}
		if (lib_strlen(bf)==2) ishead=0;
	}
	if (moved) goto failed;
	if (failed) goto failed;
	if (chunked) { // added by chrisp, alternate http protocol used by some http servers.
		int chunk;
		for (;TRUE;) {
			if (!tcp_readline(c,bf,BFSZ-1)) break;
			chunk = hex_to_ulong(bf);
			if (chunk==0) break; 
			for (;chunk>0;) {
				int w = chunk; 
				if (w>BFSZ-1) w = BFSZ - 1;
				count = tcp_read_timed(c,bf,w, tout*1000);
				if (count<=0) goto failed;
				sz+=cb(FALSE,param,bf,count);
				chunk -= count; 
			}
			count = tcp_read_timed(c,bf,2, tout*1000); // read trailing crlf 
			imsg("tcp: sz is now %d - read trailing crlf %d\n",sz,count);
		}
	} else {
	  while(count = tcp_read_timed(c,bf,BFSZ-1, tout*1000)) {
		if (count<0) {
			sprintf(reason,"Connection closed c=%d",count);
			tcp_free(&c);
			return FALSE;
		}
		sz+=cb(FALSE,param,bf,count);
		if (gotsize && (sz>=downloadsize)) break;
	  }
	}
failed:
	tcp_free(&c);

	imsg("tcp: Downloaded %d/%d bytes\n",sz,downloadsize);
	return TRUE;
}


int tcp_post_do(char *type, char *xurl, char *proxy, char *localname, char *reason, int binmode, int (*progress_fn)(void *param, int so_far, int total),void *progress_param, char *content, int content_len, int *code, int (*auth_fn)(void *param, char*challenge, char*method, char*url, char*bf, int bfsz), void *auth_param )
/*
Use progress_fn for obtaining download progress.
If progress_fn returns false, then the download is aborted.
*/
{
	/* Use date on file so we only fetch file if it's been modified */
	// Returns 'true' if file is successfully fetched, or if file has not been modified since date specified */
	char host[BFSZ];
	char url[BFSZ];
	Chan *c=NULL;
	int sz=0;
	char rest[BFSZ];
	int count;
	char *u=url;
	time_t server_copy_last_modified_at=0;
	char tmp3[BFSZ];
	char bf[BFSZ];
	int gotsize = FALSE;
	char fname[BFSZ];
	int downloadsize = 0;
	char *s;
	char tmp[BFSZ];
	int ishead = TRUE;
	FILE *fnew;
	int ntry=0;
	int tout = download_tout;
	int chunked=FALSE;
	int do_ssl=FALSE;
	int port=80;
#define REQSZ 4000
	char req[REQSZ];
	int v11_local = TRUE;
	char authorisation[BFSZ], auth_response[BFSZ];
	int auth_retry=FALSE, auth_retried=FALSE;
	int code204_no_body=FALSE;

	ncpy(authorisation,"",BFSZ-1);
	*code = 0;
	ncpy(url,xurl,BFSZ-1);
	if (content==NULL) content="";
try_again:
	imsg("tcp: Downloading (%s) from (%s) via (%s), timeout %d seconds\n",localname,url,proxy,download_tout);
	tcp_new(&c);
	ncpy(rest,url,BFSZ-1);
	ncpy(fname,localname,BFSZ-1);
	ncat(fname,"_tmp",BFSZ-1);

	/* url = http://host.name:port/restofit */
	u = url;
	if (strncmp(u,"http:",5)==0) u += 5;
	if (strncmp(u,"https:",6)==0) {u += 6; do_ssl=TRUE; port=443;}
	if (strncmp(u,"//",2)==0) u += 2;
	ncpy(host,u,BFSZ-1);
	s = strstr(host,"/"); 
	if (s!=NULL) ncpy(rest,s,BFSZ-1);
	if (s!=NULL) *s = 0;
	if (lib_strlen(proxy)==0) proxy = host; 

	sprintf(reason,"%s Worked", type);

	imsg("tcp: Open connection to (%s) and fetch (%s) %s ssl=%d, port=%d\n",proxy,rest,v11_local ? "1.1" : "1.0" ,do_ssl,port);

	if (is_exit) return FALSE; 
	if (!tcp_open(c,proxy,port,NULL)) {
		sprintf(reason,"tcp: open (%.80s) %d failed (%.80s)\n",proxy,port,tcp_error(c,tmp));
		tcp_free(&c);
		return FALSE;;
	}

	if(do_ssl){
		imsg("starting ssl connection on tcp_download_file_do");

//		11/5/2021 Marijn I think this tweak is all that is needed to support SNI on outbound SSL connections
//		if (!tcp_ssl_init(c,FALSE,NULL,reason)) {
		if (!tcp_ssl_init_rhosts(c, FALSE, NULL, reason, proxy, NULL)) {
			imsg("ssl init failed %s\n",reason);
			tcp_free(&c);
			return FALSE;	
		}
	}
	strcpy(req,"");

	if (v11_local) req_printf(req,REQSZ,"%s %s HTTP/1.1\r\n",type,rest);
	else req_printf(req,REQSZ,"%s %s HTTP/1.0\r\n",rest);

	if (!req_printf(req,REQSZ,"Host: %s\r\n",host)) imsg("tcp_printf failed host request\n"); 
	if (v11_local) {
		if (!req_printf(req,REQSZ,"Connection: close\r\n")) imsg("tcp_printf failed connection\n"); 
	}
	//tcp_printf(c,"Connection: Keep-Alive\r\n"); 1.1
	if(auth_fn){
		auth_fn(auth_param,auth_response,type,url,authorisation,BFSZ);
		if(lib_strlen(authorisation)>0){
			req_printf(req,REQSZ,"Authorization: %s\r\n",authorisation);
		}
	}
/*
	if(digest_fn){
		ncpy(bf,rest,BFSZ-1);
		s=strchr(bf,'?');
		if(s) *s=0;
		digest_fn(digest_param,auth_challenge,type,bf,digest);
		if(lib_strlen(digest)>0){
			req_printf(req,REQSZ,"Authorization: Digest %s\r\n",digest);
		}
	}else if (user && lib_strlen(user)>0) {
		sprintf(bf,"%.100s:%.100s",user,pass);
		tcp_mime_encode_base64(bf,lib_strlen(bf)+1,bf2,BFSZ-1);
		req_printf(req,REQSZ,"Authorization: Basic %s\r\n",bf2);
	}
*/
	if (do_gzip) {
		req_printf(req,REQSZ,"Accept-Encoding: gzip\r\n");		
	}

	if(strcmp(type,"POST")==0){
		req_printf(req,REQSZ,"Content-Type: application/x-www-form-urlencoded\r\n");
	}else if(strcmp(type,"REPORT")==0){
		req_printf(req,REQSZ,"Content-Type: text/xml\r\n");
		req_printf(req,REQSZ,"Depth: 1\r\n");
	}else if(strcmp(type,"PROPFIND")==0){
		req_printf(req,REQSZ,"Depth: 1\r\n");
	}else if(strcmp(type,"MKCOL")==0){
		req_printf(req,REQSZ,"Content-Type: text/xml; charset=\"utf-8\"\r\n");
	}

	//Content-Type: text/calendar; charset=utf-8\r\n??
	req_printf(req,REQSZ,"Content-Length: %d\r\n",content_len);
	req_printf(req,REQSZ,"\r\n");

	imsg("tcp_request_sending: %s\n",req);

	if (tcp_write(c,req,lib_strlen(req))!=lib_strlen(req)) {
		char tmp[BFSZ];
		imsg("tcp_write of request failed %s\n",tcp_error(c,tmp));
	}
	if (tcp_write(c,content,content_len)!=content_len) {
		char tmp[BFSZ];
		imsg("tcp_write of request failed %s\n",tcp_error(c,tmp));
	}
	tcp_nodelay(c,TRUE); // try and make it flush the buffer... 

	fnew=fopen(fname,binmode ? "wb" : "w");
	if(fnew==NULL) {
		sprintf(reason,"tcp: open (%.80s) failed (%.90s)\n",fname,strerror(errno));
		tcp_free(&c);
		return FALSE;;
	}

	tcp_set_read_timeout(c,tout*1000);

	while( ishead ) {
		if (!tcp_readline(c,bf,BFSZ-1)) {
			char tmp[BFSZ];
			sprintf(reason,"Error reading headers %.200s",tcp_error(c,tmp));
			tcp_free(&c);
			if (fnew!=NULL) fclose(fnew); fnew = NULL;
			return FALSE;
		}
		//imsg("tcp_fetch_header: %s\n",bf);

		if (strncmp(bf,"HTTP",4)==0) {
			int res=atoi(bf+9);
			*code = res;

			if (res>200 && res<300){ 
				// all good
				if(res==204){
					code204_no_body = TRUE;
				}
			}else if (atoi(bf+9)==304){ 
			}else if (atoi(bf+9)==401 && !auth_retried) { /* Need the authentication challenge header */
				auth_retry=TRUE;
			}else if (atoi(bf+9)==302) { /* moved request is normal for caldav */
			}else if (atoi(bf+9)==207) { /* multistatus for caldav */
			}else if (atoi(bf+9)!=200) { /* Something wrong didn't get file we wanted*/
				ncpy(reason,bf+9,BFSZ-1);
				fclose(fnew); fnew = NULL;
				if (atoi(reason)==302 && ntry<3) { // redirect, not used by surgemail, used by chris...
					for (;tcp_readline(c,bf,BFSZ-1);) {
						if (strncmp(bf,"Location: ",10)==0) {
							ncpy(url,bf+10,BFSZ-1);
							ntry++;
							imsg("Processing redirect to (%s)\n",url);
							
							tcp_free(&c); c = NULL;
							goto try_again;
						}
					}
				}
				tcp_free(&c); c = NULL;
				return FALSE;
			}
		}

		if (strncmpnc(bf, "Transfer-Encoding: chunked",26)==0) chunked = TRUE;

		if (strncmpnc(bf, "Content-Length: ",16)==0)  {
			downloadsize = atoi( bf+16 );
			gotsize = TRUE;
		}
		if (strncmpnc(bf, "Last-Modified: ",15)==0) {
			server_copy_last_modified_at = web_date_get_correct(bf+15);
		}
		if (strncmpnc(bf, "WWW-Authenticate: ",18)==0) {
			ncpy(auth_response, bf+18, BFSZ-1);
		}
		if (lib_strlen(bf)==0) ishead=0;
	}

	if(auth_retry){
		tcp_free(&c); c = NULL;
		fclose(fnew); fnew = NULL;
		ishead = TRUE;
		downloadsize = 0;
		auth_retry = FALSE;
		auth_retried = TRUE;
		goto try_again;	
	}

	if (chunked) { // added by chrisp, alternate http protocol used by some http servers.
		int chunk;
		imsg("Trying to read in 'chunked' mode... \n");
		for (;TRUE;) {
			if (!tcp_readline(c,bf,BFSZ-1)) break;
			chunk = hex_to_ulong(bf);
			imsg("tcp: Reading chunk of %d bytes (%s)\n",chunk,bf);
			if (chunk==0) break; 
			for (;chunk>0;) {
				int w = chunk; 
				if (w>BFSZ-1) w = BFSZ - 1;
				count = tcp_read_timed(c,bf,w, tout*1000);
				if (count<=0) goto failed;
				sz+=(int)fwrite(bf,1,count,fnew);
				chunk -= count; 
			}
			count = tcp_read_timed(c,bf,2, tout*1000); // read trailing crlf 
			imsg("tcp: sz is now %d - read trailing crlf %d\n",sz,count);
		}
	} else {

	  if(gotsize && downloadsize==0) 
		goto done;	// Nothing more expected!

		//imsg("tcp: reading data in normal http mode %d bytes\n",downloadsize);
	  while(count = tcp_read_timed(c,bf,BFSZ-1, tout*1000)) {

		if (count<0) {
			sprintf(reason,"Connection closed c=%d",count);
			imsg("tcp: Download failed %d/%d - connection closed %d %s %s\n",sz,downloadsize,count,tcp_error(c,tmp3),localname);
			tcp_free(&c);
			fclose(fnew); fnew = NULL;
			return FALSE;
		}
		sz+=(int)fwrite(bf,1,count,fnew);
		if (is_exit) {
			sprintf(reason,"Server shutting down");
			imsg("tcp: Download failed - Server shutting down\n");
			tcp_free(&c);
			fclose(fnew); fnew = NULL;
			return FALSE;
		}
		if (progress_fn) {
			//imsg("Calling progress function\n");
			if (!progress_fn(progress_param,sz,downloadsize)) {
				sprintf(reason,"Canceled");
				imsg("tcp: Download failed - Canceled\n");
				tcp_free(&c);
				fclose(fnew); fnew = NULL;
				return FALSE;
			}
		}
		// added by chrisp so it will stop when it's got the whole file

		if (gotsize && (sz>=downloadsize)) break;
		if (!gotsize && sz>10000000) break;
	  }

	}
failed:
done:
	fclose(fnew); fnew=NULL;

	tcp_free(&c);

	imsg("tcp: Downloaded %d/%d bytes %s\n",sz,downloadsize,localname);
	if (((gotsize && (downloadsize==sz)) || ((sz>0) && (downloadsize==0))) || code204_no_body) {
		if (remove(localname)!=0) imsg("Remove of (%.80s) failed %s\n",localname,strerror(errno));
		if (rename(fname,localname)!=0) {
			sprintf(reason,"Rename (%.80s) failed %.80s",localname,strerror(errno));
			return FALSE;
		}
		// This is very bad due to daylight saving it is not reliable and screws up the date.
		// to fix I will apply fiddle factor - make it 3 hours newer here. 
		if (server_copy_last_modified_at>0) {
			lib_set_mtime(localname,server_copy_last_modified_at + 60*60*25);
		}
	}else{
		sprintf(reason,"Error downloading %.80s sz=%d want=%d (%d)\n",localname,sz,downloadsize,gotsize);
		if (downloadsize==0) if (sz>0) return TRUE;
		return FALSE;
	}
	return TRUE;
}





#endif
int tcp_fd_setsize(void)
{
	return FD_SETSIZE;
}

typedef struct BLIST {char *data; int isfree; int len; struct BLIST *next;} Blist;
static Blist *blist;
static int blist_tot;
int tcp_blist_tot(void)
{
	return blist_tot;
}
char *tcp_bf_get(int len)
{
	Blist *b=NULL;
	tcp_mutex_wait();
	for (b=blist;b!=NULL;b=b->next) {
		if (b->isfree && b->len==len) {
			b->isfree = FALSE;
			break;
		}
	}
	tcp_mutex_release();
	if (b==NULL) {
		b =  calloc(1,sizeof(Blist)); //d_callocid(DMID,1,sizeof(Blist),__FILE__,__LINE__);
		b->data = calloc(1,len); // d_callocid(DMID,1,len,__FILE__,__LINE__);
		b->len = len;
		b->isfree = FALSE;

		tcp_mutex_wait(); // Only hold mutex when we have to...
			blist_tot += len;
			b->next = blist;
			blist = b;
		tcp_mutex_release();
	}
	return b->data;
}
int tcp_bf_free(char **pbf)
{
	char *bf = *pbf;
	Blist *b=NULL;
	if (bf==NULL) return TRUE;
	tcp_mutex_wait();
	for (b=blist;b!=NULL;b=b->next) {
		if (b->data==bf) {
			b->isfree = TRUE;
			*pbf = NULL;
			break;
		}
	}
	tcp_mutex_release();
	if (!b) {
		emsg("tcp_bf_free failed %p blist %p\n",bf,blist);
		lib_abort();
	}
	return TRUE;
}
