/* My owned buffered IO routines, why?, because we don't trust the IO 
routines on most of the machines DNEWS runs on, we expect their 
efficiency would be sub optimal. */

/* To start with, we will have emulation routines using 
normal buffered IO, this allows easy testing and comparison
later */
#include <stdlib.h>
#include <stdio.h>
#include <errno.h>

#ifdef __MWERKS__
#include "Mac.h"
#endif
/*RHP no sys/ for mac thanks*/
#ifdef Mac
#include <stat.h>
#else
#include <sys/stat.h>
#endif


#ifdef WIN32
#include <windows.h>
#include <io.h>
#endif
#include <ctype.h>
#include <string.h>
#include <assert.h>
#ifdef VMS
#include <unixio.h>
#endif
#ifndef WIN32
#ifndef Mac
#ifndef NOUNISTD
#include <unistd.h>
#endif
#endif
#endif        
#ifdef OS2
#include <io.h>
#endif
#ifndef NO_FCNTL
#include <fcntl.h>
#else
/*RHP not for mac*/
#ifndef Mac
#include <file.h>
#endif
#endif

#include "mutex.h"
#include "dbf.h"
#include "emsg.h"
#include "lib.h"
#define IDXSIZE 20
void b_setmap(int h, int sz);
void b_iread(int h, char *bf, int pnt, int len);
void b_freeany(int h, int n);
int b_read(int h, char *bf, int pnt, int len) ;
void b_write(int h, char *bf, int pnt, int len);
void b_freeall(int h);
static int nblocks;

void fx_init(void);
int fx_open(const char *s, const char *mde);
int fx_read(int h,char *bf, int sz);
int fx_write(int h,const char *bf, int sz);
int fx_write_block(int h);
int fx_seek(int h,int offset, int origin);
int fx_tell(int h);
int fx_flush(int h);
int fx_close(int h);
int fx_eof(int h);
int fx_xlen(int h);

/*
dbf_read(n,bf)
        search for n in from-to ranges, split request so we are only
        asking for ranges exactly in 'szbf' parts. , search list to
        see if in cache, if not, then read, then copy data, and go
        to next bit.  Check if reading past endoffile, and only copy
        existing data.

dbf_write(n,bf);
        split request, for each bit find relevant buffer, or read it
        if we don't have it (if past end of file then just create 
        empty one), write it to buffer, set buffer dirty.
        (write oldest dirty buffer if necessar to get a new one)
dbf_close(n,bf)
        write all dirty buffers.
For the index, we will keep 100 buffers of 4K, 400K,  

typedef {int nbf, szbf; char **bf; int from,to; } Bfile;
bf = mymalloc(nbf*szbf);

bf[10];
*/
#undef mymalloc
#undef myfree
#define mymalloc(x) mymalloc2(x,M_DBF)
#define myfree(x) myfree2(x,M_DBF)

#define USE_FOPEN TRUE

#ifdef USE_FOPEN
static FILE *myh[100];
static char *myh_name[100];
static int myh_line[100];
static int myh_threadrunning[100];
static char *myh_bf[100];
static int myh_isbf[100];
static int myh_mapsz[100];

static int myh_isthread[100];
static int myh_pnt[100];
static int myh_len[100];
static int myh_eof[100];
static int myh_atend[100];
static int myh_nblocks[100];

#define BLOCK_SIZE (IDXSIZE*200)

static char **myh_b[100];
#ifdef WIN32
#define SET_BIN TRUE
#endif
#ifdef OS2
#define SET_BIN TRUE
#endif
#ifdef NETWARE
#define ADD_BIN TRUE
#endif
#ifdef Mac
#define ADD_BIN TRUE
#endif
#ifdef VMS
#define ADD_BIN TRUE
#endif

int dbf_nfree(void)
{
        int tot=0;
        int i;
        for (i=0; i<100; i++) {
                if (myh[i]==NULL) tot++;
        }
/*	if (tot<95) */
        for (i=0; i<100; i++) {
		if (myh[i]!=NULL) if (myh_name[i]!=NULL) {
			dmsg("dbf: handle [%d] = %s \n",i,myh_name[i]);
		}
        }
        return tot;
}
int mynexth(void)
{
        int i;
	tout_mutex_wait();
        for (i=0; i<100; i++) {
		if (!myh_threadrunning[i]) if (myh[i]==NULL) {
			tout_mutex_release();
			return i;
		}
        }
	tout_mutex_release();
        emsg("No free handles for my file routines \n");
        exit(1);
        return 0;
}
static int nmiss,ntot;
int dbf_read(int h, void *buffer, unsigned int count)
{
        int from,r;
        char bf[BLOCK_SIZE];
        int offset,got;
        int pnt;
        if (myh[h]==NULL) {emsg("myread handle null %d\n",h); return 0;}
        if (!myh_isbf[h]) {
		if (myh_isthread[h]) thout_wait(h); /* Wait until output threads are idle */
		errno = 0;
                return fread(buffer,1,count,myh[h]);
        } else {
                ntot++;
                pnt = myh_pnt[h];
                if (pnt>myh_len[h]) {
                        myh_eof[h] = TRUE;
                        return 0;
                }
                if (b_read(h,(char *)buffer,pnt,count)) { 
                        myh_pnt[h] += count;
                        return count;
                }
                nmiss++;
                from = pnt-(pnt % BLOCK_SIZE);
                fseek(myh[h],from,SEEK_SET);
		errno = 0;
                r = fread(bf,1,BLOCK_SIZE,myh[h]);
                if (r<0) return r;
                if (r<BLOCK_SIZE) {
                        offset = pnt % BLOCK_SIZE;
                        if (offset>r) {
                                myh_eof[h] = TRUE; 
                                return 0;
                        }
                        got = r - offset;
                        if (got> (int) count) got = count;
                        memcpy((char *)buffer,bf+offset,got);
                        imsg("Got part of record %d bytes \n",got);
                        myh_pnt[h] += got;
                        return got;
                }
                b_iread(h,bf,pnt-(pnt % BLOCK_SIZE),BLOCK_SIZE);
                if (!b_read(h,(char *)buffer,pnt,count)) {
                        emsg("Thats odd, It was just read in\n");
                } 
                myh_pnt[h] += count;
                return count;
        }                               
}                                        
#ifdef WIN32
#define write _write
#endif
int dbf_writex(int handle, const void *buffer, unsigned int count, char *fname, int line)
{
        int r;
        int h = handle;
        if ((myh[handle]==NULL) || (handle<0)) {emsg("mywrite handle null %d %s:%d\n",handle,fname,line); return 0;}
	if (myh_isthread[handle]) {
		if (thout_write(handle,buffer,count)) {
			return count;
		} /* else do normal write */
	}
        r = fwrite(buffer,1,count,myh[handle]); 
/*r = write(fileno(myh[handle]),buffer,count);*/
        if (myh_isbf[h]) {
                myh_pnt[h] += count;
                if (myh_pnt[h] > myh_len[h]) myh_len[h] = myh_pnt[h];
                b_write(handle,(char *) buffer,myh_pnt[h],count);
        }
        return r;
}
int dbf_tell(int handle)
{
	if (myh_isthread[handle]) thout_wait(handle); /* Wait until output thread is idle */
        return ftell(myh[handle]);
}
int dbf_seekx(int handle, long offset, int origin, char *fname, int line) /* true on success */
{
        int h = handle;
        if (handle<0) {emsg("Seek to closed file %s:%d [%d] %d\n",fname,line,handle,offset); return 0;}
        dmsg("dbf: dbf_seek[%d] %p off=%d origin=%d end=%d %s:%d\n",handle,myh[handle],offset,origin,myh_atend[h],fname,line);

	if (myh_isthread[handle]) {
		if (origin==SEEK_END) if (offset==0) return 0;
		thout_wait(handle); /* Wait until output threads are idle, although this never happens  */
        }
	if (origin==SEEK_SET) {
                myh_pnt[h] = offset;
		myh_atend[h] = FALSE;
        } else if (origin==SEEK_END) { 
                myh_pnt[h] = myh_len[h] + offset;
		if (offset==0) {
			if (myh_atend[h]) return TRUE;
			myh_atend[h] = TRUE;
		}
        } else if (origin==SEEK_CUR) {
                myh_pnt[h] += offset;
		myh_atend[h] = FALSE;
        }
        return (fseek(myh[handle],offset,origin)==0);
}
void dbf_bfsz(int h, int sz)
{
	if (myh_bf[h]!=NULL) myfree(myh_bf[h]);
        myh_bf[h] = mymalloc(sz);
#ifndef Mac
#ifndef VMS
        setvbuf(myh[h],myh_bf[h],_IOFBF,sz);
#endif
#endif
}
static int change_time;
int dbf_changed(int h)
{	
	int t;
	int r=FALSE;
	t = dbf_len(h);
	if (t<change_time) r = TRUE;
	change_time = r;
	return r;
}
int dbf_openx(const char *fname, char *xmde, int nbf, int szbf, int line)
{
        int f = mynexth();
        char mde[BFSZ];
#ifdef SET_BIN
        int oldmode;
        oldmode = _fmode;
        _fmode = _O_BINARY;
#endif
#ifdef ADD_BIN     
        strcpy(mde,xmde); strcat(mde,"b");
#else
        strcpy(mde,xmde);
#endif
        if (is_readonly()) {
                if ((strchr(mde,'a')!=NULL) || (strchr(mde,'w')!=NULL)) {
                        emsg("slave: Attempt to open %s, %s\n",fname,mde);
                        return -1;
                }
        }
	myh_isthread[f] = FALSE;
        myh_mapsz[f] = 0;
        myh_isbf[f] = FALSE;
        myh_len[f] = 0;
        myh_pnt[f] = 0;
        myh_atend[f] = FALSE;
        myh_eof[f] = FALSE;
        myh[f] = myfopen(fname,mde);

        if (myh_name[f]==NULL) myh_name[f] = mymalloc(200);
        ncpy(myh_name[f],(char *) fname,100);
	myh_line[f] = line;
#ifdef SET_BIN
        _fmode = oldmode;
#endif
        dmsg("dbf: db.c:%d dbf_open[%d]{%s} (%s) %p\n",line,f,fname,xmde,myh[f]);
        if (myh[f]==NULL) return -1;
        myh_len[f] = dbf_len(f);
        if (strstr(mde,"a")!=NULL) {
                myh_pnt[f] = myh_len[f];
		dbf_bfsz(f,100000); /* Writing to a bucket file so a bit bigger is a good idea */
        } else {
		dbf_bfsz(f,32000);
	}
        return f;
}
static int bfh;

char *dbf_status(void)
{
        static char bf[400];
        lprintf(bf,400-1,"DBI Misses %d/%d, blocks %d=%dk",nmiss,ntot,nblocks,nblocks*BLOCK_SIZE/1000);
        return bf;      
}
void dbf_setbf(int h, int n)
{
        myh_isbf[h] = TRUE;
        myh_nblocks[h] = n;
        bfh = h;
}
int dbf_flush(int h)
{
        if (myh[h]==NULL) return TRUE;
	if (myh_isthread[h]) thout_wait(h); /* Wait until output thread is idle */
        if (fflush(myh[h])==0) return TRUE;
        return FALSE;
}

int dbf_closex(int handle, int line)
{
        int r;

        if (myh[handle]==NULL) {emsg("myclose handle null %d line=%d\n",handle,line); return 0;}
        dmsg("dbf: dbf_close[%d] (%s) db.c:%d\n",handle,myh_name[handle],line);
	if (myh_isthread[handle]) thout_close(handle); /* Wait until output thread is idle */
        r = fclosep(myh[handle]);
                if (myh_bf[handle]!=NULL) myfree(myh_bf[handle]);
                myh_bf[handle] = NULL;
        myh[handle] = NULL;
        if (myh_isbf[handle]) b_freeall(handle);
        if (r==0) return TRUE;
        return FALSE;
}
#ifdef WIN32
#define fileno _fileno
#endif
int dbf_len(int handle)
{
        if (handle<0) {
                emsg("dbf_len called with handle -1\n");
                return 0;
        }
	if (myh_isthread[handle]) thout_wait(handle); /* Wait until output thread is idle */
        return file_length(fileno(myh[handle]));
}
int dbf_eof(int h)
{
        if (myh_isbf[h]) return myh_eof[h];
	if (myh_isthread[h]) thout_wait(h); /* Wait until output thread is idle */
        return feof(myh[h]);
}

int dbf_usethread(int h)
{
	if (thout_init(h)) {
		myh_isthread[h] = TRUE;
	}
	return TRUE;
}
/*
Bufferred IO operations 

16MB of buffers, 4k * 4k

Read operations must read the whole block if it exists, and then 
cache it, if the whole block can't be read, then don't cache. 

Write operations are ignored unless the whole block is already in memory
*/
void b_setmap(int h, int sz)
{
        char **m;
        if (sz==0) sz = 1000;
        if (myh_mapsz[h]==0) {
                myh_b[h] = mymalloc(sz * sizeof(char *));
                memset(myh_b[h],0,sz * sizeof(char *));
                myh_mapsz[h] = sz;      
                emsg("Allocated map of %d items\n",sz);
                return; 
        }       
        emsg("reallocated map of %d items\n",sz);       
        m = mymalloc(sz * sizeof(char *));
        memset(m,0,sz*sizeof(char *));
        memcpy(m,myh_b[h],myh_mapsz[h]*sizeof(char *));
        myfree(myh_b[h]);
        myh_b[h] = m;
        myh_mapsz[h] = sz;      
}
void b_iread(int h, char *bf, int pnt, int len)
{
        int n;
        static char **block;

        if (!myh_isbf[h]) return ;
        n = pnt / BLOCK_SIZE;
        if ((block==NULL) || (myh_mapsz[h]<=n)) {
                b_setmap(h,n+10);                       
        }
        block = myh_b[h];
        if (block[n]==NULL) {
                if (len!=BLOCK_SIZE) return; /* Only cache full blocks */
                block[n] = mymalloc(BLOCK_SIZE);
                emsg("Block allocated %d %x\n",n,block[n]);
                nblocks++;
                if (nblocks>myh_nblocks[h]) {
                        b_freeany(h,n);  /* Free any other block :-) */
                }
        }
        pnt = pnt % BLOCK_SIZE;
        memcpy(block[n]+pnt,bf,len);
        return ;
}
void b_freeall(int h)
{
        int i;
        char **block;

        block = myh_b[h];
        if (block==NULL) return ;
        for (i=0 ; i<myh_mapsz[h]; i++) {
                if (block[i]!=NULL) {
                        nblocks--;    
                        myfree(block[i]);
                        block[i] = NULL;
                }
        }
        myfree(block);
        myh_b[h] = NULL;
}
void b_freeany(int h, int n)
{
        static int i;
        int isagain = FALSE;
        char **block;
        block = myh_b[h];
again:
        for (; i<myh_mapsz[h]; i++) {
                if (i!=n) if (block[i]!=NULL) {
                        nblocks--;    
                        myfree(block[i]);
                        block[i] = NULL;
                        return; 
                }
        }
        i = 0;
        if (isagain) return;
        isagain = TRUE;
        goto again;
}
int b_read(int h, char *bf, int pnt, int len) 
{
        char **block;
        int n;
        if (!myh_isbf[h]) return FALSE; /* It wasn't in the cache */
        n = pnt / BLOCK_SIZE;
        if (myh_mapsz[h]<=n) {
                b_setmap(h,n+10);                       
        }
        block = myh_b[h];
        if (block==NULL) return FALSE; /* Nothing is buffered yet */
        if (block[n]==NULL) return FALSE;
        pnt = pnt % BLOCK_SIZE;
        if ((pnt + len - 1)>BLOCK_SIZE) return FALSE; /* Straddles block */ 

        memcpy(bf,block[n] + pnt,len);
        return TRUE;
}
void b_write(int h, char *bf, int pnt, int len)
{
        b_iread(h,bf,pnt,len);
}
#endif

/*------------------------------------------------------*/
/*  Map entire index into memory                        */
/*------------------------------------------------------*/
static int nomap = TRUE;
static int isfx = TRUE;
#define MAXIBF 4000
static char *ibf[MAXIBF];  /* 400mb index */
static int ibf_changed[MAXIBF];  
static int ibf_f= -1;
static int ibf_len,ibf_pnt;
static int ibfsz = 100000;  /* must be devisable by index items */
void dbi_setmap(int x)
{
        nomap = !x;
}
void dbi_setfx(int x)
{
        if (x>0) isfx = TRUE;
        if (x<0) isfx = FALSE;
}
int dbi_read(int handle, void *buffer, unsigned int xcount)
{
        int b,off;
        int count = (int) xcount;
        if (isfx) return fx_read(handle,buffer,xcount); 
        if (nomap) return dbf_read(handle,buffer,xcount);       
        b = ibf_pnt / ibfsz;
        off = ibf_pnt % ibfsz;
        if (ibf[b]==NULL) return 0;
        if ((ibf_pnt + count)> ibf_len) count = ibf_len - ibf_pnt;
        if (count<0) count = 0;
        memcpy(buffer,ibf[b]+off,count);
        ibf_pnt += count;
        if (count<(int) xcount) emsg("End of file at %d %d\n",ibf_pnt,count);
        return count;
}
int dbi_changed(int handle)
{
	if (isfx) return fx_changedf(handle);
	return dbf_changed(handle);
}
int dbi_write(int handle, const void *buffer, unsigned int count)
{
        int b,off;
        if (isfx) return fx_write(handle,buffer,count);
        if (nomap) return dbf_write(handle,buffer,count);
        b = ibf_pnt / ibfsz;
        off = ibf_pnt % ibfsz;
        if (ibf[b] == NULL) ibf[b] = mymalloc(ibfsz);
        memcpy(ibf[b]+off,buffer,count);
        ibf_pnt += count;
        ibf_changed[b] = TRUE;
        if (ibf_len<ibf_pnt) ibf_len = ibf_pnt;
        return count;
}
int dbi_seek(int handle, long offset, int origin)
{
        if (isfx) return fx_seek(handle,offset,origin);
        if (nomap) return dbf_seek(handle,offset,origin);
        if (origin==SEEK_SET) {
                ibf_pnt = offset;
        } else if (origin==SEEK_CUR) {
                ibf_pnt += origin;
        } else if (origin==SEEK_END) {
                ibf_pnt = offset + ibf_len;
        }
        if (ibf_pnt>(ibf_len)) {
                emsg("dbi_seek, attempt to seek past end of file %d %d\n",ibf_pnt,ibf_len);
                ibf_pnt = ibf_len;
        }
        return TRUE;
}
int dbi_open(const char *fname, char *mde, int nbf, int szbf)
{
        int r=0,i;
        if (isfx) return fx_open(fname,mde);
        if (nomap) {
                r = dbf_open(fname,mde,nbf,szbf);
                return r;
        }
	change_time = 0;
        emsg("dbi buffer, open\n");
        ibf_f = dbf_open(fname,mde,nbf,szbf);
        if (ibf_f<0) return -1;
        emsg("dbi buffer, reading into mem buffer %s %s\n",fname,mde);
        for (i=0;(!dbf_eof(ibf_f)) && (i<MAXIBF);i++) {
                if (ibf[i] == NULL) ibf[i] = mymalloc(ibfsz);
                ibf_changed[i] = FALSE;
                r = dbf_read(ibf_f,ibf[i],ibfsz);
                dmsg("dbi: read %d bytes\n",r);
                if (r<ibfsz) break;
        }
        ibf_len = i * ibfsz + r;
        ibf_pnt = 0;
        emsg("Open done\n");
        return 50;
}
int dbi_close(int handle)
{
        if (isfx) return fx_close(handle);
        if (nomap) return dbf_close(handle);
        emsg("dbi_close %d\n",handle);
        for (;!dbi_doflush(handle);) ;  /* flush until done */
        dbf_close(ibf_f);
        return TRUE;
}
int dbi_len(int handle)
{
        if (isfx) return fx_xlen(handle);
        if (nomap) return dbf_len(handle);
        return ibf_len;
}
int dbi_eof(int handle)
{
        if (isfx) return fx_eof(handle);
        if (nomap) return dbf_eof(handle);
        if (ibf_pnt>ibf_len) return TRUE;
        return FALSE;
}
int dbi_tell(int handle)
{
        if (isfx) return fx_tell(handle);
        if (nomap) return dbf_tell(handle);
        return ibf_pnt;
}
int dbi_flush(int h)
{
        if (isfx) return fx_flush(h);   
        if (nomap) return dbf_flush(h); 
        dbi_doflush(h);
        return TRUE;
}
int dbi_doflush(int h)
{
        int b;
        int r;
        int fin = TRUE;
        int ndone = 0;
        static int upto = 0;
        int n = (ibf_len / ibfsz) + 1;
        
again:;
          for (b=upto; (b<n) && (ndone<10) ; b++) {
                if (!ibf_changed[b]) continue;
                ibf_changed[b] = FALSE;
                ndone++;
                fin = FALSE;
                dbf_seek(ibf_f,b*ibfsz,SEEK_SET);
                r = ibfsz;
                if (b==n-1) r = ibf_len % ibfsz; 
                dmsg("dbi: Writing index block [%d], size %d\n",b,r);
                dbf_write(ibf_f,ibf[b],r);
        }
        if (upto>0) if (ndone<10) {upto = 0; goto again;}
        if (b==n) upto = 0;
        upto = b;
	/*dbf_flush(ibf_f); I think I added this so I think it's safe to remove, not sure */
        return fin;
}
/*------------------------------------------------------*/
/* File bufferring because builtin stuff is crap        */
#define MAXFX 10000
#define FXSZ 8000
int maxfx = 10;
static int fx_f = -1;
static char *fx_blocks[MAXFX];  /* 10 * 8K = 80K,  8mb , now the limit is 80mb, index_nbuffer 9000 */ 
static char fx_changed[MAXFX];
static int fx_age[MAXFX];
static int fx_b[MAXFX]; /* the block currently stored here */
static int fx_readn=1;  /* Number of blocks to read at a time */
static int fx_len,fx_pnt;
int fx_setmax(int x)
{
	if (x<=0) {
		x = 10;
		if (atoi(config_get(C_DISKSPACE))>3000) x = 100;
		if (atoi(config_get(C_DISKSPACE))>7000) x = 300;
		if (atoi(config_get(C_DISKSPACE2))>3000) x = 300;
		if (atoi(config_get(C_DISKSPACE3))>3000) x = 500;
		if (atoi(config_get(C_DISKSPACE))>20000) x = 500;
		if (atoi(config_get(C_DISKSPACE4))>3000) x = 500;
		imsg("Auto setting index_nbuffer %d\n",x);		
	}
	if (x>=MAXFX) x = MAXFX-1;
	maxfx = x;
	return TRUE;
}
static int fx_totalx;
void fx_init(void)
{
        static int done;
        int i;
        if (done) return;
        done = TRUE;
        for (i=0; i<maxfx; i++) {
                fx_blocks[i] = mymalloc(FXSZ);
				memset(fx_blocks[i],0,FXSZ);
        }
		fx_totalx = FXSZ * maxfx;
}
int fx_total(void)
{
	return fx_totalx;
}
int fx_open(const char *s, const char *mde)
{
        int i;
        fx_init();
        for (i=0; i<maxfx; i++) {
                fx_changed[i] = FALSE;
                fx_b[i] = -1;
        }
        fx_f = dbf_open(s,(char *) mde,1,1);
        if (fx_f<0) return -1;
        fx_len = dbf_len(fx_f);
        fx_pnt = 0;
        return 70;
}
int fx_read(int xhand,char *bf, int sz)
{
        int b = fx_pnt / FXSZ;
        int h = b % maxfx;
        int off = fx_pnt % FXSZ;
        int r;
	static int isreadonly;
	static int done;
	if (!done) {
		isreadonly = is_readonly();
		done = TRUE;
	}
        if (isreadonly) {
                if ((lib_gmtimex() - fx_age[h])>20) {
                        fx_len = dbf_len(fx_f);                        
			goto ragain;
                }
        }
        if (fx_b[h]!=b) {
                if (fx_changed[h]) fx_write_block(h);
ragain:         dbf_seek(fx_f,b*FXSZ,SEEK_SET);
                dmsg("fx:  Reading block [%d] %d\n",h,b);
       		memset(fx_blocks[h],0,FXSZ);
		r = dbf_read(fx_f,fx_blocks[h],FXSZ);
                if (is_readonly()) fx_age[h] = lib_gmtimex();
                fx_b[h] = b;
                fx_changed[h] = FALSE;
        }
        if ((fx_pnt + sz)> fx_len) {
                sz = fx_len - fx_pnt;
                if (sz<0) sz = 0;
                dmsg("fx: End of file %d %d %d\n",sz,fx_len,fx_pnt);
        }
        memcpy(bf,fx_blocks[h]+off,sz);
        fx_pnt += sz;
        return sz;
}
int fx_write(int xhand,const char *bf, int sz)
{
        int b = fx_pnt / FXSZ;
        int h = b % maxfx;
        int off = fx_pnt % FXSZ;
        int r;
        if (fx_b[h]!=b) {
                if (fx_changed[h]) fx_write_block(h);
                dbf_seek(fx_f,b*FXSZ,SEEK_SET);
                dmsg("fx: write:  Reading block [%d] %d\n",h,b);
                r = dbf_read(fx_f,fx_blocks[h],FXSZ);

                fx_b[h] = b;
        }
        fx_changed[h] = TRUE;
        memcpy(fx_blocks[h]+off,bf,sz);
        fx_pnt += sz;
        if (fx_len<fx_pnt) fx_len = fx_pnt;
        return sz;
}
int fx_write_block(int h)
{
        int r,sz;
        dbf_seek(fx_f,fx_b[h]*FXSZ,SEEK_SET);
        sz = FXSZ;
        if (fx_len/FXSZ == fx_b[h]) {
                sz = fx_len % FXSZ;
        }
        dmsg("fx: write_block:  Writing block [%d] %d %d bytes \n",h,fx_b[h],sz);
        r = dbf_write(fx_f,fx_blocks[h],sz);
        fx_changed[h] = FALSE;
        return TRUE;
}
int fx_seek(int xhand, int offset, int origin)
{
        if (origin==SEEK_SET) {
                fx_pnt = offset;
        } else if (origin==SEEK_CUR) {
                fx_pnt += origin;
        } else if (origin==SEEK_END) {
                fx_pnt = offset + fx_len;
        }
        return TRUE;
}
int fx_tell(int xhand)
{
        return fx_pnt;
}
int fx_flush(int xhand)
{
        int i;
        for (i=0; i<maxfx; i++) {
                if (fx_b[i]>=0) if (fx_changed[i]) fx_write_block(i);
                fx_changed[i] = FALSE;
        }
        if (fx_f>=0) dbf_flush(fx_f);
        return TRUE;
}
int fx_changedf(int handle)
{
	return dbf_changed(fx_f);
}
int fx_close(int xhand)
{
        fx_flush(xhand);        
        if (fx_f>=0) dbf_close(fx_f);
        fx_f = -1;
        return TRUE;
}
int fx_eof(int xhand)
{
        if (fx_pnt>fx_len) return TRUE;
        return FALSE;
}
int fx_xlen(int xhand)
{
        return fx_len;
}
/*------------------------------------------------------*/

/* test caching routines, not real*/ 
/*
static char *dcache_fname[100];
static int dcache_i[100];
static int dcache_open[100];
static int nd;
int dbf_openx(const char *fname, char *mode, int a, int b, int line)
{
	int i;
	for (i=0; i<nd; i++) {
		if (strcmp(dcache_fname[i],fname)==0) {
			imsg("re-using handle %d\n",i);
			dcache_open[i] = TRUE;
			return dcache_i[i];
		}
	}
	if (nd<80) {
		i = nd++;
	} else {
		for (i=0; i<nd; i++) {
			if (!dcache_open[i]) {
				dbf_closex_raw(i,line);
				imsg("Had to close handle %d\n",i);
				break;
			}
		}
	}

	dcache_i[i] = dbf_openx_raw(fname,mode,a,b,line);
	dcache_fname[i] = strdup(fname);
	dcache_open[i] = TRUE;
	return dcache_i[i];
}
int dbf_closex(int handle, int line)
{
	int i;
	for (i=0; i<nd;i++) {
		if (dcache_i[i] == handle) dcache_open[i] = FALSE;	
	}
	return TRUE;
}
*/
static int nthread;
static int th_h[100];
static int th_busy[100];
static int th_error[100];
static int th_close[100];
static int th_len1[100];
static int th_len2[100];
static char *th_bf1[100];
static char *th_bf2[100];
static int th_h[100];
static int maxlen;
int thout_switch(int h);
int thout_wait_idle(int h);
int thout_n(void)
{
	return nthread;
}
int thout_init(int h)
{
#ifdef DOTHREADS
	Thread *t;
	/* Create the thread*/
	tout_mutex_wait();
	th_h[h] = h;
	imsg("thout: Output init h=%d %s\n",h,myh_name[h]);
	
	if (maxlen==0) maxlen = conf_item_max()+102000;
	th_bf1[h] = mymalloc(maxlen+100);
	th_bf2[h] = mymalloc(maxlen+100);
	th_error[h] = 0;

	myh_threadrunning[h] = TRUE;
	nthread++;
	tout_mutex_release();
	t = th_new(thout_main,&th_h[h]);
	if (t==NULL) {
		emsg("Thread output creation failed %s\n",strerror(errno));
		return FALSE;
	}
	th_free(&t);
	imsg("thout: Output thread created %d byte buffer\n",maxlen);
#endif
	return TRUE;
}
int thout_write(int h, const char *bf, int len)
{
#ifdef DOTHREADS
//	tmsg("thout: write %d %d bytes\n",h,len);
	if (len>=maxlen) {
		thout_wait(h);
		return FALSE; /* Too big to write so do it the old fashioined way  */
	}
	/* If too much waiting send it first */
	if (len+th_len1[h]>maxlen) {
		thout_switch(h); /* Switch to new buffer, wait if necessary */ 
	} 
	tout_mutex_wait();
	memcpy(th_bf1[h]+th_len1[h],bf,len);
	th_len1[h] += len;
	tout_mutex_release();
	if (th_error[h]>0) {
		emsg("thout: error writing item (%s)\n",strerror(th_error[h]));
	}
#endif
	return TRUE;
}
int thout_switch(int h)
{
#ifdef DOTHREADS
	char *s;


	thout_wait_idle(h);
	tout_mutex_wait();
	s = th_bf1[h];
	th_bf1[h] = th_bf2[h];
	th_bf2[h] = s;
	th_len2[h] = th_len1[h];
	th_len1[h] = 0;
/*	tmsg("thout: switch %d len now %d\n",h,th_len2[h]);*/
	th_busy[h] = TRUE;
	tout_mutex_release();

#endif
	return TRUE;
}

int thout_wait_idle(int h)
{
#ifdef DOTHREADS
	int k;
	for (k=0;TRUE;k++) {
		if (!th_busy[h]) break;
		lib_msleep(1);
//		if (k%1000==3) tmsg("thout: wait idle %d\n",h);
	}
	return TRUE;
#endif
}
int thout_main(void *arg)
{
#ifdef DOTHREADS
	int h = *(int *) arg;;
	/* Wait for data ready flag, then start writing */
	/* After writing, reset */

	for (;TRUE;) {
		if (th_busy[h]) thout_dowrite(h);
		tout_mutex_wait();
		if (!th_busy[h]) if (th_close[h]) {
			break;
		}
		tout_mutex_release();
		lib_msleep(1);
	}
	myh_threadrunning[h] = FALSE;
	tout_mutex_release();
#endif
	return 0;
}

int thout_dowrite(int h)
{
#ifdef DOTHREADS
	int r;
	fseek(myh[h],0,SEEK_END);
	if (th_len2[h]>0) {
/*imsg("Writing to file (%d) b ytes\n",th_len2[h]);*/
	        r = fwrite(th_bf2[h],1,th_len2[h],myh[h]); 
		if (r!=th_len2[h]) {
			th_error[h] = errno;
		}
	} 
	mutex_wait(M_OUT);
	th_len2[h] = 0;
	th_busy[h] = FALSE;
	mutex_release(M_OUT);
#endif
	return TRUE;
}

int thout_close(int h)
{
	/* Wait for writing, then set 'close' flag, wait until closed */
#ifdef DOTHREADS
	imsg("thout close %d\n",h);
	thout_wait(h);
	tout_mutex_wait();
	th_close[h] = TRUE;
	tout_mutex_release();

	for (;TRUE;) { /* Wait until it's really finished */
		tout_mutex_wait();
		if (!myh_threadrunning[h]) break;
		tout_mutex_release();
		lib_msleep(1);
	}
	tout_mutex_release();
	
	tout_mutex_wait();
	th_close[h] = FALSE;
	myfree(th_bf1[h]); th_bf1[h] = NULL;
	myfree(th_bf2[h]); th_bf2[h] = NULL;
	th_len1[h] = 0;
	th_len2[h] = 0;
	myh_isthread[h] = FALSE;
	nthread--;
	tout_mutex_release();
	imsg("thout: Output closed h=%d\n",h);
#endif
	return TRUE;	
}
int thout_wait(int h)
{
	/* Wait until writing finished, then return */
#ifdef DOTHREADS
	if (!myh_isthread[h]) return TRUE;
	thout_wait_idle(h);
	thout_switch(h);
	thout_wait_idle(h);	
#endif
	return TRUE;
}
