/*
Todo:
	tellnews pause Reason
	tellnews unpause
	Why do I get duplicate id's when converting?
	bug, any file errors when we send in articles?
	bug on win95, it said 0 free disk space ? 
	check path of open, to see if 'update' could close something that should be open
Tests
  Cancel item
  Reading item that is not converted
  Crashing upgrade and restarting it
  Stopping upgrade and continuing
  Streaming writing wont' work because of file length problem,
  	test for significant speed difference before fixing.

*/
/* Copyright (C) 1995 NetWin World, New Zealand, All rights reserved. */

#include <stdlib.h>

#include <stdio.h>
/*RHP stuff for mac*/
#ifdef __MWERKS__
#include "Mac.h"
#endif

#ifdef WIN32
#include <io.h>
#endif
#include <ctype.h>
#include <string.h>
#include <assert.h>
#ifdef VMS
#include <unixio.h>
#endif


#ifndef WIN32
/*RHP not for mac thanks*/
#ifndef Mac
#ifndef NOUNISTD
#include <unistd.h>
#endif
#endif
#endif        


#ifdef OS2
#include <io.h>
#endif

#ifndef NO_FCNTL
#include <fcntl.h>
#else
/*RHP not for mac*/
#ifndef Mac
#include <file.h>
#endif
#endif
#include "emsg.h"
#include "sary.h"
#include "str.h"
#include "lib.h"
#include "group.h"
#include "hist.h"
#include "suck.h"
#include "feed.h"
#include "expire.h"
#include "control.h"
#include "item.h"
#include "db.h"
#include "pipelib.h"

#ifdef WIN32
#define USE_MYPIPES
#endif

#ifdef isunix
#define USE_MYPIPES
#endif

/*#ifdef WIN32
#define USE_FSTREAMS TRUE
#endif
*/

/*
#ifndef USE_MYPIPES
#define myread read
#define mywrite write
#define mylseek lseek
#define myopen xopen
#define myclose xclose
#endif
*/

/*
States we can be in:
        Appending an article to a file block.
        Re-writing a file block.

Main operations:
        Normal adding - reading  append.
        Expire                   rewrite individual blocks 
        Purge Lost               same
        Rebuild History          same   
        Update                   (purge lost will do this)

can't allow write to aold formatin new format!!!

*/


enum {DB_CLOSED,DB_APPEND, DB_READ, DB_NEW};
static char *mode_name[]={"DB_CLOSED","DB_APPEND","DB_READ","DB_NEW"};
#define MAX_OPEN 1000
#ifdef WIN32
static int max_open=10;     /* Could probably work upto 40 or so. */
#else
#ifdef OS2
static int max_open=3;
#else
static int max_open=10;         /* Could probably be 100 */
#endif
#endif
static int dbgrp[MAX_OPEN];
static int dbfg[MAX_OPEN];
static int dbf[MAX_OPEN];
static int dbmode[MAX_OPEN];
static int dbsent[MAX_OPEN];
static int dblen[MAX_OPEN];
static int ndb_open;
#define HEAD_SIZE 102
#define dbg if (1==2)

/* Each pointer is 1.2K   1MB for 1000, 4mb for 4000 */
#define MAX_CACHE 20000
static int max_cache = 200;
Hrec *head_cache[MAX_CACHE];
static int head_fg[MAX_CACHE];
static int head_grp[MAX_CACHE];
static int head_changed[MAX_CACHE];
static int nchanged,new_grp,new_fg;

#define SIZEOFHEAD (sizeof(Hrec)*HEAD_SIZE)
/*
100.idx
        [0-99] = Items
        [100].pnt = version (1),  .len = Compressed.  
100.itm
        ~(len):article^
        ~(len):article^
        If loose sync, search for  ^~nnn:  
*/
int db_writes(int p, char *s);

int db_file_length(char *fname);
static int lastgrp= -1, lastfg, lastmode, lastp;
void db_exit(void)
{
 	pi_end();
}
void db_init(void)
{
        int i;
        static int done=FALSE;
	static int max_os2;
        if (done) return;
        done = TRUE;
#ifdef OS2
	max_os2 = _grow_handles(40);  /* ActuHdl contains number of handles actually disponibles */
 	if (max_os2>25) max_open = 10; /* Grow file handles worked */
	imsg("OS2 file handles increased to %d\n",max_os2);
#endif
        i = atoi(config_get(C_DB_MAX));
        if (i>0) max_open = i;
        i = atoi(config_get(C_DB_INDEX));
        if (i>0) max_cache = i;
#ifdef USE_MYPIPES
	if (max_open>20) max_open=20;
	pi_init(atoi(config_get(C_NPIPES)));
#endif
        imsg("db: db_init ok cache files %d indexes %d %s\n",max_open,max_cache,pi_status());
        for (i=0; i<MAX_CACHE; i++) head_grp[i] = -1;
        for (i=0; i<MAX_OPEN; i++) { dbgrp[i] = -1; dbf[i] = -1;}
        new_grp = -1;
        lastgrp = -1;

}
void db_convert3(int grp, int num)
{
	char idxname[BFSZ];
	char gname[BFSZ];
	int p,tlen,hlen;
	int fg,fi;
	char *s;
	char bf[BFSZ];
	int offset,xeof,i,n; 
	int g;
 	cvt_numfile2(num,&fg, &fi);
 
 	/* delete .idx file */
        strcpy(idxname,sys_iname(grp,fg));

	remove(idxname);
 	/* read .itm file */
 	p = db_open(grp,fg,DB_READ);  if (p<0)  {dmsg("db: no db_open\n"); return ;}
	offset = 0;
	xeof = FALSE;
	for (i=0;i<100 && !xeof;i++) {	
		n = myread(dbf[p],bf,200);   bf[200]=0; /* increase after testing small */
		if (n<10) xeof = TRUE;
		if (n<=0) break;
		/* ~1213:2100:From: Brett Meehan*/

		if (bf[0]!='~') {
			bf[50] = 0;
		 	emsg("Item out of sync {%s}\n",bf);
			break;
		}
		tlen = atoi(bf+1);  /* Probably need correction here */
		s = strchr(bf,':');
		if (s==NULL) {emsg("Lost sync\n"); break;}
		s++;
		num = atoi(s);
		s = strchr(s,':');
		if (s==NULL) {emsg("Lost sync\n"); break;}
	
		tlen += 1 + s-bf+1;
		hlen = 0;
		
		/* ^~26:6:@test.two:4 */
		if (s[1]=='@') {
			ncpy(gname,s+2,BFSZ-1);
			s = strchr(gname,':');
			if (s!=NULL) *s =0;
			hlen = atoi(s+1);
			g = group_find(gname);
			if (g<0) {
			 	emsg("Link to missing group {%s}\n",gname);
				goto next_one;
			}
			db_idx_add(grp,fg,num,-3,group_getgroupid(g),hlen);
			goto next_one;
		}

		s = strstr(s,"\r\n\r\n");
		for (;s==NULL;) {
			hlen += 100;
			memmove(bf,bf+100,101); /* 81 gets the null too */
			n = myread(dbf[p],bf+100,100);   bf[200]=0; 
			if (n<10) xeof = TRUE;
		 	s = strstr(bf,"\r\n\r\n");
			if (n<20) break;
		}
		if (s==NULL) {emsg("No end of header found\n"); break;}
		hlen += s-bf+2;
		db_idx_add(grp,fg,num,offset,tlen,hlen);
next_one:
 		offset += tlen;
		if (mylseek(dbf[p],offset,SEEK_SET) < 0) { emsg("Seek to %d failed, end of valid data\n"); break; }
	}
}


int db_add_link(int grp, int num, String *xitem);
int db_add(int grp, int num, String *xitem, int islink)
{
	char xtop[BFSZ];
        char *eoh = "\015\012\015\012";
        char *s;
        int pp;
        int fg,fi,p;
	int xlen;
	int tlen;
	int totlen;
	int clen;
	int hlen; 
 	if (islink) {
	 	return db_add_link(grp,num,xitem);
	}
        group_addsize(grp,str_len(xitem));
        cvt_numfile2(num,&fg, &fi);
        p = db_open(grp,fg,DB_APPEND);
	clen = dblen[p];
        if (p<0) {emsg("db: db_open failed in add, %d %d \n",grp,fg); return FALSE;}


 	xlen = str_len(xitem);
        sprintf(xtop,"~%d:%d:",str_len(xitem),num);
	tlen = strlen(xtop);
	totlen = xlen + tlen + 1;
        pp = p;
        if (dbmode[p]==DB_NEW) pp = max_open;
       
 
        s = strstr(str_s(xitem),eoh);
        if (s==NULL) {emsg("item: Item has no body, will not save \n"); return FALSE;}
        hlen = s - str_s(xitem) + 2 + tlen;

  	str_addsn(xitem,"^");
        db_writes(pp,xtop);
        db_write(pp,xitem,totlen);

	db_idx_add(grp,fg,num,clen,totlen,hlen);
        dblen[pp] += totlen;

	str_setlen(xitem,xlen);
        diskuse_add(totlen); /* not counting size of index */
        return TRUE;
}
/* @group:item\n\n@group:item */ 
int db_add_link(int grp, int num, String *xitem)
{
	int n,g;
	char bf[BFSZ];
	char *s;
 	int fg,fi;
	cvt_numfile2(num,&fg, &fi);
	ncpy(bf,str_s(xitem),BFSZ-1);
	if (bf[0]!='@') {emsg("Store failed (%s)\n",bf); return FALSE;}
	s = strchr(bf,':');  if (s==NULL) goto fail;
	*s = 0;
	g = gid_find_g(bf+1);
	s++;
	n = atoi(s);
	return db_idx_add(grp,fg,num,-3,g,n);
fail:
	emsg("Bad link name (%s)\n",bf);
	return FALSE;
}
static int idxf = -1;
static char last_idx_file[BFSZ];
void db_idx_flush(void);
int db_idx_add(int grp, int fg, int num, int pnt, int tlen, int hlen)
{
        int n;
	int fi;
        char iname[BFSZ];
	int hp;
	Hrec d;
        Hrec *h;

	d.pnt = pnt;
	d.len = tlen;
	d.hlen = hlen;
	fi = num % 100;

        hp = db_find_head_cached(grp,fg); /* don't load it if we don't have it */
        if (hp>=0) { /* Update in memory cache of index */
	        h = head_cache[hp];
        	if (h==NULL) {emsg("db: head_cache is null %d %d \n",grp,fg); return FALSE;}
		h[fi].len = tlen;
		h[fi].pnt = pnt;
		h[fi].hlen = hlen;
	}
        strcpy(iname,sys_iname(grp,fg));
	if (strcmp(last_idx_file,iname)!=0) {
		db_idx_flush();
	} 
	if (idxf<0) {
        	idxf = myopen(iname,O_WRONLY | O_APPEND | O_CREAT | O_BINARY,U_RW,grp,fg);
		if (idxf<0) {	/* Might need to create the directory */
                        lib_createdir(iname);
	        	idxf = myopen(iname,O_WRONLY | O_APPEND | O_CREAT | O_BINARY,U_RW,grp,fg);
		}
        	if (idxf<0) {filemsg("fatal: Can't write index file {%s}\n",iname); return FALSE;}
		strcpy(last_idx_file,iname);
	}
        n = mywrite(idxf,to_frec(fi,&d),13,-1); 
        if (n<sizeof(d)) {filemsg("Can't write whole index file {%s}\n",iname); myclose(idxf); idxf=-1; return FALSE;}
	return TRUE;
}
void db_idx_flush(void)
{
	if (idxf<0) return;
	strcpy(last_idx_file,"");
	myclose(idxf);
	idxf = -1;
}
int db_kill(int grp, int num)
{
        char *eoh = "\015\012\015\012";
        Hrec *h;
        int fg,fi,hp;

        cvt_numfile2(num,&fg, &fi);
        hp = db_find_head_write(grp,fg);
        if (hp<0) {emsg("db: kill, db_find_head failed in add, %d %d \n",grp,fg); return FALSE;}
        h = head_cache[hp];
        if (h==NULL) {emsg("db: head_cache is null %d %d \n",grp,fg); return FALSE;}
	h[fi].len = -1;
	h[fi].pnt = -1;
	h[fi].hlen = -1;
        head_changed[hp] = TRUE;
        db_write_head(hp);
        return TRUE;
}
int db_find(int grp, int fg, int isread)
{
        int i;
        static int upto;

        if (!isread) if ((grp==new_grp) && (fg==new_fg)) return max_open;

        for (i=0; i<max_open; i++) {
                if (dbgrp[i]==grp) if (dbfg[i]==fg) return i;
        }
        upto++;
        if (upto>=max_open) upto=0;
        return upto;
} 
int db_findnum(int grp, int *xfirst, int *xlast) /* Not used, but would be faster */
{
        int i;
        int first = *xfirst;
        int last = *xlast;
        int fg,fi;
        int foundone=FALSE;
        int num = 0;
        int lastfg= -2;
        int hp;
        Hrec *h;

        *xfirst = last+1;
        for (i=first; i<=last; i++) {
                cvt_numfile2(i,&fg, &fi);
                if (fg != lastfg) {
                        hp = db_find_head_read(grp,fg);
                        if (hp<0) break;
                        h = head_cache[hp];
                }
                lastfg = fg;
                if (h[fi].pnt>=0) {
                        if (!foundone) *xfirst = i;
                        foundone = TRUE;
                        num++;
                }
        }
        return num;
}
int db_clearahead(int grp, int num);
int db_readtest(int grp, int from, int to)
{
        String *tmp;
        int i;

        for (i=from; i<=to; i++) {
          tmp = db_get(grp,i,TRUE);
          if (tmp==NULL) {
                db_clearahead(grp,i);   
                if (group_getfrom(grp)==i) {
                        group_setstart(grp,i+1);
                }                       
          } else group_addrange(grp,1);
        }
        return TRUE;
}
int db_clearahead(int grp, int num)
{
        int fg,fi;
        int hp;
        Hrec *h;

        cvt_numfile2(num,&fg, &fi);
        hp = db_find_head_write(grp,fg);
        if (hp<0) {emsg("db: db_find_head failed in add, %d %d \n",grp,fg); return FALSE;}
        h = head_cache[hp];
        if (h==NULL) {emsg("db: head_cache is null %d %d \n",grp,fg); return FALSE;}
        h[fi].len = -1;
        h[fi].hlen = -1;
        h[fi].pnt = -1;
        imsg("Cleared link to missing item %s item %d \n",group_name(grp),num);
        head_changed[hp] = TRUE;
        return TRUE;
}
int db_exists(int grp, int num)
{
        int fg,fi;
        int hp;
        Hrec *h;
        cvt_numfile2(num,&fg, &fi);
        hp = db_find_head_read(grp,fg);
        if (hp<0) {imsg("hp less than zero\n"); return FALSE;}
        h = head_cache[hp];
        /* dmsg("db: exists? grp=%d num=%d fi=%d %d \n",grp,num,fi,h[fi].pnt);*/
        if (h[fi].pnt>=0) {return TRUE; }
        return FALSE;
}
static int inupdate;
int db_update(int grp,int fg)
{
        String *shead,*sbody,*tmp;
        int islink,islink2;
        int i,ffg,ffi;

        inupdate = TRUE;
        shead = str_new();
        if (db_open(grp,fg,DB_NEW)<0) return FALSE;

        for (i=fg;i<fg+100;i++) {
                cvt_numfile(i,&ffg, &ffi);
                tmp = db_get_raw(grp,i,TRUE,&islink);
                if (tmp!=NULL) str_copy(shead,tmp);
                sbody = db_get_raw(grp,i,FALSE,&islink2);
                if ((shead!=NULL) && (tmp!=NULL)) {
                        item_purgewrite(grp,i,shead,sbody,islink);
                }
        }

        db_close_new(grp,fg);
        inupdate = FALSE;
        str_free(shead);
        return TRUE;
}
int db_open_new(int grp,int num)
{
        dmsg("db: db_open_new called %d %d \n",grp,num);
        return db_open(grp,num,DB_NEW);
}
int db_open(int grp, int num, int mode)
{
        int p;
        static int failedfg,failedgrp= -1;
	int fl;
        int fg,fi;
        int f,isread;
        static char lastfail[BFSZ];
      
        cvt_numfile2(num,&fg, &fi);

        if (mode==DB_READ) isread = TRUE; else isread = FALSE;
        
        if ((failedgrp==grp) && (failedfg==fg) && (isread)) {
                return -1;
        }
        failedgrp = -1;
        /* if (!isread) */
        if ((lastgrp==grp) && (lastfg==fg) && (lastmode==mode)) {
                p = lastp; 
                if (dbgrp[p]==grp && dbfg[p]==fg) {
                        if (dbmode[p]==mode) return p;
			if (dbmode[p]==DB_NEW) {
				emsg("db illegal mode change for file {%s:%d} %d\n",group_name(grp),num,mode);
				emsg("db File is currently locked by expire\n");
				return -1;
			}
                }
        }
        p = db_find(grp,fg,isread);
        
        lastgrp = grp; lastfg = fg; lastmode = mode; lastp = p;
        if (dbgrp[p]==grp && dbfg[p]==fg) {
                dmsg("dbx: Success, cached [%d] \n",lastp);
                if (dbmode[p]==mode) return p;
        }
        if (dbf[p]>=0) db_close(p);

 dmsg("dbx: Realopen [%d] {%s} %s \n",p,sys_gname(grp,fg),mode_name[mode]);
        if (mode==DB_APPEND) {
                f = myopen(sys_gname(grp,fg),O_WRONLY | O_APPEND | O_CREAT | O_BINARY,U_RW,grp,fg);
                if (f<0) {
                        lib_createdir(sys_gname(grp,fg));
                        f = myopen(sys_gname(grp,fg),O_WRONLY | O_APPEND | O_CREAT | O_BINARY,U_RW,grp,num);
                        if (f<0) {
                          filemsg("Couldn't open to write {%s} \n",sys_gname(grp,fg));
                          return -1;
                        }
                }
		fl = my_file_length(f,fl);
                dblen[p] = fl;
                dbsent[p] = 0;
        } else if (mode==DB_NEW) {
                dbmode[p] = DB_CLOSED;
                dbmode[max_open] = DB_CLOSED;
                f = myopen(sys_gname(grp,fg),O_RDONLY | O_BINARY,0,grp,fg);
                if (f<0) { fimsg("Couldn't open to read {%s} \n",sys_gname(grp,fg)); return -1; }
		fl = my_file_length(f,fl);
                diskuse_add(-fl);
                dbf[p] = f;
                f = myopen(sys_nname(grp,fg),O_WRONLY | O_APPEND | O_CREAT | O_BINARY,U_RW,grp,num);
                if (f<0) { filemsg("Couldn't open to write {%s} \n",sys_nname(grp,fg)); return -1; }
                dbf[max_open] = f;
                if (head_cache[max_cache]==NULL) {
                        head_cache[max_cache] = mymalloc(SIZEOFHEAD);
                }

                db_clear_head(head_cache[max_cache]);
                dblen[max_open] = 0;
                dbsent[p] = 0;
                lastmode = -1;
                new_grp = grp;
                new_fg = fg;
                mode = DB_READ;
                dbmode[max_open] = DB_APPEND;
                dbgrp[max_open] = grp;
                dbfg[max_open] = fg;
                f = dbf[p];
        } else if (mode==DB_READ) {
                f = myopen(sys_gname(grp,fg),O_RDONLY | O_BINARY,0,grp,fg);
                dbsent[p] = 0;
                if (f<0) {
                        failedgrp = grp;
                        failedfg = fg;
                        if (strcmp(lastfail,sys_gname(grp,fg))!=0) {
                                fimsg("Couldn't open to read {%s} \n",sys_gname(grp,fg));
                        }
                        ncpy(lastfail,sys_gname(grp,fg),BFSZ-1);
                        lastp = -1;
                        return -1;
                }
        }
        
        ndb_open++;
        dbmode[p] = mode;
        dbgrp[p] = grp;
        dbfg[p] = fg;
        dbf[p] = f;
        dmsg("dbx: New open OK! %d %d %s  \n",grp,fg,mode_name[mode]);
        return p;
}

int db_file_length(char *fname) 
{
	int f;
	int r;
	f = xopen(fname,O_RDONLY | O_BINARY,0);
	if (f<0) return 0;
	r = file_length(f);
	xclose(f);
	return r;
}
void db_fixlength(char *s) /* fixlength grp fg */
{
	int p;
	int hp;
        Hrec *h;
	int grp,fg;
	int was;
	emsg("Fixlength called {%s} \n",s);
	s = strtok(s," "); if (s==NULL) return;
	grp = atoi(s);
	s = strtok(NULL," ");  if (s==NULL) return;
	fg = atoi(s);
		
        hp = db_find_head_read(grp,fg);
        if (hp<0) {imsg("db: Can't open header in fixlen %d %d \n",grp,fg); return;} /* Couldn't open index file */
        p = db_open(grp,fg,DB_READ);
        if (p<0) {emsg("db: db_open failed in fixlen, %d %d \n",grp,fg); return;}
    

        if (hp<0) {imsg("db: fixlength Can't open header %d %d \n",grp,fg); return ;} /* Couldn't open index file */
        h = head_cache[hp];
	was = h[100].hlen;
	h[100].hlen = db_file_length(sys_gname(grp,fg));	
	dblen[p] = h[100].hlen;
        head_changed[hp] = TRUE;
  	emsg("Fixed length %d --> %d of %s\n",was,h[100].hlen,sys_gname(grp,fg));
}
int db_write(int p, String *s, int flen)
{
        int r;
        if (dbf[p]<0) {emsg("File is not open %d \n",p); return 0;}
        r = mywrite(dbf[p],str_s(s),str_len(s),flen);
        if (r<str_len(s)) {filemsg("Error writing article file\n"); return FALSE;}
        return TRUE;
}
int db_writes(int p, char *s)
{
        size_t r;
        if (dbf[p]<0) {emsg("File is not open %d \n",p); return 0;}
        r = mywrite(dbf[p],s,strlen(s),-1);
        if (r<strlen(s)) {filemsg("Error writing article file\n"); return FALSE;}
        return TRUE;
}
int db_hash(int grp, int fg)
{
        int r =  (fg+grp+grp*((fg/100) % 32000)) % max_cache;
        if (r<0) return -r;
        return r;
}
int db_close(int p)
{
        if (dbmode[p]!=DB_CLOSED) {
                if (p<0) {emsg("db file was never open \n"); return FALSE;}
                if (dbf[p]<0) {emsg("db file is already closed %d \n",p); return FALSE;}
                dmsg("db: db_close %d f=%d \n",p,dbf[p]);
                myclose(dbf[p]);
                ndb_open--;
        } else imsg("db: not open %d, why close it? \n",p);
        dbf[p] = -1;
        dbmode[p] = DB_CLOSED;
        return TRUE;
}
int db_close_new(int grp, int num)
{
        int p,hp;
        Hrec *h;
        int fg,fi;
        cvt_numfile2(num,&fg, &fi);

        dmsg("db: close_new %d %d\n",grp,num);

        p = db_find(grp,fg,TRUE);
        hp = db_find_head_read(grp,fg);
        if (hp<0) return FALSE; /* Couldn't open index file */
        h = head_cache[hp];
        if (h==NULL) {emsg("Null header pointer\n"); return FALSE;}
        db_close(p); db_close(max_open);
        
        new_grp = -1; new_fg = -1;

        if (remove(sys_gname(grp,fg))!=0) fimsg("remove failed %s\n",sys_gname(grp,fg));
        if (dblen[max_open]==0) {
                imsg("item: item file empty, deleting it %s\n",sys_nname(grp,fg));
                remove(sys_nname(grp,fg));
                remove(sys_iname(grp,fg));
                db_clear_head(h);
                return TRUE;
        }
        if ( rename(sys_nname(grp,fg),sys_gname(grp,fg))<0) {filemsg("db: purgeclose rename: {%s} {%s} len %d\n",sys_nname(grp,fg),sys_gname(grp,fg),dblen[max_open]);  return FALSE;}
        memcpy(h,head_cache[max_cache],SIZEOFHEAD);

        db_write_head(hp);
        return TRUE;
}
String *db_old_raw(int f, int fg, int fi, int wanthead, int *islink, Hrec *h);
String *db_old_raw(int f, int fg, int fi, int wanthead, int *islink, Hrec *h)
{
        static String *rslt;
        int offset,len,nread;

        *islink = FALSE;
        if (rslt==NULL) rslt = str_new();
        str_clear(rslt);

        if (h==NULL) {emsg("db: header null panic \n"); dnews_ex("Null header"); }
        if (h[fi].pnt== -1) {dmsg("db: get_raw= -1 \n"); return NULL;}
        if (wanthead) {
                offset = h[fi].pnt;
                len = h[fi].hlen;
        } else {
                /* -2, skips the blank line between header and body */
                offset = h[fi].pnt+h[fi].hlen+2;
                len = h[fi].len-h[fi].hlen-2;
        }
        if (len<0) {
                emsg("db: old-read len < 0,  %d:%d  offset %d:%d \n",fg,fi,offset,len);
                return NULL;
        }
        if ((len>10000000) || (offset<0)) {
                emsg("db: old-read len > 10mb,  %d:%d  offset %d:%d \n",fg,fi,offset,len);
                return NULL;
        }
        if (len>1000000) emsg("db: warning old-read len > 1mb, %d:%d  offset %d:%d \n",fg,fi,offset,len);

        if (mylseek(dbf[f],offset,SEEK_SET)<0) {
                filemsg("db: seek failed %d %d, offset %d \n",fg,fi,offset);
                return NULL;
        }

        dmsg("db: read2[%d] len %d \n",f,len);
        str_setlen(rslt,len+2); 
        nread = myread(dbf[f],str_s(rslt), len);
        str_setlen(rslt,len);   
        if (nread<len) {
                filemsg("db: old read failed %d %d, offset %d \n",fg,fi,offset);
                return NULL;
        }
        if ((len<200) && (str_s(rslt)[0] == '@')) *islink = TRUE;
        return rslt;
}
static int store_len;
int db_last_len(void)
{
        if (store_len<0) return 0;
        if (store_len>2000000) return 0;
        return store_len;
}
String *db_get_raw(int grp, int num,int wanthead, int *islink)
{
        int x;
        return db_get_raw_size(grp,num,wanthead,islink,0,7000000,&x);
}
String *db_get_raw_size(int grp, int num,int wanthead, int *islink, int from, int from_len, int *total)
{
        static String *rslt,*rslt2;
        int fg,fi,offset,len,nread;
        char abit[BFSZ];
	static char bf[BFSZ];
        int hp;
        Hrec *h;
        int f;
        char *s;
        *total = 0;
        cvt_numfile2(num,&fg, &fi);

        hp = db_find_head_read(grp,fg); if (hp<0) {dmsg("db: no db_findhead\n"); return NULL;}
        f = db_open(grp,num,DB_READ); if (f<0)  {dmsg("db: no db_open\n"); return NULL;}
        h = head_cache[hp]; if (h==NULL) {emsg("head null?? \n"); return NULL;}


        dmsg("db: db_get_raw grp=%d num=%d wanthead=%d \n",grp,num,wanthead);
        *islink = FALSE;
        if (rslt==NULL) rslt = str_new();
        str_clearbig(rslt);
        if (rslt2==NULL) rslt2 = str_new();
        str_clearbig(rslt2);

        dbg dmsg("item: db_get_raw2 f=%d hp=%d \n",f,hp);
        if (h==NULL) {emsg("db: header null panic \n"); dnews_ex("Null header2"); }
        if (h[fi].pnt== -1) {dmsg("db: get_raw=-1 \n"); return NULL;}
	if (h[fi].pnt==-3) {
		if (gid_find_group(h[fi].len)==NULL) {
			emsg("Link to gid that no longer exists %d\n",h[fi].len);
			return NULL;
		}
	 	sprintf(bf,"@%s:%d%s%s@%s:%d",
	 		gid_find_group(h[fi].len),h[fi].hlen,
			EOL,EOL,
	 		gid_find_group(h[fi].len),h[fi].hlen);
		str_copysn(rslt2,bf);
		*total = strlen(bf);
		*islink = TRUE;
		return rslt2;
	}

        if (h[fi].hlen<0) {
                *islink = TRUE;
                offset = h[fi].pnt;
                len = h[fi].len;
        } else {
                if (*islink)  {
                        offset = h[fi].pnt;
                        len = h[fi].len;
                } else if (wanthead) {
                        offset = h[fi].pnt;
                        len = h[fi].hlen;
                        store_len = h[fi].len;
                } else {
                        /* -2, skips the blank line between header and body */
                        offset = h[fi].pnt+h[fi].hlen+2;
                        len = h[fi].len-h[fi].hlen-2;
                }
        }
        if (mylseek(dbf[f],offset+from,SEEK_SET)<0) {
                filemsg("db: seek failed %d %d, offset %d \n",grp,num,offset);
                return NULL;
        }

        if (len>100000) imsg("db: large item %s:%d size=%d \n",group_name(grp),num,len);
        if (len>10000000 || len<0) {
                emsg("db: Item can't be > 10mb, must be corrupt index\n");
                return NULL;
        }
        *total = len;
        if (len>from_len) {
                len = from_len;
                if (from+len > *total) len = *total - from;
        }
        str_setlen(rslt,len+2); 
        str_setlen(rslt2,len+2);
        dmsg("db: read[%d] len %d \n",f,len);
        nread = myread(dbf[f],str_s(rslt), len);
        if (nread<len) {
                filemsg("Read error on group %d:%d, offset %d  wanted %d got %d \n",grp,num,offset,len,nread);
                return NULL;
        }
        if (!(*islink)) {
          if (wanthead) {
                if (str_s(rslt)[0] != '~') {
                        ncpy(abit,str_s(rslt),20);
                        emsg("Starting ~ missing %s:%d, offset %d %d {%s}\n",group_name(grp),num,offset,len,abit);
                        return NULL;
                }
          } else if (str_s(rslt)[len-1] != '^') {
                if ((*total == len) && (from==0)) { 
                        ncpy(abit,str_s(rslt)+len-15,20);
                        emsg("End ^ of item missing %s:%d, offset %d %d {%s} \n",group_name(grp),num,offset,len,abit);
                        return NULL;
                }
          }
        }

        /* Strip off extra info, also deal with link a bit */
        /* ~(len):article^ */

        str_lock(rslt);
        if (*islink) {           /* Only safe because rslt2 is already allocated !!!*/
                dmsg("db: islink {%s} \n",str_s(rslt));
                s = strchr(str_s(rslt),':'); if (s==NULL) goto return_null;
                s = strchr(s+1,':'); if (s==NULL) goto return_null;
                str_copys(rslt2,s+1,len-(s-str_s(rslt)+1));
                s = strstr(str_s(rslt2),"\015\012"); 
                if (s==NULL) {emsg("Dud link \n"); goto return_null;}
                str_setlen(rslt2,s-str_s(rslt2)+2);
        } else if (wanthead) { /* Step past length and item number */
                s = strchr(str_s(rslt),':'); if (s==NULL) goto return_null;
                s = strchr(s+1,':'); if (s==NULL) goto return_null;
                store_len -= 1+((s+1)-(str_s(rslt)));
                str_copys(rslt2,s+1,len-(s-str_s(rslt)+1));
        } else {
                if (*total == from + len) 
                        str_copys(rslt2,str_s(rslt),len-1);
                else {
                        str_copy(rslt2,rslt);
                        str_setlen(rslt2,len);
                }
        }
        str_unlock(rslt);
        str_clearbig(rslt);

        return rslt2;
return_null: 
        str_unlock(rslt);
        return NULL;
}

String *db_get(int grp, int num,int wanthead)
{
        int x;
        return db_get_size(grp,num,wanthead,0,7000000,&x);
}
static int store_grp, store_num;
void db_get_actual(int *grp, int *num)
{
        *grp = store_grp;
        *num = store_num;
}
String *db_get_size(int grp, int num,int wanthead, int from, int from_len, int *total)
{
        static String *tmp;
        int islink;
        char bf[BFSZ];
        char *s;
        char abit[BFSZ];

        dmsg("db: db_get %d %d \n",grp,num);
        if (grp<0) return NULL; /* Just incase :-) */ 
try_again:
        store_grp = grp; store_num = num;
        tmp = db_get_raw_size(grp,num,wanthead,&islink,from,from_len,total);
        if (tmp==NULL) return tmp;
        if (!islink) return tmp;
        s = str_s(tmp);
        if (*s!='@') goto link_fail;
        ncpy(bf,s+1,100);
        s = strtok(bf,":");  if (s==NULL) goto link_fail;
        grp = group_find(s);
        if (grp<0) {wmsg("Link points to noexistant group {%s} \n",s); return NULL;}
        s = strtok(NULL,":");  if (s==NULL) goto link_fail;
        num = atoi(s);
        dmsg("item: db_get try_again %d:%d \n",group_name(grp),num);
        goto try_again;

link_fail:
        ncpy(abit,str_s(tmp),10);
        emsg("db: @ sign missing in symbolic link %s:%d {%s}\n",group_name(grp),num,abit); 
        return NULL;  /* was return tmp and that caused a crash ??? */
}

int db_find_head_read(int grp, int fg)
{
        return db_find_head(grp,fg,TRUE);
}
int db_find_head_write(int grp, int fg)
{
        return db_find_head(grp,fg,FALSE);
}
int db_find_head(int grp, int fg, int isread)
{
        int r;
        if (!isread) if ((grp==new_grp) && (fg==new_fg)) return max_cache;
        r = db_hash(grp,fg);
/*r = 1;
if (head_fg[r]>0) return r;
  */
        if (head_fg[r]==fg) if (head_grp[r]==grp) {
                return r;
        }
        if (head_changed[r]) db_write_head(r);
        if (!db_read_head(r,grp,fg)) return -1;
        return r; 
}
int db_find_head_cached(int grp, int fg)
{
        int r;
        if ((grp==new_grp) && (fg==new_fg)) return max_cache;
        r = db_hash(grp,fg);
        if (head_fg[r]==fg) if (head_grp[r]==grp) {
                return r;
        }
        return -1; 
}
char *to_frec(int i, Hrec *r)
{
 	char bf[BFSZ];
	bf[0] = i;
	memcpy(bf+1,&r->pnt,4);
	memcpy(bf+5,&r->len,4);
	memcpy(bf+9,&r->hlen,4);
	return bf;
}
int to_hrec(Hrec *r,char *bf)
{
	memcpy(&(r->pnt),bf+1,4);
	memcpy(&(r->len),bf+5,4);
	memcpy(&(r->hlen),bf+9,4);
 	return bf[0];
}
int db_write_head(int r)
{
        Hrec *h;
        int idx;
        char iname[BFSZ];
	int v;

        if (r<0) {emsg("Write head bad value %d \n",r); return FALSE;}
        strcpy(iname,sys_iname(head_grp[r],head_fg[r]));
        idx = myopen(iname,O_WRONLY | O_CREAT | O_BINARY,U_RW,head_grp[r],head_fg[r]);
        if (idx<0) {filemsg("fatal: Can't write index file {%s}\n",iname); return FALSE;}

	h = head_cache[r];
	for (v=0; v<99; v++) {
		if (h[v].pnt!=-1) {
		 	mywrite(idx,to_frec(v,&(h[v])),13,-1);
		}
	}
        myclose(idx);
        head_changed[r] = FALSE;
        return TRUE;
}
int db_clear_head(Hrec *h)
{
        int i;
        if (h==NULL) {emsg("Panic, clear head to null header??\n"); return FALSE;}
        for (i=0; i<HEAD_SIZE; i++) {
                h[i].len = -1;
                h[i].pnt = -1;
                h[i].hlen = -1;
        }
        h[100].len = -2;  /* Version 2.0 of database format */  
        return TRUE;
}
int db_read_head(int r, int grp, int fg)
{
        int idx;
        char iname[BFSZ];
	char bf[5000];
        Hrec *h,anh;
	int z,v;
        int n;
        dmsg("db: Actual read_head[%d] %d %d\n",r,grp,fg);

        head_grp[r] = grp;
        head_fg[r] = fg;
        head_changed[r] = FALSE;
        strcpy(iname,sys_iname(head_grp[r],head_fg[r]));
        if (head_cache[r]==NULL) {
                head_cache[r] = mymalloc(SIZEOFHEAD);
        }

        if (head_cache[r]==NULL) {emsg("Mem excceeded in read_head\n"); return FALSE;}
        h = head_cache[r];
try_again:
        db_clear_head(h);
        idx = myopen(iname,O_RDONLY | O_BINARY,U_RW,-1,-1);
        if (idx<0) { /* Doesn't exist yet probably */
                return TRUE;
        }
        dmsg("db: read_head  %d %s:%d size=%d\n",r,group_name(head_grp[r]),head_fg[r],SIZEOFHEAD);
        n = myread(idx,bf,SIZEOFHEAD*2);
	myclose(idx);
	if (n==SIZEOFHEAD) {
		emsg("Converting header on the fly for %s %d\n",group_name(grp),fg);
		db_convert3(grp,fg);
		goto try_again;
	}	
	if ((n%13) != 0 ) {
		emsg("Index file bad length, deleting\n");
  		remove(iname);
		return TRUE; /* returning empty header */	
	}	

	for (z=0; z<n; z+=13) {
		v = to_hrec(&anh,bf+z) ;
		h[v] = anh;	
	}

        return TRUE;
}

int db_flush2(void) /* Call flush, but don't close */
{
        int i;
        time_show();
        
        dbg dmsg("db: flush2\n");
	db_idx_flush();

	wmsg("db: Closing database files\n");
        for (i=0; i<max_open; i++) {
                if (dbmode[i]==DB_APPEND) {
                        db_close(i);
                }
        }
	wmsg("db: Writing index files\n");
        for (i=0; i<max_cache; i++) {
                if (head_changed[i]) db_write_head(i);
        }
	wmsg("db: Writing index files completed\n");
        return TRUE;
}
int db_flush_slowx(void);
int db_flush_old(void)
{
 	if (config_true(C_DB_FLUSH2)) {
		db_flush2();
	}
	return TRUE;
}
int db_flush_slow(void)
{
	if (config_true(C_DB_FLUSH2)) return TRUE;
	stat_start(X_FLUSHSLOW);
       	db_flush_slowx();
	stat_start(X_FLUSHSLOW);
	return TRUE;
}
int db_flush_slowx(void)
{
	static int uptoi,uptox;
	static int last;
	if ((lib_gmtime() - last) < 4) return TRUE;
	last = lib_gmtime();

	db_idx_flush();

	if (uptoi>=max_open) uptoi = 0;
        for (; uptoi<max_open; uptoi++) {
                if (dbmode[uptoi]==DB_APPEND) {
			dmsg("db: closing item file %d\n",uptoi);
                        db_close(uptoi);
			break;
                }
        }
	uptoi++;

	if (uptox>=max_cache) uptox = 0;
        for (; uptox<max_cache; uptox++) {
                if (head_changed[uptox]) {
			dmsg("db: closing index file %d\n",uptox);
                	db_write_head(uptox);
			break;
		}
        }
	uptox++;

	pi_sync_all();

        return TRUE;
}

static int mynread,mysizeread;
static int mynseek,mynopen;
char *mystats(void)
{
        static char bf[500];
        strcpy(bf,"");
        if (mynread==0) return bf;
        sprintf(bf,"%d reads, %d bytes read, %d seeks, %d opens\015\012",
                        mynread,mysizeread,mynseek,mynopen);
        return bf;
}

#ifdef USE_FSTREAMS
static FILE *myh[100];
int mynexth(void)
{
        int i;
        for (i=0; i<100; i++) {
                if (myh[i]==NULL) return i;
        }
        emsg("No free handles for my file routines \n");
        return 0;
}
int myread(int handle, void *buffer, unsigned int count)
{
        mynread++;
        mysizeread += count;
        if (myh[handle]==NULL) emsg("myread handle null %d\n",handle);
        return fread(buffer,1,count,myh[handle]);
}
int mywrite(int handle, const void *buffer, unsigned int count)
{
        if (myh[handle]==NULL) emsg("mywrite handle null %d\n",handle);
        return fwrite(buffer,1,count,myh[handle]);
}
long mylseek(int handle, long offset, int origin)
{
        mynseek++;
        return fseek(myh[handle],offset,origin);
}
int myopen(const char *fname, int oflag, int pmode)
{
        int f = mynexth();
        char *mde;
        mynopen++;
        mde = "rb";
        pmode = 0; /* not used on nt anyway */ 
        if (oflag & O_WRONLY) mde = "wb";
        if (oflag & O_RDONLY) mde = "rb";
        if (oflag & O_APPEND) mde = "ab";
        myh[f] = myfopen(fname,mde);
        if (myh[f]==NULL) return -1;
        return f;
}
int myclose(int handle)
{
        int r;
        if (myh[handle]==NULL) {emsg("myclose handle null %d\n",handle); return 0;}
        r = fclose(myh[handle]);
        myh[handle] = NULL;
        return r;
}
#endif

#ifndef USE_MYPIPES
int myread(int handle, void *buffer, unsigned int count)
{
 	return read(handle,buffer,count);
}
int mywrite(int handle, char *buffer, unsigned int count, int flen)
{
 	return write(handle,buffer,count);
}
long mylseek(int handle, long offset, int origin)
{
 	return lseek(handle,offset,origin);
}
int myopen(const char *fname, int oflag, int pmode, int gid, int num)
{
 	return xopen(fname,oflag,pmode);
}
int myclose(int handle)
{
	return xclose(handle);
}
char *pi_status(void)
{
	return "Internal Writers";
}
#endif

