// New spam module for recognizing 'content' in an automated fashion, accurate and with high speed updates....
// Prefix is 'spamc' spam contents 

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>

#ifndef FALSE
#define FALSE (0)
#endif

#ifndef TRUE
#define TRUE (!FALSE)
#endif

#include "../sam/lib.h"
#include "../sam/lprintf.h"
#include "../sam/rstring.h"
#include "../sam/mime.h"
#include "../sam/tcplib.h"
#include "../sam/spamc.h"
#include "../sam/emsg.h"
#include "../sam/keep.h"
#include "../sam/fmalloc.h"
#include "../sam/trace.h"
#define DMID 14
#include "debug.h"

typedef struct NODE {int hash; char flags; struct NODE *next;} Node;
typedef struct FNODE {int hash; char flags;} Fnode;
#define dtrace(str) trace_add(__FILE__,__LINE__,str)
Node *sc_node_add(int h, Node *table[], int flags);
Node *sc_node_find(int h, Node *table[]);
int sc_add_hash(int crc, int flags);
int sc_test_hash(int crc, int *bad, int *good);
int spamc_process_one(Kwordlist *w, int isspam, int isscan, char *fname);

#define NODE_HASH 9000
static Node *node_table[NODE_HASH]; // Table of nodes  flags==1 bad isspam, flags==2 good, flags==3 both...
static Fmalloc *mem;
#ifdef DO_THREADS
#include "../sam/thread.h"
static Mutex *m;
#endif

void spamc_init(void)
{
#ifdef DO_THREADS
	if (m==NULL) m = th_mutex_new("spamc");
#endif
	if (mem==NULL) mem = fmalloc_new();

}
#ifndef MUTEX_TIME
#define MUTEX_TIME 600000
#endif

static int mutex_i;
static int (*mutex_cb)(int vm, int isget, int seconds);
int spamc_set_mutex(int (*cb)(int vm, int isget, int seconds), int m)
{
		mutex_i = m;
		mutex_cb = cb;
		return TRUE;
}

int spamc_mutex_wait(void)
{
	if (mutex_cb) return mutex_cb(mutex_i,TRUE,600);
#ifdef DO_THREADS
		spamc_init();
		if (!th_mutex_wait(m,MUTEX_TIME)) { /* 10 minutes */
			lib_abort();
		}
#endif
	return TRUE;
}
void spamc_mutex_release(void)
{
	if (mutex_cb) { mutex_cb(mutex_i,FALSE,600); return; }
#ifdef DO_THREADS
	if (!th_mutex_release(m)) {
		lib_abort();
	}
#endif
}
#define MAXLANG 50

static int tot_isspam,tot_notspam;
static int stats_nspam,stats_nnotspam,stats_nzero;

char *spamc_status(char *temp)
{
	lprintf(temp,BFSZ-1,"%3.2f%% (db %d/%d) spam=%d ok=%d zero=%d",100.0*stats_nspam/(stats_nzero+1), tot_isspam,tot_notspam,stats_nspam,stats_nnotspam,stats_nzero);
	return temp;
}

int spamc_load_table(char *dbfile)
{
	Fnode fn;
	FILE *f;

	spamc_init();
	f = fopen(dbfile,"rb");
	if (f==NULL) return FALSE;
	spamc_mutex_wait();
	for (;!feof(f);) {
		if (fread(&fn,sizeof(fn),1,f) != 1) break;
		if (fn.hash<0) continue;
		sc_node_add(fn.hash,node_table,fn.flags);
	}
	spamc_mutex_release();
	fclose(f);
//	printf("tot_notspam %d tot_isspam %d\n",tot_notspam,tot_isspam);
	return TRUE;
}
int utf_space(char *in, char *out);
int spamc_store_table(char *dbfile)
{
	Node *n;
	Fnode fn;
	int h;
	FILE *f;
	f = fopen(dbfile,"wb");
	if (f==NULL) {
		printf("could not write (%s) %s\n",dbfile,strerror(errno));
		return FALSE;
	}
	spamc_mutex_wait();
	for (h=0; h<NODE_HASH; h++) {
		for (n=node_table[h]; n!=NULL; n=n->next) {
			fn.hash = n->hash;
			fn.flags = n->flags;
			fwrite(&fn,sizeof(fn),1,f);	
		}
	}
	spamc_mutex_release();
	fclose(f);
	return TRUE;
}
static int sc_disabled;
int spamc_clear(void)
{
	Node *n,*next;
	int h;
	sc_disabled = TRUE;
	spamc_mutex_wait();
	tot_isspam = tot_notspam = 0;

	for (h=0; h<NODE_HASH; h++) {
		for (n=node_table[h]; n!=NULL; n=next) {
			next = n->next;
			fmalloc_free(mem,n);
		}
		node_table[h] = NULL;
	}
	fmalloc_freeall(mem); // actually free them.  
	mem = fmalloc_new();
	spamc_mutex_release();
	return TRUE;
}

char *spamc_wash_html(char *bf)
{
	char *s;
	int skip=FALSE;
	char xbf[BFSZ];
	char *out=xbf;
	for (s=bf;*s!=0;s++) {
		// strip out &nbsp 
		if (strncmp(s,"&nbsp",4)==0) s+=4; 
		
		if (strncmp(s,"<!--",4)==0) {
			s = strstr(s,"-->");
			if (s==NULL)  break;
			s += 3;
			if (*s==0) break;
		}
		if (*s=='<') skip = TRUE;
		if (!skip) *out++ = *s;
		if (*s=='>') skip = FALSE;
		if (out-xbf>BFSZ-10) break;
		//	<!-- =2Estyle1 {font-family: Arial, Helvetica, sans-serif} -->
		if (*s==0) break;
	}
	*out++ = 0;
	ncpy(bf,xbf,BFSZ-1);
	return bf;
}
int spamc_token(char *in, char *tmp, char **p, int maxp)
{
	int np;
	int i;
	int x;
	char *w;
	char tmp2[BFSZ];
	ncpy(tmp2,in,BFSZ-1);
	utf_space(tmp2,tmp);
	strlwr(tmp);
	np = token_split(tmp,p,", ;:-\r\n\t\"\\/<>[]{}()#$%!@",maxp);

	for (i=0; i<np; i++) {
		w = p[i];

		x = lib_strlen(w);
		if (x>0) if (w[x-1]=='.') w[x-1] = 0;
		//printf("Word was (%s)\n",w);
	}
	return np;
}

Kwordlist *spamc_import_file(char *fname, char *mem, int memlen, char *lang)
{
	// Process the message with 'mime' library, find the first text or html part and 
	// read the words we find....
	Kwordlist *wlist=NULL;
	char raw[BFSZ];
	char tmp[BFSZ];
	char bin[BFSZ];
	char wtemp[BFSZ];
	int inpdf=FALSE;
	Mime *m;
	int totlen = 0;
	int image_size=0;
	int pdf_size=0;
	int k;
	int len;
	int inimage=FALSE;
	FILE *f=NULL;
	int isjpg=FALSE;
	char *p[500];
	int np;
	int i;
	int nw=0; // Stop at 300 words. 
	char words[12000];
	int skip_head = FALSE;
	int lscores[MAXLANG];
	lg_scores_init(lscores);
	strcpy(lang,"");
	strcpy(words,"");

	mime_new(&m);
	if (fname==NULL) mime_setmem(m,mem,memlen); // read direct from memory buffer if necessary. 
	else {
		f = fopen(fname,"rb");
		if (f==NULL) {
			mime_free(&m);
			return NULL;
		}
	}
	wlist = kword_new();
	m->loose_header = TRUE;
	for (k=TRUE;k;) {
		k = mime_line(m,f,raw,bin,&len,BFSZ-1);
//printf("k=%d, len=%d (%s)\n",k,len,bin);
		switch (k) {
		case MIME_END:
			break;
		case MIME_HEADER:
			skip_head = FALSE;
			break;
		case MIME_CONTENT:
			break;
		case MIME_HEADER_END:
			skip_head = FALSE;
			inimage = FALSE;
			inpdf=FALSE;
			strlwr(m->content);
			if (strstr(m->content,"image")!=NULL) {
				if (strstrnc(m->content,"jpeg")!=NULL) isjpg = TRUE;
				if (strstrnc(m->content,"jpg")!=NULL) isjpg = TRUE;
				inimage = TRUE;
			}
			if (strstr(m->content,"application/pdf")!=NULL) {
				inpdf = TRUE;
			}
			if (strstr(m->content,"multipart")!=NULL) skip_head = TRUE;
			break;		
		case MIME_BODY:
			if (len<=0) continue;
			if (skip_head) continue;
			// Should buffer up 10K so we get multi line urls'
			if (inimage) image_size += len; 
			if (inpdf) pdf_size += len;			

			// Turn this into a list of words to add....
			bin[len] = 0;
//printf(" got data %s\n",bin);
			if (totlen+len>8000) break;
			strcpy(words+totlen,bin);
			totlen += lib_strlen(bin);
			break;
		case MIME_BOUNDARY:
			break;
		}
	}
	mime_free(&m);
	if (f) fclose(f); f = NULL;
	// Now cut up 'words' buffer....
	strlwr(words);
//printf("Before wash: %s\n",words);
	spamc_wash_html(words);

	np = spamc_token(words,wtemp,p,200);

	for (i=0; i<np; i++) {
//printf("Word [%d/%d] %s\n",i,np,p[i]);
		if (i>200) break;
		lg_process(lscores,p[i]);
		ncpy(tmp,p[i],BFSZ-1); tmp[26] = 0;  //why are we truncating to 10 characters I wonder ? I know to simpliify lines of dashes etc...
		kword_add(wlist,tmp);
	}
	wlist = kword_reverse(wlist);
	lg_results(lscores,lang);
	return wlist;		
}
static int false_pos,false_neg, false_zero;
static int total_done;
static int nskip;
static int nright;
int spamc_process_filemem(char *fname, char *mem, int len, char *lang) // CRASH IN THIS FUNCTION...
{
	int ret=FALSE;
	Kwordlist *w;
	dtrace("Import file");
	w = spamc_import_file(fname,mem,len,lang);
	dtrace("Import file-done");
	if (w) {
		ret = spamc_process_one(w,FALSE,TRUE,fname);
		dtrace("spamc_process done");
		kword_free(w);
	}
	return ret;
}

int spamc_process_file(char *fname, int isspam, int isscan, char *lang)
{
	int ret = FALSE;
	Kwordlist *w=NULL;
	imsg("spamc: Processing %s\n",fname);
	w = spamc_import_file(fname,NULL,0,lang);
	if (w!=NULL) {
		ret = spamc_process_one(w,isspam,isscan,fname);
		kword_free(w);  // crashed here...
	}
	return ret;
}
int spamc_process_path(Chan *c, char *dir, int isspam, int isscan) // needs to write the hash info...
{
	// Read files in the specified directory and convert to one line entries....
	WILD_HANDLE *h;
	char fname[BFSZ];
	WILD_DATA mydata;
	char wild[BFSZ];
	int k=0;
	char lang[BFSZ];
	int more;
	memset(&mydata,0,sizeof(mydata));
	if (c) tcp_printf(c,"%s folder (%s)\r\n",isscan ? "Scanning" : "Training" ,dir);
	lprintf(wild,BFSZ-1,"%s/*",dir);

	false_pos = total_done = nright = false_neg = total_done = false_zero = 0; // clear stats.

	more = wild_first(&h, wild, &mydata);
	if (!more) if (c) tcp_printf(c,"No files found %s\n",strerror(errno));
	for (;more;more=wild_next(h, &mydata)) {
		k++;
		if (!mydata.isnorm) {
			continue;
		}
		lprintf(fname,BFSZ-1,"%s/%s",dir,mydata.name);
		//if (k<20) printf("Found (%s)\n", fname);
		spamc_process_file(fname,isspam,isscan,lang);
	}
	wild_end(&h);

	if (c) {
		if (isscan)	tcp_printf(c,"False pos %3.2g%%, false neg %3.2g%%, unsure %3.2g%%, correct %3.2g%% of %d\n",false_pos*100.0/total_done,false_neg*100.0/total_done,false_zero*100.0/total_done,nright*100.0/total_done,total_done);
		else tcp_printf(c,"Skipped %d of %d\r\n",nskip,total_done);
	}

	return TRUE;
}
int spamc_do_scale(int *bad, int *good)
{
	if ((tot_isspam ==0) || (tot_notspam==0)) return TRUE;
	if (tot_isspam<tot_notspam) {
		*bad = *bad * tot_notspam / tot_isspam;
	} else {
		*good = *good * tot_isspam / tot_notspam;
	}
	return TRUE;
}
#define MAXNODES 500000
int spamc_trim(void)
{
	static int upto;
	int h;
	Node *n,*next;

	h = upto;
	for (;tot_isspam+tot_notspam > MAXNODES;) {
		for (h=upto; h<NODE_HASH; h++) {
			for (n=node_table[h]; n!=NULL; n=next) {
				next = n->next;
				if (n->flags & 1) tot_isspam--;
				if (n->flags & 2) tot_notspam--;
				// free(n); // CRASHING ON THIS FREE... BECAYSE WRONG TYPE OF FREE!!!
				fmalloc_free(mem,n);
			}
			node_table[h] = NULL;
			if (tot_isspam+tot_notspam < MAXNODES) break;
		}
		if (tot_isspam+tot_notspam < MAXNODES) break;
		if (upto==0) {
			tot_isspam = tot_notspam = 0;
			break;
		}
		if (upto>0) upto = 0;		
	}
	upto = h;
	return TRUE;
}
//static Node *good[NODE_HASH]; // Table of good nodes
//static Node *bad[NODE_HASH];  // table of bad nodes

#define NWORDS 4 
int spamc_process_one(Kwordlist *words, int isspam, int isscan, char *fname)
{
	int h=0;
	int k;
	int good,bad;
	int isbad;
	Kword *w,*x;
	Node *unclear=NULL;
	int ret=0;
	// Return 1 = spam, -1 = not spam, 0 = unknown.

	good = bad = 0;

	if (isscan) if (sc_disabled) return 0;
	if (words==NULL) return 0;
	imsg("spamc: processing %s\n",fname);

			// Hash each group of '3' words. 
	spamc_mutex_wait();
	total_done++;

	dtrace("scan1");
	if (!isscan) {
		if (words) for (w=words->head;w!=NULL;w=w->next) {
			h = 0;
			for (x=w,k=0; x!=NULL && k<NWORDS; x=x->next,k++) {
				h += lib_str_hash(x->word,100000000);
			}
			sc_test_hash(h,&bad,&good);
		}

		if (isspam) {
			if ((bad>good*2) && (bad+good>10)) {
				imsg("Skipping as already correct score %d/%d %s\n",bad,good,fname);
				nskip++;
				spamc_mutex_release();
				return TRUE;
			}
		} else {
			if ((good>bad*2) && (bad+good>10)) {
				imsg("Skipping as already correct score %d/%d %s\n",bad,good,fname);
				nskip++;
				spamc_mutex_release();
				return TRUE;
			}
		}
	}

	dtrace("scan3");
	if (words) for (w=words->head;w!=NULL;w=w->next) { // bug was fixed here.
		h = 0;
		for (x=w,k=0; x!=NULL && k<NWORDS; x=x->next,k++) {
			h += lib_str_hash(x->word,100000000);
		}
		if (isscan) { 
			dtrace("scan3.1");
			sc_test_hash(h,&bad,&good);
		} else {
			dtrace("scan3.2");
			sc_add_hash(h,isspam ? 1 : 2);
		}
	}
	dtrace("scan4");
	spamc_trim();
	dtrace("scan5");


	spamc_mutex_release();
	if (isscan) {
		isbad = FALSE;
		if (bad+good>5) {
			dtrace("scan5.1");
			spamc_do_scale(&bad,&good);
			if (bad>((1.5*good)+5)) ret = 1;
			if (good>((1.5*bad)+5)) ret = -1;
		}
		if (tot_isspam<9000) ret = 0;
		if (tot_notspam<9000) ret = 0; // insufficient data so don't set results at all! 
		dtrace("scan6");

		if (ret==1) isbad = TRUE;

		if (ret==1) stats_nspam++;
		if (ret==-1) stats_nnotspam++;
		if (ret==0) stats_nzero++;

		// printf("Score %d/%d %s\n",bad,good,fname);
		if (ret!=0) {
			if (isbad!=isspam) {
				if (isbad) false_pos++; else false_neg++;
				
				//printf("%s %s %d/%d %s\n",fname,isspam ? "isspam" : "notspam",bad,good,isbad!=isspam ? "FAILED" : "OK");
			} else {
				nright++;
			}
		} else {
			//imsg("%s Unidentified %d/%d\n",fname,bad,good);
			false_zero++;
		}
	
	}
	return ret;
}


int sc_node_hash(int h)
{
	int r = h % NODE_HASH;
	if (r<0) return -r;
	return r;
}
Node *sc_node_find(int h, Node *table[])
{
	Node *n;
	int t = sc_node_hash(h);
	for (n=table[t];n!=NULL;n=n->next) {
		if (n->hash==h) return n;
	}
	return NULL;
}
Node *sc_node_add(int h, Node *table[], int flags)
{
	Node *n;
	int t = sc_node_hash(h);
	n = fmalloc_calloc(mem,1,sizeof(Node));
	n->next = table[t];
	n->flags = flags;
	table[t] = n;
	n->hash = h;

	if (flags&1) tot_isspam++ ;
	if (flags&2) tot_notspam++;


	return n;
}
int sc_add_hash(int crc, int flags)
{
	Node *b;
	b = sc_node_find(crc,node_table);
	if (b==NULL) sc_node_add(crc,node_table,flags);
	else {
		if (!(b->flags&1)) if (flags&1) tot_isspam++ ;
		if (!(b->flags&2)) if (flags&2) tot_notspam++ ;
		b->flags |= flags;
	}
	return TRUE;
}
int sc_test_hash(int crc, int *bad, int *good)
{
	Node *b;

	b = sc_node_find(crc,node_table);
	if (b==NULL) return TRUE;
	if (b->flags==1) (*bad) += 1;
	if (b->flags==2) (*good) += 1;
	return TRUE;
}

int spamc_reenable(void)
{
	spamc_store_table("spamc_table.dat");
	sc_disabled = FALSE; 
	return TRUE;
}
int spamc_train(Chan *c, char *path, int isspam)
{
	return spamc_process_path(c,path,isspam,FALSE);
}
int spamc_test(Chan *c, char *path, int isspam)
{
	return spamc_process_path(c,path,isspam,TRUE);	
}
	


/*
language_guess.dat Data file format:


English: word word word word
French: word word word
Dutch: word word word
Dutch: word word word
Dutch: word word word
Dutch: word word word
Japanese: word word word

If utf8 if starts 1110 or 11110 then treat as a word in it's own right.  (3 bytes, 4 bytes) 


*/


static char *languages[MAXLANG];
typedef struct LG {
	char *word; 
	int hits;
	int language; 
	struct LG *next; 
} Lg;
/* New module, recognize the 'language' used in an email and bounce if not understood by customer. */
#define LGHASH 2000
static Lg *lgtable[LGHASH];
int lg_scores_init(int *scores)
{
	int i;
	for (i=0; i<MAXLANG; i++) scores[i] = 0;
	return TRUE;
}
int lg_findadd(char *lang)
{
	int i;
	for (i=0; i<MAXLANG; i++) {
		if (languages[i]==NULL) break;
		if (strcmpnc(lang,languages[i])==0) return i;
	}
	if (i<MAXLANG-1) {
		languages[i] = strdup(lang);
		return i;
	}
	return -1;

}
static int lg_ok;	
int utf_space(char *in, char *out)
{
        int n=0;
		int len=0;
		unsigned char *e;

     	for (e=in;*e!=0;e++) {
				if (len>BFSZ-10) break;
				if ((*e & 0xf0) == 0xd0) {  // 3 byte unicode
					out[len++] = *e++; if (*e==0) break;
					out[len++] = *e++; if (*e==0) break;
					out[len++] = *e++; if (*e==0) break;
					out[len++] = ' ';
					continue;					
				}
				if ((*e & 0xf8) == 0xf0) {  // 4 byte unicode
					out[len++] = *e++; if (*e==0) break;
					out[len++] = *e++; if (*e==0) break;
					out[len++] = *e++; if (*e==0) break;
					out[len++] = *e++; if (*e==0) break;
					out[len++] = ' ';
					continue;					
				}
				out[len++] = *e;
		}
		out[len++] = 0;
		return TRUE;
}


int lg_load(char *datafile)
{
	FILE *f;
	char bf[BFSZ];
	char bf2[BFSZ];
	char *p[200];
	int n;
	int l;
	int i;
	Lg *t;
	f = fopen(datafile,"rb");
	if (f==NULL) return FALSE;
	for (;!feof(f);) {
		if (fgets(bf,BFSZ-1,f)==NULL) break;
		utf_space(bf,bf2);
		n = token_split(bf2,p," \r\n\t",100);
		if (n<2) continue;
		l = lg_findadd(p[0]);
		if (l>=0) {
			for (i=1; i<n; i++) {
				int h = lib_str_hash(p[i],LGHASH);
				t = calloc(1,sizeof(Lg));
				t->word = strdup(p[i]);
				t->next = lgtable[h];
				t->language = l;
				lgtable[h] = t;
				lg_ok = TRUE;
			}
		}
	}
	fclose(f);
	imsg("language: lg_ok = %d\n",lg_ok);
	return TRUE;
}
int lg_dump(Chan *c)
{
	int h;
	Lg *t;
	for (h=0; h<LGHASH; h++) {
		for (t=lgtable[h];t!=NULL;t=t->next) {
			tcp_printf(c,"%d %s %s\r\n",t->hits,languages[t->language],t->word);
		}
	}
	return TRUE;
}
int lg_process(int *scores,char *word)
{
	Lg *t;
	int h;

	h = lib_str_hash(word,LGHASH);

	for (t=lgtable[h];t!=NULL;t=t->next) {
		if (strcmp(word,t->word)==0) {
			t->hits++; 
			scores[t->language]++;
		}
	}
	return TRUE;
}
int lg_results(int *scores, char *result)
{
	int i;
	int max=0;
	if (!lg_ok) {
		strcpy(result,""); return FALSE;
	}
	strcpy(result,"Unknown");
	for (i=0; i<MAXLANG; i++) {
		//if (scores[i]>0) printf("language: result was (%s) %d\n",languages[i],scores[i]);
		if (scores[i]>max) {
			ncpy(result,languages[i],BFSZ-1);
			max = scores[i];
		}
	}
	//imsg("language: result was (%s) %d\n",result,max);
	return TRUE;
}
