/*
int aspam_rules_bodyhash(Ruleset *r, char *line)
  
	Automatic generation of rules, takes a message and finds this info.
 Total     aspam  mfilter both
 Total [1] 1725=81.5% 2054=97.1% 2105=99.5%
 Total [2] 1609=76.0% 1974=93.3% 2098=99.1%
 Total [3] 1440=68.1% 1968=93.0% 2092=98.9%
 Total [4] 1253=59.2% 1966=92.9% 2077=98.2%
 Total [5] 1011=47.8% 1966=92.9% 2067=97.7%
 Total [6] 839=39.7% 1943=91.8% 2054=97.1%
 Total [7] 620=29.3% 1942=91.8% 2037=96.3%
 Items per second 8.33 2116 items
 Log files mfilter.log, aspam.log, both.log

test whyspam? email address
make g_spam_internal disable smitecrc.
add command to 'run through test set' and print out figures, and list failures'
Add command to 'remove' a rule which is faulty
if (istrusted) skip spam stuff (add setting to not do this)

*/

#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ctype.h>
#include <errno.h>
#include "emsg.h"
#ifndef TRUE
#define FALSE 0
#define TRUE (!FALSE)
#endif
#include "defines.h"
#include "lprintf.h"
#include "lib.h"
#include "tcplib.h"
#define DMID 14
#include "debug.h"
#include "aspam.h"
#include "keep.h"
#include "rstring.h"
#include "mime.h"
#include "debug_marijn.h"
/*
#undef malloc
#undef calloc
#undef strdup

#define malloc real_malloc
#define calloc real_calloc
#define strdup real_strdup
  */
static unsigned char *save_pbad,*save_pgood;

#define MAXID 9
typedef struct RULESET {
		char *source; // base, isspam, notspam, catcher
		char *user; // If submitted by authenticated user then remember who so we can double value when message is 'to' them.
		char *from; int nfrom; int isfrom;
		char *exip; int nexip; int isexip;
		char *bodyhash;
		Wordlist *urls; int nurl_found; int nurls;
		Wordlist *match; int nmatch;
		Wordlist *non; int nnon;
		int mask[MAXID+1]; /* Mask of words in 'match' which we found in this message. */ 
		int mask_non[MAXID+1]; /* Mask of words in 'match' which we found in this message. */ 
		int ntot; /* Total hits on this rule*/
		int isbad; /* */
		int nbad,ngood; /* Additional votes for this rule which may affect how we use it */
		int hits;
		int totwords; /* Total 'words' found in message, excluding html etc, indicative of match and spam to some extent*/
		struct RULESET *next;
} Ruleset; 
typedef struct WRULE { Ruleset *rule; char *word; int id; struct WRULE *next; } Wrule;
static int is_forgery;
static char forgery_msg[BFSZ];

int aspam_rules_analyze(double *ret, Wordlist **result);
static Ruleset *set_bad;
//static Ruleset *set_white;
#define RULESZ 2000
static Wrule *rules_url[RULESZ];
static Wrule *rules_good[RULESZ];
static Wrule *rules_from[RULESZ];
static Wrule *rules_exip[RULESZ];
static Wrule *rules_match[RULESZ];
static Wrule *rules_non[RULESZ];
static int totwords;

#define URLLEN 45
typedef struct WTABLE {char word[12]; int ngood; int nbad; struct WTABLE *next;} Wtable; 
typedef struct UTABLE {char word[URLLEN]; int ngood; int nbad; struct UTABLE *next;} Utable; 
#define HASHSZ 10000
int aspam_bodyhash_clear(void);
static Wtable *wnon[HASHSZ];
static Wtable *whash[HASHSZ];
static Wtable *whash_all[HASHSZ];
static Wtable *whash_english[HASHSZ];
#define URLSZ 200
static Utable *urls[URLSZ];
static Keeplist *kwnon,*kwhash,*kwhash_english,*kwhash_all;
static Keeplist *kwrule;
static int show;
int poly_init(void);
int aspam_rules_bodyhash(Ruleset *r, char *line);
int aspam_score_body(int *h_val,int nh, char *fname);

int aspam_words_free(Wtable **t, Keeplist **k);
int aspam_rules_load_one(Ruleset **rr,Wordlist *wl);
int wrule_find_goodurl(char *word);
int aspam_url_free(void);
int aspam_rule_show(Ruleset *r, char *out);
int aspam_scan_rules(double *ret, Wordlist **result);
int aspam_rules_reset(void);
int wrule_add(Wrule **table,int hashsz, Ruleset *r, char *word, int id);
int aspam_rules_free_one(Ruleset **rr);
int aspam_rules_load(char *rfile);
int aspam_words_add(char *bin, int len, int isbad, int *h_val, int *pnh);
int aspam_words_file(void);
int english_find(char *word);
int english_add(char *bf, int ngood, int nbad);
int aspam_url_show(Wordlist **rules);
int aspam_url_add(char *word);
int english_find2(char *word, int *ngood, int *nbad);
int aspam_words_choose(Wordlist **rules, int isbad, double *bscore, char *info);
int aspam_non_choose(Wordlist **rules, int isbad);
int aspam_rules_match(Ruleset *r, char *line, int isnon);
int aspam_rules_url(Ruleset *r, char *line);
int aspam_rules_from(Ruleset *r, char *line);
int aspam_rules_exip(Ruleset *r, char *line);
int wrule_find_match(char *word);
int aspam_uwords_free(Utable **t, int hashsz);
int wrule_free(Wrule **table,int hashsz);
int aspam_rule_showr(Ruleset *r, Wordlist **result);
static int nbad, ngood;
int aspam_scan_urls(Wordlist **rules,char *bin,int len)
{
	char *s;
	char *t;
	char url[BFSZ];
	if (len<0) lib_abort();
	strlwr(bin);
	bin[len] = 0;
	for (s=bin;s<bin+len;s++) {
		// href="www.ljsw.com"
		if (*s=='h') if (strncmp(s,"href=\"",6)==0) {
			s+=6;
			if (strncmp(s,"https:",6)==0) s+= 5;
			if (strncmp(s,"http:",5)==0) s+= 5;
			goto found;
		}
		// http://xxx.yyy.com 
		// http://xxx.yyy.com=yyy.com 
		if (*s=='w') if (strncmp(s,"www.",4)==0) {
			goto found;
		}
		if (*s=='h') if (strncmp(s,"http",4)==0) {
			s += 4;
			if (*s=='s') s++;
			if (*s!=0) s++;
found:		if (*s=='/') s++;
			if (*s=='/') s++;

			ncpy(url,s,100);
			t = strchr(url,'/');
			if (t!=NULL) *t = 0;
			t = strchr(url,'\r');
			if (t!=NULL) *t = 0;
			t = strchr(url,'\n');
			if (t!=NULL) *t = 0;
			t = strchr(url,':');
			if (t!=NULL) *t = 0;
			t = strchr(url,'"');
			if (t!=NULL) *t = 0;
			t = strchr(url,' ');
			if (t!=NULL) *t = 0;
			t = strchr(url,'?');
			if (t!=NULL) *t = 0;
			t = strchr(url,'>');
			if (t!=NULL) *t = 0;
			t = strchr(url,'<');
			if (t!=NULL) *t = 0;
			t = strchr(url,')');
			if (t!=NULL) *t = 0;
			t = strchr(url,'%');
			if (t!=NULL) *t = 0;
			strlwr(url);

			t = strchr(url,'=');
			if (t!=NULL) {
				aspam_url_add(t+1);
				*t = 0;
			}
			aspam_url_add(url);
		}
	}
	return TRUE;
}
static int doword;
int aspam_doword(int x)
{
	doword = x;
	return TRUE;
}
static int donone;
int aspam_donone(void)
{
	donone = TRUE;
	doword = TRUE;
	return TRUE;
}
static int doscanner;
int msg_aspam_at(char *msg);
int aspam_doscan(int x)
{
	doscanner = x;
	return TRUE;
}
static int addif;
int aspam_doif(int x)
{
	addif = x;
	return TRUE;
}
static char from[BFSZ];
static char exip[BFSZ];
static double global_score;
static int added;
int aspam_added(void)
{
	return added;
}
int aspam_scan_addif(Wordlist **rules, char *fname, int isbad, char *reason, double *score, Wordlist **result, char *rfname, int force)
{
	Ruleset *r=NULL;
	Wordlist *w;
	FILE *f;
	static int tot,nadded;
	if (!addif) return aspam_scan(rules,fname,isbad,reason,score,result);
	doscanner = TRUE;
	tot++;
	added = FALSE;
	aspam_scan(rules,fname,isbad,reason,score,result);
	word_free(rules);
	word_free(result);
	if (force || (global_score<5 && isbad) || (!isbad && (global_score>1)) ) {
imsg("Adding new rule score %.1f %.1f %s\n",global_score,*score,fname);
//printf("Adding new rule score %.1f %.1f %s\n",global_score,*score,fname);
		added = TRUE;
		doscanner = FALSE;
		aspam_scan(rules,fname,isbad,reason,score,result);
		f = fopen(rfname,"ab");
		imsg("aspam: Adding to (%s) file (%s) isbad=%d\n",rfname,fname,isbad);
		if (f!=NULL) {
			for (w=*rules;w!=NULL;w=w->next) {
				fprintf(f,"%s\r\n",w->word);
			}
			fprintf(f,"\r\n");
			fclose(f);
		}
		/* And add the rule immediately */
		// chrisp commented out as aspam seems to be disabled everywhere else...
		//aspam_rules_load_one(&r,*rules);
//		word_free(rules);	
		nadded++;
		imsg("train: Added new rule %d/%d\n",nadded,tot);
	} else {
		imsg("train: New rule not needed score was %.1f %.1f for %s\n",global_score,*score,isbad ? "BAD" : "GOOD");
//printf("New rule not needed score was %.1f %.1f\n",global_score,*score);
	}
	return TRUE;
}
#define MAXBIG 64000
static char *mem;
static int mem_len;
int aspam_setmem(char *pmem, int len)
{
	mem = pmem;
	mem_len = len;
	return TRUE;
}
static double bscore;
static double poly_score;
double aspam_bscore(void)
{
	return bscore;
}
static char zexip[BFSZ];
int aspam_setexip(char *s)
{
	ncpy(zexip,s,BFSZ-1);
	return TRUE;
}
static int poly_disable;
int poly_set_disable(int x)
{
	if (x) poly_disable =  TRUE;
	return TRUE;
}
static int notascii;
static char charset[BFSZ];
int aspam_notascii(char *ch)
{
	ncpy(ch,charset,BFSZ-1);
	return notascii;
}
static int is_naked_cr;
int aspam_nakedcr(void)
{
	return is_naked_cr;
}
int aspam_isforgery(char *ch)
{
	ncpy(ch,forgery_msg,BFSZ-1);
	return is_forgery;
}
int aspam_scan_notascii(char *big, int sz)
{
	int i;
	char *s;
	if (sz>4000) sz = 4000;
	for (i=0; i<sz; i++) {
		if (big[i] == 'c') {
			if (strncmp(big+i,"charset=",8)==0) {
				s = big+i+8;
				if (*s=='"') s++;
				ncpy(charset,s,30);
				s = strchr(charset,'"'); if (s!=NULL) *s = 0;
				s = strchr(charset,'\r'); if (s!=NULL) *s = 0;
				s = strchr(charset,'\n'); if (s!=NULL) *s = 0;
				s = strchr(charset,' '); if (s!=NULL) *s = 0;
				strlwr(charset);
				//imsg("aspam: Found charset (%s) (debugging line to be removed later)\n",charset); 
				if (strncmp(charset,"iso-8859",8)!=0) {
					notascii = TRUE;
				}
			}
		}
	}
	return TRUE;
}
char *trim_wbr(char *url, char *out)
{
	char *s;
	char *f;
	s = strstrnc(url,"<wbr>");
	if (s==NULL) {
		ncpy(out,url,BFSZ-1);
		return out;
	}
	ncpy(out,url,BFSZ-1);
	s = strstrnc(out,"<wbr>");
	*s = 0;
	f = strstrnc(url,"<wbr>");
	ncat(out,f+5,BFSZ-1);
	return out;	
}
char *aspam_trim_url(char *url)
{
	char *s;
	if (strncmp(url,"http://",7)==0) url += 7;
	if (strncmp(url,"https://",8)==0) url += 8;
	for (s=url;*s!=0;s++) {
		if (*s=='/') break;
		if (*s==' ') break;
		if (*s=='"') break;
		if (*s==':') break;
		if (*s=='?') break;
		if (*s=='\r') break;
		if (*s=='\n') break;
		if (*s=='\t') break;
	}
	*s = 0;
	return url;
}

int aspam_scan_url_forgeries(char *big, int sz)
{
	int i;
	char tmp6[BFSZ];
	char href[BFSZ];
	char second[BFSZ];
	char *s;
	sz -= 10;
	if (sz>20000) sz = 20000;
	for (i=0; i<sz; i++) {
		if (big[i] == 'h') {
			if (strncmp(big+i,"href=\"",6)==0) {
//printf("(%.200s)\n",big+i-20);
				s = big+i+6;
				ncpy(href,s,100);
//printf("1(%.200s)\n",href);
				aspam_trim_url(href);
//printf("2(%.200s)\n",href);

				s = strchr(s,'"');
				if (s==NULL) continue;
				s = strchr(s,'>');
				if (s==NULL) continue;
				s++;
				for (;*s!=0 && sisspace(*s);s++) ;
				ncpy(second,s,100);
//	imsg("aspam: url search (%s) (%s)\n",href,second);
				if (strncmpnc(second,"http",4)!=0) continue;
				aspam_trim_url(second);
				trim_wbr(second,tmp6); ncpy(second,tmp6,BFSZ-1);
//	printf("second (%s) href (%s)\n",second,href);
				s = strchr(second,'<'); if (s!=NULL) *s = 0;
				if (!lib_compare_base(second,href)) {
					imsg("aspam: url forgery (%s) (%s)\n",href,second);
					lprintf(forgery_msg,BFSZ-1,"(%s) (%s)",href,second);
					is_forgery = TRUE;

//printf("(%.100s)\n",big);
					break;
				} else {
//					imsg("aspam: url ok (%s) (%s)\n",href,second);
				}
			}
		}
	}
	return TRUE;
}
static int 	mime_isimage,mime_isjpeg,mime_isgif,mime_image_size,mime_nimage;

int aspam_scan(Wordlist **rules, char *fname, int isbad, char *reason, double *ret_score, Wordlist **result)
{
	int g1;
	int i;
	Mime *m;
	FILE *f=NULL;
	char tmp2[BFSZ];
	int k;
	int nlines=0;
	//char reason2[BFSZ];
	char bf[BFSZ];
	char raw[BFSZ];
	char bin[BFSZ];
//	char big[MAXBIG+10];
	static char *big=NULL;
	int len=0;
	char *s;
	int fixed=0;
	char *upto;
	int ret=FALSE;
	int inhead=TRUE;
	int bigsz=0;
	int drop=FALSE;
	char subj[BFSZ];
	int g2;
	int h_val[10];
	int nh;
	int mayburl;
	int inimage=FALSE;

	mime_isimage = mime_isjpeg = mime_isgif = mime_nimage = FALSE;
	mime_image_size = 0;


	notascii = FALSE;
	is_naked_cr = FALSE;
	is_forgery = FALSE;
	aspam_url_free();
	if (big==NULL) big  = malloc(MAXBIG+10);
	g1 = 2222222;
	g2 = 2222222;
	mime_new(&m);
	bscore = 0;
	totwords = 0;
	strcpy(subj,"");
	if (isbad) nbad++;
	else ngood++;
	strcpy(from,"");
	strcpy(exip,"");
	strcpy(reason,"None-aspam");
	surbl_reset();

	lprintf(bf,BFSZ-1,"source: %s base %s",isbad ? "bad" : "good" ,fname);
	word_add(rules,bf);
	con_reset();
	msg_aspam_at("scana1");
	
	if (mem!=NULL) mime_setmem(m,mem,mem_len);
	else {
		f = fopen(fname,"rb");
		if (f==NULL) {
			lprintf(reason,BFSZ-1,"virus_extract FAILED %s %s",fname,strerror(errno));
			mime_free(&m);
			return FALSE;
		}
		setvbuf(f,NULL,_IOFBF,32000);
	}
	msg_aspam_at("scana2");
	m->loose_header = TRUE;
	for (k=TRUE;k;) {
		int i;
		if (nlines>600) break;
		nlines++;
		k = mime_line(m,f,raw,bin,&len,BFSZ-1);
		switch (k) {
		case MIME_END:
			break;
		case MIME_HEADER:
			if (strncmp(raw,"X-External-IP: ",15)==0) {
				ncpy(tmp2,raw+15,BFSZ-1);
				s = mystrtok(tmp2,"\r\n\t ",&upto);
				if (s!=NULL) ncpy(exip,s,100);
			}
			if (strncmp(raw,"X-Originating-IP: ",18)==0) {
				ncpy(tmp2,raw+18,BFSZ-1);
				s = mystrtok(tmp2,"\r\n\t ",&upto);
				if (s!=NULL) ncpy(exip,s,100);
			}
			if (strncmp(raw,"From: ",6)==0) {
				lib_fix_email(raw+6,tmp2);
				s = mystrtok(tmp2,"\r\n\t ",&upto);
				if (s!=NULL) ncpy(from,s,100);
			}
			if (strncmp(raw,"Subject: ",9)==0) {
				ncpy(subj,raw+9,100);
			}
			aspam_scan_notascii(raw,lib_strlen(raw));
			break;
		case MIME_CONTENT:
			aspam_scan_notascii(raw,lib_strlen(raw));
			break;
		case MIME_HEADER_END:
			inhead = FALSE;
			inimage = FALSE;
			drop = FALSE;
			strlwr(m->content);
			if (strstr(m->content,"image")!=NULL) {
				drop = TRUE;
				mime_isimage = TRUE;
				if (strstrnc(m->content,"jpeg")!=NULL) mime_isjpeg = TRUE;
				if (strstrnc(m->content,"jpg")!=NULL) mime_isjpeg = TRUE;
				if (strstrnc(m->content,"gif")!=NULL) mime_isgif = TRUE;
				mime_nimage++;
				inimage = TRUE;
			}
			if (strstr(m->content,"applicaton")!=NULL) drop = TRUE;
			if (strstr(m->content,"audio")!=NULL) drop = TRUE;
			if (strstr(m->content,"video")!=NULL) drop = TRUE;
			break;		
		case MIME_BODY:
			// Should buffer up 10K so we get multi line urls'
			if (inimage) mime_image_size += len; 
			
			if (drop) break;
			if (strncmp(bin,"X-SpamDetect",12)==0) break;
			if (len<0) len = 0;
			if (len>=0) bin[len] = 0;
			for (i=0; i<len-1; i++) { // changed so doesn't check the last one...
				if (fixed>5) break;
				if (bin[i]==0) bin[i] = ' ';
				if (bin[i]=='\r') {
					if (i<len-1) if (bin[i+1]!='\n') {
						imsg("aspam: Stripped naked CR character, probably is spam\n");
						fixed++;
						is_naked_cr = TRUE;
						safe_memmove(bin+i,bin+i+1,len-i+1,bin,BFSZ-1);
						len--;
					}
				}
			}
			strlwr(bin);
			len = lib_strlen(bin); /* get rid of nulls  - this might let spam through if they are smart */
//printf("bin (%s)\n",bin);
			if (strncmp(bin,"x-external-ip: ",15)==0) {
				ncpy(tmp2,bin+15,BFSZ-1);
				s = mystrtok(tmp2,"\r\n\t ",&upto);
				if (s!=NULL) ncpy(exip,s,100);
			}
			if (strncmp(bin,"x-originating-ip: ",18)==0) {
				ncpy(tmp2,bin+18,BFSZ-1);
				s = mystrtok(tmp2,"\r\n\t ",&upto);
				if (s!=NULL) ncpy(exip,s,100);
			}
			if (strncmp(bin,"from: ",6)==0) {
				lib_fix_email(bin+6,tmp2);
				s = mystrtok(tmp2,"\r\n\t ",&upto);
				if (s!=NULL) ncpy(from,s,100);
			}

			// Chrisp removed this for mime_quoted as it was breaking urls by adding a space in them
			// Chrisp added this back, because clearly it should be a space (or new line) 
			// so now I need to see an example with a broken web link to examine
			// Ok I've seen an exmaple where the spaces are clearly before the equals signs when they should exist, now I need
			// to see an example where the spaces are 'missing'.... 
			mayburl = FALSE;
			if (strstr(bin,"http:")!=NULL) mayburl = TRUE;
			if (strstr(bin,"https:")!=NULL) mayburl = TRUE;
			//if (len+100<BFSZ) if ((m->encoding == MIME_NONE) || (((m->encoding == MIME_QUOTED) && !mayburl)) ) {
			// chrisp removed for example 'sample/forge_url.txt 
			if (len+100<BFSZ) if ((m->encoding == MIME_NONE) ) {
				bin[len++] = ' ';
				bin[len] = 0;
			}

			if ((len+bigsz+BFSZ+1000)<MAXBIG) {
				safe_memcpy(big+bigsz,bin,len,big,MAXBIG);
				bigsz = bigsz + len;
				big[bigsz] = 0;
			} 
			break;
		case MIME_BOUNDARY:
			break;
		}
	}
	imsg("aspam: image2 info, nimage=%d image_size=%d isgif=%d isjpeg=%d\n",mime_nimage,mime_image_size,mime_isgif,mime_isjpeg);
	if (f!=NULL) fclose(f); f = NULL;
	imsg("aspam: xxstep3\n");
	mime_free(&m);
	imsg("aspam: xxstep4\n");
	msg_aspam_at("scana3");
	big[bigsz] = 0;
	ncat(big,"\r\n",MAXBIG-1);
	ncat(big,subj,MAXBIG-1);
	imsg("aspam: xxstep5\n");
	strlwr(big);

//#ifdef TRUE_xxxx
#ifdef TRUE_xxx
	{
FILE *f;
f = fopen("aspam.dump","wb");
if (f!=NULL) {
	fwrite(big,1,bigsz,f);
	fclose(f);	
}		
	}
#endif

	bigsz += lib_strlen(subj)+2;
	imsg("aspam: notascii check\n");
	aspam_scan_notascii(big,bigsz);
	imsg("aspam: Scan url forgeries, size %d\n",bigsz); // It didn't get to here...
	aspam_scan_url_forgeries(big,bigsz);

	msg_aspam_at("scana4");
	aspam_scan_urls(rules,big,bigsz);
	msg_aspam_at("scana5");
	aspam_words_add(big,bigsz,isbad,h_val,&nh);
	msg_aspam_at("scana6");

	strcpy(tmp2,"");
	for (i=0; i<nh; i++) {
		lprintf(bf,BFSZ-1,"%d ",h_val[i]);
		ncat(tmp2,bf,BFSZ-1);
	}
	lprintf(bf,BFSZ-1,"bodyhash: %s",tmp2); word_add(rules,bf);
//printf("%s\n",bf);

	if (lib_strlen(from)>2) {
		lprintf(bf,BFSZ-1,"from: %s",from); word_add(rules,bf);
	}
	if (lib_strlen(exip)>2) {
		lprintf(bf,BFSZ-1,"exip: %s",exip); word_add(rules,bf);
	} else if (lib_strlen(zexip)>0) {
		lprintf(bf,BFSZ-1,"exip: %s",zexip); word_add(rules,bf);
	}

	if (doscanner) { 
//		char binfo[BFSZ];
// My tests show 'no' meaning in the words of any significance.  
// the filters work due to header crap and that isn't justified. 
//		aspam_words_choose(rules,isbad,&bscore,binfo); 
//		lprintf(bf,BFSZ-1,"BScore: %2.1f %s",bscore,binfo);
//		word_add(result,bf);
		msg_aspam_at("scanb1");

		aspam_score_body(h_val,nh,fname);
		aspam_scan_rules(ret_score,result);
	} else if (doword) aspam_words_file(); /* Add words from this file onto totals */
	else {
		char binfo[BFSZ];
		aspam_words_choose(rules,isbad,&bscore,binfo); /* Choose the most 'bad' words in this message */
		aspam_non_choose(rules,isbad); /* Choose the most 'bad' words in this message */
	}

	msg_aspam_at("scanb12");
	aspam_url_show(rules);
	msg_aspam_at("scanb13");
	aspam_url_free();
	msg_aspam_at("scanb14");
	aspam_words_free(wnon,&kwnon);
	msg_aspam_at("scanb15");
	if (g1!=g2) lib_abort();
	if (g1!=2222222) lib_abort();

	
//rename("poly.dead","poly.debug");
//lib_file_copy(fname,"poly.dead",reason2);
	poly_score = 0;
	if (!poly_disable) {
		poly_scan(NULL,fname,isbad,&poly_score,!doscanner);
	}
//remove("poly.dead");
	msg_aspam_at("scanb16");
	
	return TRUE;
}
static double url_scale=1.0;
int aspam_url_scale(double x)
{
	url_scale = x;
	return TRUE;
}
// these are poly counters.
static int goodsz=1; // Size of sample - used to normalize.
static int badsz=1; // Size of sample - used to normalize if one sample is larger

double aspam_poly_score_v(double v)
{
	return 0; // chrisp disabled, as seems to provide nothing useful in tests.

	if (goodsz < 5) return 0; // Not enough data to work. 
	if (badsz < 50) return 0; // Not enough data to work. 

	if (v < 0.41) return -1;
	if (v > 0.49) return 1.0;
	if (v > 0.54) return 2.0;
	return 0;
}
double aspam_poly_score(void)
{
	// Chrisp removed until I've rewritten fully. 
	return aspam_poly_score_v(poly_score);
}
#include "aspam_tld.h"
typedef struct TLD {char *tld; struct TLD *next;} Tld;
#define TLDHASH 1001
static Tld *tld_table[TLDHASH];
int aspam_tld_load(void)
{
	int i;
	int h;
	Tld *t;
	for (i=0; tld_list[i]!=NULL; i++) {
		h = lib_str_hash(tld_list[i],TLDHASH);
		t = calloc(1,sizeof(Tld));
		t->tld = strdup(tld_list[i]);
		t->next = tld_table[h];
		tld_table[h] = t;
	}
	return TRUE;
}
int aspam_tld_find(char *lastbit)
{
	int h = lib_str_hash(lastbit,TLDHASH);
	Tld *t;
	for	(t = tld_table[h];t!=NULL;t=t->next) {
		if (strcmp(lastbit,t->tld)==0) return TRUE;
	}	
	return FALSE;
}

int aspam_word_add(Wtable **whash, int hashsz, char *word, int nbad, int ngood,Keeplist **k);
int aspam_uword_add(Utable **whash, int hashsz, char *word, int isbad);
int aspam_url_add(char *word)
{
	// Search back from end for tld
	// If domain ends in .com, .net, .biz, then we want 2 dots, else we want 3 
	//
	char *s;
	int dots=0;
	int want=2;
	s = word + lib_strlen(word)-1;
	if (s<word) return FALSE;
	for (;s>=word;s--) {
		if (*s=='@') {
			word = s+1;
			break;
		}
		if (*s=='.') {
			dots++;
			if (dots==2) {
				if (aspam_tld_find(s+1)) want=3; else want=2;
			}
			if (dots==want) {
				if (!sisdigit(s[1])) {
					word = s+1;
					break;
				}
			}
		}
	}
	if (strchr(word,'.')==NULL) return TRUE;
	if (lib_strlen(word)<3) return TRUE; // Just skip words that are too short. 

	surbl_remember(word);

	return aspam_uword_add(urls,URLSZ,word,TRUE);
}
int aspam_url_show(Wordlist **rules)
{
	char bf[BFSZ];
	Utable *w,*wnext;
	int i;
	for (i=0; i<URLSZ; i++) {
		for (w=urls[i];w!=NULL;w=wnext) {
			wnext = w->next;
			lprintf(bf,BFSZ-1,"url: %s %d",w->word,w->nbad);
			strlwr(bf);
			word_add(rules,bf);
			if (show) printf("%s\n",bf);
		}
	}
	return TRUE;
}
int aspam_url_free(void)
{
	Utable *w,*wnext;
	int i;
	for (i=0; i<URLSZ; i++) {
		for (w=urls[i];w!=NULL;w=wnext) {
			wnext = w->next;
			free(w);
		}
		urls[i] = NULL;
	}
	return TRUE;
}
Wtable *aspam_word_find(Wtable **whash, int hashsz, char *word)
{
	Wtable *w;
	int h;
	h = lib_str_hash(word,hashsz);
	for (w = whash[h]; w!=NULL; w=w->next) {
		if (strncmp(word,w->word,12)==0) {
			return w;
		}
	}
	return NULL;
}
int aspam_word_add(Wtable **whash, int hashsz, char *xword, int nbad, int ngood,Keeplist **k)
{
	Wtable *w;
	int h;
	char word[BFSZ];
	ncpy(word,xword,12);
	h = lib_str_hash(word,hashsz);
	for (w = whash[h]; w!=NULL; w=w->next) {
		if (strncmp(word,w->word,12)==0) {
			w->ngood += ngood;
			w->nbad += nbad;
			return TRUE;
		}
	}
	w = keep_malloc_align(sizeof(Wtable),k);
	w->ngood += ngood;
	w->nbad += nbad;
	strncpy(w->word,word,12);
	w->next = whash[h];
	whash[h] = w;
	return TRUE;
}
int aspam_uword_add(Utable **vurls, int hashsz, char *xword, int isbad)
{
	Utable *w;
	int h;
	char word[BFSZ];
	ncpy(word,xword,URLLEN-1);
	h = lib_str_hash(word,hashsz);
	for (w = vurls[h]; w!=NULL; w=w->next) {
		if (strncmp(word,w->word,URLLEN)==0) {
			if (isbad) w->nbad++;
			else w->ngood++;
			return TRUE;
		}
	}
	w = calloc(1,sizeof(Utable));
	if (isbad) w->nbad++;
	else w->ngood++;
	ncpy(w->word,word,URLLEN-1);
	w->next = vurls[h];
	vurls[h] = w;
	return TRUE;
}
int aspam_remove_html(char *bin, int len)
{
	int i;
	int inhtml = FALSE;
	int inquote = FALSE;
	char *out=bin;
	char c;
	for (i=0; i<len; i++) {
		c = bin[i];
		if (inhtml) {
			if (inquote) {
				if (c=='"') {
					inquote = FALSE;
				}
			} else {
				if (c=='>') inhtml = FALSE;
				if (c=='"') {
					inquote = TRUE;
				}
			}
		} else {
			if (inquote) {
				if (c=='"') {
					inquote = FALSE;
				}
			} else {
				if (c=='<') {
					inhtml = TRUE;
				}
				if (c=='"') {
					inquote = TRUE;
				}
			}
			if (c!='<' && c!='>') if (c>31) *out++ = c;
		}
	}
	*out++ = 0;
	return (int)(out-bin);

}
int aspam_show(int x)
{
	show = x;
	return TRUE;
}
int aspam_content_score(char *xbin, int len, char *h1, char *h2)
{
	char tmp3[BFSZ];
	double v;
	char *bin = malloc(len+1);
	strcpy(h1,""); strcpy(h2,"");
	memcpy(bin,xbin,len+1);
	strlwr(bin);
	len = aspam_remove_html(bin,len);
	con_reset();
	con_scan_message(bin,len);
	phrase_scan_message(bin,len);
//printf("Phrase scan being called\n");
	if (phrase_score(tmp3)) {
		lprintf(h1,BFSZ-1,"X-PhraseHits: %s",tmp3);
		imsg("phrasehits: results (%s)\n",tmp3);
	}
	if (con_score(&v,h2)) {
	}
	free(bin);
	return TRUE;
}
int aspam_words_add(char *bin, int len, int isbad, int *h_val, int *pnh)
{
	char **p=calloc(sizeof(char *),5000); //[5000];
	int np;
	int i;
	int ngood,nbad;
	char *wrd;
	int eob=FALSE;
	int ise;
	int val=0;
	int h_got = 0;
	int nh=0;
	strlwr(bin);
	if (show) printf("Before %s",bin);
	aspam_remove_html(bin,len);
//	strlwr(bin);
	//con_scan_message(bin,len); // moved to sf_mfilter calls...
	//phrase_scan_message(bin,len);
	for (i=0; i<10; i++) h_val[i] = 0;
	nh = 0;
	if (show) printf("AFter  %s\n",bin);
	np = token_split(bin,p," \t@'\"?&=!#*^~%()[]{}|;:,+<>|/\\\r\n",4000);
	for (i=0; i<np; i++) {
//		strlwr(p[i]);
//		printf("%s ",p[i]);
//		if (i%10==9) printf("\n");

		wrd = p[i];
		if (lib_strlen(wrd)>0) {
			if (wrd[lib_strlen(wrd)-1]=='.') {
				wrd[lib_strlen(wrd)-1]=0;
				if (h_got>3) {
					eob = TRUE;
				}
			}
		}
		// Meaningless words we pick up due to formatting/mime stuff
		if (strcmp(wrd,"nbsp")==0) continue;
		if (strcmp(wrd,"html")==0) continue;
		if (*wrd=='.') wrd++; // Get rid of leading dots. 
		if (lib_strlen(p[i])>2) {

			val += lib_str_hash(wrd,100000);
			h_got++;
			if (eob) {
				eob = FALSE;
				if (nh<5) h_val[nh++] = val;
				val = 0; // hash of this block. 
				h_got = 0; // number of words in this  block 

			}
			totwords++;
			ise = english_find2(wrd,&ngood,&nbad);
			if (!ise || (ngood<1000 && nbad>=0) )  { // If uncommon english word or not a word at all
				if (donone) {
					if (!ise) {
						aspam_word_add(whash,HASHSZ,wrd,isbad ? 1 : 0,isbad ? 0 : 1,&kwhash);
					}
				} else {
					if (ise) {
						aspam_word_add(whash,HASHSZ,wrd,isbad ? 1 : 0,isbad ? 0 : 1,&kwhash);
					} else {
						if (!sisdigit(*wrd)) {
							aspam_word_add(wnon,HASHSZ,wrd,isbad ? 1 : 0,isbad ? 0 : 1,&kwnon);
						}
					}
				}
			}
		}
	}
	*pnh = nh;
	free(p);
	return TRUE;
}
typedef struct WSORT {char *word; int score; double rscore;} Wsort;
int aspam_cmp(const void *aa, const void *bb)
{
	Wsort *a = (Wsort *)aa;
	Wsort *b = (Wsort *)bb;
//	printf("Sorting %d %s\n",a->score,a->word);
	return a->score-b->score;
}
int aspam_cmpb(const void *aa, const void *bb)
{
	Wsort *a = (Wsort *)aa;
	Wsort *b = (Wsort *)bb;
//	printf("Sorting %d %s\n",a->score,a->word);
	if (a->rscore>b->rscore) return 1;
	if (a->rscore<b->rscore) return -1;
	return 0;
}
int aspam_words_choose(Wordlist **rules, int isbad, double *bscore, char *binfo)
{
	int i;
	int score;
	int sample;
	double v;
	int tot;
	int ngood,nbad;
	char bf[BFSZ];
	Wtable *w;
	int k=0;
	Wsort *stable;
	int nwords=0;
	*bscore = 0;
	strcpy(binfo,"");
	for (i=0; i<HASHSZ; i++) {
		for (w = whash[i]; w!=NULL; w=w->next) {
			nwords++;
		}
	}
	if (nwords==0) return FALSE;
	stable = calloc(nwords,sizeof(Wsort));
	for (i=0; i<HASHSZ; i++) {
		double d;
		for (w = whash[i]; w!=NULL; w=w->next) {
			english_find2(w->word,&ngood,&nbad);
			stable[k].word = w->word;
			if (isbad) stable[k].score = ngood-nbad;
			else stable[k].score = nbad-ngood;
			d = (double) nbad/(double)nbad+ngood;
			if (d<.01) d = .01;
			if (d>.99) d = .99;
			stable[k].rscore = d;
			// Normalize it first.  
			tot = ngood+nbad;
			if (tot==0) tot = 1;
			v = 100*stable[k].score/tot;
			if (tot<1000) v -= (1000-tot); /* Strongly favour rare words to decrease false matches */
			stable[k].score = (int) v;
			k++;
		}
	}
	if (nwords>1) qsort(stable,nwords,sizeof(Wsort),aspam_cmpb);

	score = 0;
	sample = 7; 
	if (sample>nwords) sample = nwords;
	strcpy(binfo,"");
	*bscore = 0;
	for (i=0; i<sample; i++) {
		*bscore += stable[i].rscore;
		lprintf(bf,BFSZ-1,"%s=%.2f ",stable[i].word,stable[i].rscore);
		ncat(binfo,bf,BFSZ-1);
	}
	for (i=nwords-1; i>=nwords-sample; i--) {
		*bscore += stable[i].rscore;
		lprintf(bf,BFSZ-1,"%s=%.2f ",stable[i].word,stable[i].rscore);
		ncat(binfo,bf,BFSZ-1);
	}
	if (sample>0) *bscore = *bscore / (sample*2);

	
	
	if (nwords>1) qsort(stable,nwords,sizeof(Wsort),aspam_cmp);
	if (show) for (i=0; i<nwords; i++) {
		english_find2(stable[i].word,&ngood,&nbad);
		if (show) printf("Words: %s %d  ngood=%d nbad=%d\n",stable[i].word,stable[i].score,ngood,nbad);
	}

	lprintf(bf,BFSZ-1,"match: ");
	sample = MAXID; if (sample>nwords) sample=nwords;
	for (i=0; i<sample; i++) {
		if (stable[i].score>1) break;
		ncat(bf,stable[i].word,BFSZ-1);
		ncat(bf," ",BFSZ-1);
	}
	if (show) printf("Chosen rules (%s)\n",bf);
	strlwr(bf);
	word_add(rules,bf);

	lprintf(bf,BFSZ-1,"totwords: %d",totwords);
	word_add(rules,bf);

	if (show) printf("Score for message %d\n",score);
	free(stable);
	aspam_words_free(whash,&kwhash);
	return TRUE;
}
int aspam_non_choose(Wordlist **rules, int isbad)
{
	int i;
	int ngood,nbad;
	char bf[BFSZ];
	Wtable *w;
	int k=0;
	int sample;
	Wsort *stable;
	int nwords=0;
	for (i=0; i<HASHSZ; i++) {
		for (w = wnon[i]; w!=NULL; w=w->next) {
			nwords++;
		}
	}
	if (nwords==0) return FALSE;
	stable = calloc(nwords,sizeof(Wsort));
	for (i=0; i<HASHSZ; i++) {
		for (w = wnon[i]; w!=NULL; w=w->next) {
			stable[k].word = w->word;
			stable[k].score = -w->ngood -w->nbad; /* Frequency of this word in file */
			k++;
		}
	}
	if (nwords>1) qsort(stable,nwords,sizeof(Wsort),aspam_cmp);
	for (i=0; i<nwords; i++) {
		english_find2(stable[i].word,&ngood,&nbad);
//		printf("Words: %s %d  ngood=%d nbad=%d\n",stable[i].word,stable[i].score,ngood,nbad);
	}
	lprintf(bf,BFSZ-1,"non: ");
	sample = MAXID;
	if (sample>nwords) sample = nwords;
	for (i=0; i<sample; i++) {
		ncat(bf,stable[i].word,BFSZ-1);
		ncat(bf," ",BFSZ-1);
	}
	strlwr(bf);
	word_add(rules,bf);

	free(stable);
	aspam_words_free(wnon,&kwnon);
	return TRUE;
}
int aspam_words_file(void)
{
	int i;
	Wtable *w;
	for (i=0; i<HASHSZ; i++) {
		for (w = whash[i]; w!=NULL; w=w->next) {
			aspam_word_add(whash_all,HASHSZ,w->word,w->nbad>0 ? 1 : 0,w->ngood>0 ? 1 : 0,&kwhash_all);
		}
	}
	aspam_words_free(whash,&kwhash);
	return TRUE;
}
int aspam_words_free(Wtable **t,Keeplist **k)
{
	int i;
	Wtable *w,*wnext;
	for (i=0; i<HASHSZ; i++) {
		for (w = t[i]; w!=NULL; w=wnext) {
			wnext = w->next;
//			free(w);
		}
		t[i] = NULL;
	}
	keep_freeall(k);
	return TRUE;
}
int aspam_add_rest(void)
{
//	Go through enlgish words, and add all with score of zero.
	int i;
	Wtable *w;
	for (i=0; i<HASHSZ; i++) {
		for (w = whash_english[i]; w!=NULL; w=w->next) {
			aspam_word_add(whash_all,HASHSZ,w->word,0,0,&kwhash_all);
		}
	}
	return TRUE;
}
int aspam_words_show(void)
{
	int i;
	Wtable *w;
	FILE *f;
	for (i=0; i<HASHSZ; i++) {
		for (w = whash_all[i]; w!=NULL; w=w->next) {
//			printf("%8d %8d %.12s\n",w->nbad,w->ngood,w->word);
		}
	}
	aspam_add_rest();
	f = fopen("words_new.dat","w");
	if (f==NULL) perror("words_new.dat");
	for (i=0; i<HASHSZ; i++) {
		for (w = whash_all[i]; w!=NULL; w=w->next) {
			/* Normalize to 10,000 sample database */
			if (donone) {
				if (w->nbad+w->ngood<3) continue;
			}
			fprintf(f,"%d %d %.12s\n",(int) (w->nbad*10000.0/nbad),(int) (w->ngood*10000.0/ngood),w->word);
		}
	}
	fclose(f);
	return TRUE;
}
int aspam_dump(char *fname)
{
	char out[BFSZ];
	Ruleset *r;
	FILE *f;
	f = fopen(fname,"w");
	if (f==NULL) return FALSE;
	for (r=set_bad; r!=NULL; r=r->next)  {
		fprintf(f,"hits: %d %s\n",r->hits,r->from);
		aspam_rule_show(r,out);
		fprintf(f,"%s\n",out);
	}
	fclose(f);
	return TRUE;
}
int aspam_free(void)
{
	Ruleset *r,*rnext;
	con_exit();
	// Free english, and free ruleset, and hash tables that point to rule sets . 
	for (r=set_bad; r!=NULL; r=rnext)  {
		rnext = r->next;
		aspam_rules_free_one(&r);
	}
	set_bad = NULL;
	// Now free hash tables
	wrule_free(rules_good,RULESZ);
	wrule_free(rules_url,RULESZ);
	wrule_free(rules_from,RULESZ);
	wrule_free(rules_exip,RULESZ);
	wrule_free(rules_match,RULESZ);
	wrule_free(rules_non,RULESZ);
	keep_freeall(&kwrule);
	// Now free english dictionary
	aspam_words_free(whash_english,&kwhash_english);
	aspam_bodyhash_clear();

	return TRUE;
}
static int done_init;
int aspam_init(char *fname, char *rfile, char *reason)
{
//	FILE *f;
//	char bf[BFSZ];
//	char *p[100];
//	int n;
	if (!done_init)  aspam_tld_load();

	done_init = TRUE;
	aspam_free();
	aspam_rules_load(rfile);
	poly_init();
	con_load();
	return TRUE;
	/*
	f = fopen(fname,"r");
	if (f==NULL) {
		lprintf(reason,BFSZ-1,"Dictionary not found (%s)",fname);
		return FALSE;
	}
	for (;!feof(f);) {
		if (fgets(bf,BFSZ-1,f)==NULL) break;
		n = token_split(bf,p," \r\n",20);
		if (n<3) continue;
		if (lib_strlen(p[2])>2) english_add(p[2],atoi(p[0]),atoi(p[1]));
	}
	fclose(f);
	if (show)  printf("Dictionary is now loaded\n");
	return TRUE;*/ 
}
int english_add(char *bf, int nbad, int ngood)
{
	aspam_word_add(whash_english,HASHSZ, bf, nbad,ngood,&kwhash_english);
	return TRUE;
}
int english_find(char *word)
{
	if (aspam_word_find(whash_english,HASHSZ,word)!=NULL) return TRUE;
	return FALSE;	
}
int english_find2(char *word, int *ngood, int *nbad)
{
	Wtable *w;
	*ngood = *nbad = 0;
	w = aspam_word_find(whash_english,HASHSZ,word);
	if (w!=NULL) {
		*ngood = w->ngood;
		*nbad = w->nbad;
		return TRUE;
	}
	return FALSE;	
}

/*

url: infowinners.info 30
url: .132.232 74
non: img mailers casinotrade yfc2 infowinners dbid eid trk lnk 
match: parent gif boca array palace spacer casino language crystal 
from: crystal_palace@mts6-ex.malibumailz1.com

*/
int aspam_rules_load_onex(Ruleset **rr, char *xbf);

int aspam_rules_load_one(Ruleset **rr, Wordlist *wl)
{
	char source[BFSZ];
	char bf[BFSZ];
	Wordlist *w;
	strcpy(source,"");
	for (w=wl;w!=NULL;w=w->next) {
		if (strncmp(w->word,"source: ",8)==0) { // type: good base filename|email@domain
			ncpy(source,w->word,BFSZ-1);
		}
	}
	if (lib_strlen(source)==0) return FALSE;

	aspam_rules_load_onex(rr,source);
	for (w=wl;w!=NULL;w=w->next) {
		if (strncmp(w->word,"source: ",8)!=0) { // type: good base filename|email@domain
			aspam_rules_load_onex(rr,w->word);
		} 
		if (lib_strlen(w->word)==0) continue;
	}
	strcpy(bf,"");
	aspam_rules_load_onex(rr,bf);
	return TRUE;
}
int aspam_rules_load_onex(Ruleset **rr, char *xbf)
{
	char *upto;
	char *s;
	Ruleset *r=*rr;
	char bf[BFSZ];
	ncpy(bf,xbf,BFSZ-1);

	if (r==NULL) *rr = r = calloc(1,sizeof(Ruleset));
	if (strcmp(bf,"")==0) {
		r->next = set_bad;
		set_bad = r;
		*rr = NULL;
		return TRUE;
	}
	if (strncmp(bf,"source: ",8)==0) { // type: good base filename|email@domain
		s = mystrtok(bf," ",&upto);
		s = mystrtok(NULL," ",&upto);
		if (s!=NULL) if (strcmp(s,"bad")==0) r->isbad = TRUE; else r->isbad = FALSE;
		s = mystrtok(NULL," ",&upto);
		if (s!=NULL) r->source = keep_strdup(&kwrule,s);
		s = mystrtok(NULL," ",&upto);
		if (s!=NULL) r->user = keep_strdup(&kwrule,s);
	}
	if (strncmp(bf,"url: ",5)==0) aspam_rules_url(r,bf+5);
	if (strncmp(bf,"totwords: ",10)==0) r->totwords = atoi(bf+10);
	if (strncmp(bf,"non: ",5)==0) aspam_rules_match(r,bf+5,TRUE);
	if (strncmp(bf,"match: ",7)==0) aspam_rules_match(r,bf+7,FALSE);
	if (strncmp(bf,"from: ",6)==0) aspam_rules_from(r,bf+6);
	if (strncmp(bf,"exip: ",6)==0) aspam_rules_exip(r,bf+6);
	if (strncmp(bf,"bodyhash: ",10)==0) aspam_rules_bodyhash(r,bf+10);
	return TRUE;
}
int aspam_rules_load(char *rfile)
{
	/* Add 'url' rules to simple hash table */
	/* Add match,non to linked list tables */
	/* Add from to simple hash table */
	FILE *f;
	int tot = 0;
	char *s;
	int isbad=FALSE;
	char bf[BFSZ];
	Ruleset *r=NULL;
	Wordlist *wl=NULL;
return TRUE; // aspam rules disabled.
	f = fopen(rfile,"rb");
	if (f==NULL) return 0;
	for (;!feof(f);) {
		if (fgets(bf,BFSZ-1,f)==NULL) break;
		s = strchr(bf,'\r'); if (s!=NULL) *s = 0;
		s = strchr(bf,'\n'); if (s!=NULL) *s = 0;
		if (lib_strlen(bf)==0) {
			aspam_rules_load_one(&r,wl);
			word_free(&wl);
			tot++;
		} else {
			word_add(&wl,bf);
		}
	}
	if (wl!=NULL) {
			aspam_rules_load_one(&r,wl);
			tot++;
			word_free(&wl);
	}
	fclose(f);
	if (r!=NULL) aspam_rules_free_one(&r);
	return tot;
}
int aspam_rules_free_one(Ruleset **rr)
{	
	Ruleset *r=*rr;
	if (r==NULL) return TRUE;
//	if (r->from!=NULL) free(r->from);
//	if (r->source!=NULL) free(r->source);
//	if (r->user!=NULL) free(r->user);
	word_free(&r->match);
	word_free(&r->non);
	word_free(&r->urls);
	free(r);
	*rr = NULL;
	return TRUE;
}

int wrule_remove_badurl(char *word);

int aspam_rules_url(Ruleset *r, char *line)
{
	char *s;
	s = strchr(line,' ');
	if (s!=NULL) *s = 0;
	word_add(&r->urls,line);
	if (!r->isbad) { // adding good rule
		wrule_add(rules_good,RULESZ,r,r->urls->word,0);
		/* And remove it from the 'bad' list if it's already in it */
		wrule_remove_badurl(line);
	} else {
		/* If this is bad rule, and url exists in 'good' rules, ignore url */
		if (wrule_find_goodurl(line)) return TRUE;
	}
	r->nurls++;
	wrule_add(rules_url,RULESZ,r,r->urls->word,0);
	return TRUE;
}
int aspam_rules_match(Ruleset *r, char *line, int isnon)
{
	/* Add info to 'r' ruleset, and add hash entries pointing to it */			
	int np;
	char *p[100];
	int i;
	np = token_split(line,p," ",30);

	if (np>9) np = 9;
	for (i=0; i<np; i++) {
		if (isnon) 	{
			if (r->nnon < MAXID) {
				word_add(&r->non,p[i]);
				wrule_add(rules_non,RULESZ,r,r->non->word,i);
				r->nnon++;
			}
		} else {
			if (r->nmatch < MAXID) {
				word_add(&r->match,p[i]);
				wrule_add(rules_match,RULESZ,r,r->match->word,i);
				r->nmatch++;
			}
		}
	}
	return TRUE;
}
#define BHMAX 1000
typedef struct BODYHASH {int h[5]; int nh; struct BODYHASH *next; Ruleset *r;} Bodyhash;
static Bodyhash *bhtable[BHMAX];
static double bodyscore;
int aspam_bodyhash_hash(int x)
{
	x = x % BHMAX;
	if (x<0) x = -x;
	return x;
}
int aspam_score_body(int *h_val,int nh, char *fname )
{
	int h;
	int i;
	Bodyhash *bh;
	int nmatch=0;
	Bodyhash *best=NULL;
	int nbest=0;
	if (nh<3) return FALSE;
	bodyscore = 0;
	h = aspam_bodyhash_hash(h_val[2]);
	for (bh=bhtable[h];bh!=NULL;bh=bh->next) {
		nmatch=0;
		for (i=0; i<nh && i <bh->nh; i++) {
			if (bh->h[i]==h_val[i]) nmatch++;
		}
		if (!bh->r->isbad) nmatch++;
		if (nmatch>nbest) {
			best = bh;
			nbest = nmatch;
		}
	}
	if (nbest+1>=nh) {
		bodyscore = 5;
		//printf("Matched %d/%d, item src=(%s) fname=(%s)\n",nbest,nh,best->r->source,fname);
		if (!best->r->isbad) bodyscore = -5;
	}
	return TRUE;
}

int aspam_bodyhash_clear(void)
{
	int i;
	for (i=0; i<BHMAX; i++) bhtable[i] = NULL;
	return TRUE;
}
int aspam_rules_bodyhash(Ruleset *r, char *line)
{
	/* Add info to 'r' ruleset, and add hash entries pointing to it */			
	int np;
	char *p[100];
	int i;
	int h;
	Bodyhash *bh;
	r->bodyhash = keep_strdup(&kwrule,line);
	np = token_split(line,p," ",30);

	if (np>9) np = 9;
	if (np<3) return TRUE;
	h = aspam_bodyhash_hash(atoi(p[2]));

	// Insert into linked list for bodies...
	bh = keep_malloc_align(sizeof(Bodyhash),&kwrule);
	bh->r = r;
	bh->next = bhtable[h];
	bhtable[h] = bh;
	for (i=0; i<np && i<5; i++) {
		bh->h[i] = atoi(p[i]);
		bh->nh = i+1;
	}
	return TRUE;
}
int aspam_rules_from(Ruleset *r, char *line)
{
	char *s;
	s = strchr(line,' ');
	if (s!=NULL) *s = 0;
	if (r->from==NULL) r->from = keep_strdup(&kwrule,line);
	wrule_add(rules_from,RULESZ,r,r->from,0);
	return TRUE;
}
char *aspam_net(char *line)
{
	char *s;
	s = strchr(line,'.');
	if (s!=NULL) s = strchr(s+1,'.');
	if (s!=NULL) s = strchr(s+1,'.');
	if (s!=NULL) *s = 0; /* Only compare network part of it */
	return line;
}

int aspam_rules_exip(Ruleset *r, char *line)
{
	char *s;
	s = strchr(line,' ');
	if (s!=NULL) *s = 0;
	aspam_net(line);
	if (r->exip==NULL) r->exip = keep_strdup(&kwrule,line);
	wrule_add(rules_exip,RULESZ,r,r->exip,0);
	return TRUE;
}

int wrule_add(Wrule **table,int hashsz, Ruleset *r, char *word, int id)
{
	Wrule *w;	
	int h; 

	h = lib_str_hash(word,hashsz);
	w = keep_malloc_align(sizeof(Wrule),&kwrule);
	w->word = word;
	w->id = id;
	w->rule = r;
	w->next = table[h];
	table[h] = w;
	return TRUE;
}
int wrule_free(Wrule **table,int hashsz)
{
	Wrule *w,*wnext;	
	int h; 
	for (h=0; h<hashsz; h++) {
		for (w=table[h];w!=NULL;w=wnext) {
			wnext = w->next;
//			free(w);
		}
		table[h] = NULL;
	}
	return TRUE;
}
int wrule_find_match(char *word)
{
	Wrule *w;	
	int h; 

	h = lib_str_hash(word,RULESZ);
	for (w=rules_match[h];w!=NULL;w=w->next) {
		if (strcmp(w->word,word)==0) {
			if (w->id<0) lib_abort();
			if (w->id>=MAXID) lib_abort();
			w->rule->mask[w->id] = TRUE;
		}
	}
	return TRUE;
}
int wrule_find_non(char *word)
{
	Wrule *w;	
	int h; 

	h = lib_str_hash(word,RULESZ);
	for (w=rules_non[h];w!=NULL;w=w->next) {
		if (strcmp(w->word,word)==0) {
			if (w->id<0) lib_abort();
			if (w->id>=MAXID) lib_abort();
			w->rule->mask_non[w->id] = TRUE;
//			printf("found_non (%s) id=%d %s\n",word,w->id,w->rule->user);
		}
	}
	return TRUE;
}
int wrule_find_from(char *word)
{
	Wrule *w;	
	int h; 

	h = lib_str_hash(word,RULESZ);
	for (w=rules_from[h];w!=NULL;w=w->next) {
		if (strcmp(w->word,word)==0) {
			w->rule->nfrom++;
			w->rule->isfrom = TRUE;	
		}
	}
	return TRUE;
}
int wrule_find_exip(char *xword)
{
	Wrule *w;	
	int h; 
	char word[BFSZ];

	ncpy(word,xword,BFSZ-1);
	aspam_net(word);
	h = lib_str_hash(word,RULESZ);
	for (w=rules_exip[h];w!=NULL;w=w->next) {
		if (strcmp(w->word,word)==0) {
//printf("Match found (%s) %s %s \n",word,w->rule->from,w->rule->user);
			w->rule->nexip++;
			w->rule->isexip = TRUE;	
		}
	}
	return TRUE;
}
int wrule_find_url(char *word)
{
	Wrule *w;	
	int h; 

	h = lib_str_hash(word,RULESZ);
	for (w=rules_url[h];w!=NULL;w=w->next) {
		if (strcmp(w->word,word)==0) {
			w->rule->nurl_found++;
		}
	}
	return TRUE;
}
int wrule_find_goodurl(char *word)
{
	Wrule *w;	
	int h; 

	h = lib_str_hash(word,RULESZ);
	for (w=rules_good[h];w!=NULL;w=w->next) {
		if (strcmp(w->word,word)==0) return TRUE;
	}
	return FALSE;
}
int wrule_remove_badurl(char *word)
{
	Wrule *w;	
	int h; 
	int n;

	h = lib_str_hash(word,RULESZ);
	for (w=rules_url[h];w!=NULL;w=w->next) {
		if (strcmp(w->word,word)==0) {
			n = lib_strlen(w->word);
			if (n>4) {
				strcpy(w->word,"DEL");
			}
		}
	}
	return FALSE;
}
int aspam_scan_rules(double *ret_score, Wordlist **result)
{
	int i;
	Wtable *w;
	Utable *u;
	int k=0;
	int nwords=0;
	for (i=0; i<HASHSZ; i++) {
		for (w = whash[i]; w!=NULL; w=w->next) {
			wrule_find_match(w->word);
		}
	}
	for (i=0; i<HASHSZ; i++) {
		for (w = wnon[i]; w!=NULL; w=w->next) {
			wrule_find_non(w->word);
		}
	}

	// Next urls	
	for (i=0; i<URLSZ; i++) {
		for (u=urls[i];u!=NULL;u=u->next) {
			wrule_find_url(u->word);
		}
	}	
	
	wrule_find_from(from);
	wrule_find_exip(exip);


	/* Now go through rules and see what scores were */
	aspam_rules_analyze(ret_score,result);

	/* now reset scores*/ 
	aspam_rules_reset();

	/* And free the whash word table for this item. */
	aspam_words_free(whash,&kwhash);
	aspam_words_free(wnon,&kwnon);
	return TRUE;
}
static int insist_exip;
int aspam_need_exip(int x)
{
	insist_exip = x;
	return TRUE;
}
int aspam_rule_score(Ruleset *r, double *score_words, double *score_non, double *score_from, double *score_urls, double *score_empty, double *score_exip, double *score)
{
	int i;
	int nm;
	int nowords=FALSE;
	nm = 0;
	*score = 0; *score_words = 0; *score_non = *score_urls = 0;
	for (i=0; i<MAXID; i++) if (r->mask[i]) nm++;
	if (r->nmatch>0) {
		*score_words = (double) nm/(double) r->nmatch*8-3;
		if (nm<r->nmatch) *score_words = *score_words / 2; // Scale down imperfect matches. 
		if (r->nmatch<4) *score_words = *score_words * (r->nmatch/4.0);
		if (totwords>20) if (r->nmatch<7) *score_words = *score_words/2;
		if (*score_words < 1.6) *score_words = 0; // If only a few match, then we don't want to know
	}
	if (totwords>20) if (r->nmatch==0) nowords = TRUE;
	// Require 'bad' matches to be fairly confident with words 
//	if (r->isbad) if (*score_words < 1) *score_words = 0; // Not a significant match
	*score = 0;
	nm = 0;
	for (i=0; i<MAXID; i++) if (r->mask_non[i]) nm++;
	if (r->nnon>0) *score_non = (double) nm/r->nnon*6-3;
	if (*score_non>0) {
		if (r->nnon<4) *score_non = *score_non * (r->nnon/4.0);	
		if (*score_non < 2) *score_non = 0;
		*score += *score_non;
	}
	*score_from = 0;
	if (r->isfrom) *score_from = 1;

	*score_exip = 0;
	if (r->isexip) *score_exip = 2;


	*score += *score_from + *score_exip;
	if (r->nurls>0) {
		*score_urls = (double) r->nurl_found/r->nurls*2.5;
		if (r->nurl_found > 0) if (*score_urls<3) *score_urls = 3;
	}
	// urls don't count much towards score for 'good' matches since they are too
	// likely to be meaningless or even bad (like yahoo.com) 
	if (!r->isbad) *score_urls = *score_urls/4.0; 
	*score_urls = *score_urls * url_scale;
	*score += *score_urls;

	// If no other matches, then scale down word matches if this message is long
	// and original sample was small
	if (*score_urls==0) if (totwords>30) if (r->nmatch<8) if (*score_words>1) {
		*score_words = *score_words / 2;
	}
	if (r->nurl_found==0) if (totwords>60) if (r->nmatch<8) if (*score_words>0) {		
		*score_words = *score_words/2;
		if (nm < r->nmatch) {	
			if (totwords>120) *score_words = *score_words/2;
		}
	}
	*score += *score_words;

	if ( (*score_from <=0) && (*score_non<0) && (*score_urls<=0) && (r->nmatch<4) && (*score_words < 2) ) {
		*score -= *score_words/2;  // Half the affect of word matches if no other matching info
	}

	*score_empty = 0;
	// if no words in message but non words matched well, then increase score.
	if (nowords) if (*score_non>2) *score_empty = 1.5;

	if (totwords<5) if (r->nnon+r->nmatch<5) if (*score_urls>0 || *score_from>0 || *score_exip>0) {
		*score_empty = 3;
	}

	*score += *score_empty;

	// For good matches, we must have an exip match, if not then scale down the match
	if (insist_exip) {
		if (!r->isbad) {
			if (!r->isexip) {
				*score = *score / 2; 
				if (*score > 6) *score = 6;
			}
		}
	}
	return TRUE;
}
int aspam_rule_show(Ruleset *r, char *out)
{
	int ret;
	Wordlist *w;
	Wordlist *result=NULL;
	ret = aspam_rule_showr(r,&result);
	strcpy(out,"");
	for (w=result;w!=NULL;w=w->next) {
		ncat(out,w->word,BFSZ-1);
		ncat(out,"\n",BFSZ-1);
	}
	word_free(&result);
	return ret;
}
int aspam_rule_showr(Ruleset *r, Wordlist **result)
{
	char bf[BFSZ];
	double words,non,from,urls,total,empty,exip;
	Wordlist *w;
	char *t;
	char tmp[BFSZ];
	int i;
	aspam_rule_score(r,&words,&non,&from,&urls,&empty,&exip,&total);

	lprintf(bf,BFSZ-1,"X-Aspam: Words %.1f ",words);
	for (w=r->match,i=r->nmatch-1; i>=0 && w!=NULL; i--,w=w->next) {
		t = "-";
		if (i<0) lib_abort();
		if (i>=MAXID) lib_abort();
		if (r->mask[i]) t = "+";
		lprintf(tmp,BFSZ-1,"%s%.12s ",t,w->word);
		ncat(bf,tmp,BFSZ-1);
	}
	word_add(result,bf);
	if (non>0) {
		lprintf(bf,BFSZ-1,"X-Aspam: NonWords %.1f ",non);
		for (w=r->non,i=r->nnon-1; i>=0 && w!=NULL; i--,w=w->next) {
			t = "-";
			if (i<0) lib_abort();
			if (i>=MAXID) lib_abort();
			if (r->mask_non[i]) t = "+";
			lprintf(tmp,BFSZ-1,"%s%.12s ",t,w->word);
			ncat(bf,tmp,BFSZ-1);
		}
		word_add(result,bf);
	}
	if (urls>0) {
		lprintf(bf,BFSZ-1,"X-Aspam: URLS scored %.1f ",urls);
		for (w=r->urls;w!=NULL;w=w->next) {
			ncat(bf,w->word,BFSZ-1);
			ncat(bf," ",BFSZ-1);
		}
		word_add(result,bf);
	}
	if (exip>0) {
		lprintf(bf,BFSZ-1,"X-Aspam: External-IP matched %s %.1f",nullok(r->exip),exip);
		word_add(result,bf);
	}
	if (from>0) {
		lprintf(bf,BFSZ-1,"X-Aspam: From %.1f (%s)",from,r->from);
		word_add(result,bf);
	}
	lprintf(bf,BFSZ-1,"X-Aspam: Best match was sample %s",win_slash(r->user,tmp));
	word_add(result,bf);
	if (empty>0) {
		lprintf(bf,BFSZ-1,"X-Aspam: Only %d words in msg scores %.1f",totwords,empty);
		word_add(result,bf);
	}
	lprintf(bf,BFSZ-1,"X-Aspam: Total %.1f",total);
	word_add(result,bf);
	return TRUE;
}
int aspam_rules_analyze(double *ret_score, Wordlist **result)
{
	Ruleset *r;
	Ruleset *bestgood=NULL;
	Ruleset *bestbad=NULL;
	int nm=0;
	char out[BFSZ];
	double good_score= 0;
	double bad_score= 0;
	double words,non,from,urls,total,empty,exip;
	for (r=set_bad; r!=NULL; r=r->next)  {
		aspam_rule_score(r,&words,&non,&from,&urls,&empty,&exip,&total);
		if (r->isbad) {
			if (total>bad_score) {
				bestbad = r;
				bad_score = total;
			}
		} else {
			if (total>good_score) {
				bestgood = r;
				good_score = total;
			}
		}
//		aspam_rule_show(r,out); printf("\n%s",out);
	}

//	printf("Scores bad=%.1f good=%.1f\n",bad_score,good_score);
	if (good_score*2 < bad_score) {
		global_score = bad_score;
		if (bestbad!=NULL) {
			bestbad->hits++;
			aspam_rule_show(bestbad,out); 
			if (show) printf("%s",out);
			aspam_rule_showr(bestbad,result);
		}
	} else {
		if (good_score > 5.8) good_score = 20;
		global_score = -good_score; /* Scaled up so 'good' matches are effective */
		if (show) printf("No rules matched sufficently well\n");
		if (bestbad!=NULL) {aspam_rule_show(bestbad,out); if (show) printf("Bestbad\n%s",out);}
		if (bestgood!=NULL) {aspam_rule_show(bestgood,out); if (show) printf("Bestgood\n%s",out);
				aspam_rule_showr(bestgood,result);
	
		}
	}
	if (bodyscore!=0) {
		char bf[BFSZ];
		if (show) printf("Body Score %g\n",bodyscore);
		lprintf(bf,BFSZ-1,"BodyScore: %g",bodyscore);
		word_add(result,bf);
		global_score += bodyscore; 
	}
	*ret_score = global_score;
	if (show) printf("\n");
	return TRUE;
}
int aspam_rules_reset(void)
{
	Ruleset *r;
	int i;
	for (r=set_bad; r!=NULL; r=r->next)  {
		for (i=0; i<MAXID; i++) {
			r->mask[i] = 0;
			r->mask_non[i] = 0;
		}
		r->isfrom = FALSE;
		r->nurl_found = 0;
		r->nexip = 0;
		r->isexip = FALSE;	
	}
	return TRUE;
}
/*-----------------------------------------------------------------------------*/
/*--------- New Poly word matching statistical method, simplified :-) ---------*/
/*-----------------------------------------------------------------------------*/
#define POLY_VERSION 2
int poly_word(int isbad, int train, char *wrd, double *score_good, double *score_bad, int *ns_good, int *ns_bad);
int poly_scan_data(Chan *c,char *msg, int isbad,double *ret_score, int istrain);
int poly_scan(Chan *c,char *fname, int isbad, double *ret_score, int istrain)
{
	Mime *m;
	FILE *f=NULL;
	int k;
	int nlines=0;
	char raw[BFSZ];
	char bin[BFSZ];
	static char big[MAXBIG+10]; // Ok because mutex is held when poly is used. 
	int len=0;
	int ret=FALSE;
	int inhead=TRUE;
	int bigsz=0;
	int drop=FALSE;
	char subj[BFSZ];


	*ret_score = 0.5;
	return TRUE;

	if (!done_init) return TRUE;
	mime_new(&m);
	if (mem!=NULL) mime_setmem(m,mem,mem_len);
	else {
		f = fopen(fname,"rb");
		if (f==NULL) {
			mime_free(&m);
			return FALSE;
		}
		setvbuf(f,NULL,_IOFBF,32000);
	}
	for (k=TRUE;k;) {
		int i;
		if (nlines>300) break;
		nlines++;
		k = mime_line(m,f,raw,bin,&len,BFSZ-1);
		switch (k) {
		case MIME_END:
			//if (strncmp(raw,"Subject: ",9)==0) {
			//	ncpy(subj,raw+9,100);
			//}
			break;
		case MIME_CONTENT:
			break;
		case MIME_HEADER_END:
			inhead = FALSE;
			drop = FALSE;
			strlwr(m->content);
			if (strstr(m->content,"image")!=NULL) drop = TRUE;
			if (strstr(m->content,"applicaton")!=NULL) drop = TRUE;
			if (strstr(m->content,"audio")!=NULL) drop = TRUE;
			if (strstr(m->content,"video")!=NULL) drop = TRUE;
			break;		
		case MIME_HEADER:
			len = lib_strlen(raw);
			ncpy(bin,raw,BFSZ-1);
			if (strncmp(raw,"X-SpamDetect:",13)==0) break;
			if (strncmp(raw,"X-Avast:",8)==0) break;
			if (strncmp(raw,"X-Aspam:",8)==0) break;
		case MIME_BODY:
			// Should buffer up 10K so we get multi line urls'
			if (drop) break;
			if (strncmp(bin,"X-SpamDetect",12)==0) break;
			if (len<0) len = 0;
			if (len>=0) bin[len] = 0;
			for (i=0; i<len; i++) {
				if (bin[i]==0) bin[i] = ' ';
			}
//			strlwr(bin);
			len = lib_strlen(bin); /* get rid of nulls  - this might let spam through if they are smart */

			if (len<500) if (m->encoding != MIME_BASE64) {
				strcat(bin," "); len++;
			}
			if ((len+bigsz+BFSZ+1000)<MAXBIG) {
				safe_memcpy(big+bigsz,bin,len,big,MAXBIG);
				bigsz = bigsz + len;
				big[bigsz] = 0;
			} 
			break;
		case MIME_BOUNDARY:
			break;
		}
	}
	msg_aspam_at("scanc1");
	if (f!=NULL) fclose(f); f = NULL;
	msg_aspam_at("scanc1.2");
	mime_free(&m);

	big[bigsz] = 0;
	ncat(big,"\r\n",MAXBIG-1);
	ncat(big,subj,MAXBIG-1);
	strlwr(big);
	bigsz += lib_strlen(subj)+2;
	msg_aspam_at("scanc2");
	poly_scan_data(c,big,isbad,ret_score,istrain);
	msg_aspam_at("scanc3");
	return TRUE;
}
#define QSIZE 5
#define TABLESZ 1000000
typedef short table_type;
#define TABLE_MAX 32000 // wAS 250 
static int skip[256];
static unsigned char *pbad,*pgood;
static unsigned int wque[QSIZE+4];
static int nwque;
// These are set to '50' to start so that with very small sample sizes the importance
// of individual rules is not over emphasized.  

int afix_int(int *x)
{
	char swap[4];
	char swap2[4];
	int z = 0x01020304;
	memcpy(swap,&z,4);
	if (swap[0]==1) {
		memcpy(swap,x,4);
		swap2[0] = swap[3];
		swap2[1] = swap[2];
		swap2[2] = swap[1];
		swap2[3] = swap[0];
		memcpy(x,swap2,4);
	} else {
#ifdef sparc
		lib_abort();
#endif
	}
	return TRUE;
}
void poly_exit(void)
{
	imsg("poly_exit\n");
	if (pbad) free(pbad); pbad = NULL; save_pbad = NULL;
	if (pgood) free(pgood); pgood = NULL; save_pgood = NULL;
}

int poly_init(void)
{
	// Read in data from disk 	
	FILE *f;
	int ver;
	int i;
	int ng,nb,nb2,ng2;
	if (pbad==NULL) save_pbad = pbad = calloc(sizeof(table_type),TABLESZ);
	if (pgood==NULL) save_pgood = pgood = calloc(sizeof(table_type),TABLESZ);
	f = fopen("poly_good.dat","rb");
	if (f!=NULL) {
		fread(&ver,sizeof(int),1,f); 
		fread(&goodsz,sizeof(int),1,f);
		if (fread(pgood,sizeof(table_type),TABLESZ,f)!=TABLESZ) ver=99;
		fclose(f);
		if (ver!=POLY_VERSION) memset(pgood,0,TABLESZ*sizeof(table_type));
	}
	if (goodsz<2) {
		f = fopen("aspam_pgood.dat","rb");
		if (f!=NULL) {
			fread(&ver,sizeof(int),1,f);  afix_int(&ver);
			fread(&goodsz,sizeof(int),1,f); afix_int(&goodsz);
			if (fread(pgood,sizeof(table_type),TABLESZ,f)!=TABLESZ) ver=99;
			fclose(f);
			imsg("aspam: Read aspam_pgood.dat successfully - ver = %d\n",ver);
		}
		if (ver!=POLY_VERSION) memset(pgood,0,TABLESZ*sizeof(table_type));
	}
	f = fopen("poly_bad.dat","rb");
	if (f!=NULL) {
		fread(&ver,sizeof(int),1,f); //afix_int(&ver);
		fread(&badsz,sizeof(int),1,f); //afix_int(&badsz);
		fread(pbad,sizeof(table_type),TABLESZ,f);
		fclose(f);
		if (ver!=POLY_VERSION) memset(pbad,0,TABLESZ*sizeof(table_type));
	}
	if (badsz<2) {
		f = fopen("aspam_pbad.dat","rb");
		if (f!=NULL) {
			fread(&ver,sizeof(int),1,f); afix_int(&ver);
			fread(&badsz,sizeof(int),1,f); afix_int(&badsz);
			if (fread(pbad,sizeof(table_type),TABLESZ,f)!=TABLESZ) ver=99;
			fclose(f);
			if (ver!=POLY_VERSION) memset(pbad,0,TABLESZ*sizeof(table_type));
			imsg("aspam: Read aspam_pbad.dat successfully - ver = %d\n",ver);
		}
	}
	if (ver!=POLY_VERSION) badsz = 1;
	if (ver!=POLY_VERSION) goodsz = 1;
	imsg("aspam loaded ver = %d (wanted %d) goodsz=%d badsz=%d\n",ver,POLY_VERSION,goodsz,badsz);
	ng = nb = nb2 = ng2 = 0;
	for (i=0; i<TABLESZ; i++) {
		if (pbad[i]!=0) nb++;
		if (pgood[i]!=0) ng++;
		if (pbad[i]>1) nb2++;
		if (pgood[i]>1) ng2++;
	}
	imsg("aspam loaded ver = %d goodsz=%d badsz=%d  nb=%d ng=%d nb2=%d ng2=%d\n",ver,goodsz,badsz,nb,ng,nb2,ng2);
	return TRUE;
}

int poly_reset(void)
{
	if (pbad==NULL) return TRUE;
	if (pgood==NULL) return TRUE;
	// Blank arrays and file ready for training from scratch. 
	goodsz = badsz = 1;
	imsg("poly_reset called, pbad=%p was %p,  pgood=%p, was %p\n",pbad,save_pbad,pgood,save_pgood);
	memset(pbad,0,TABLESZ*sizeof(table_type));
	memset(pgood,0,TABLESZ*sizeof(table_type));
	imsg("poly_reset finished\n");
	remove("poly_good.dat");
	remove("poly_bad.dat");
	return TRUE;
}
int poly_write(void)
{
	FILE *f;
	int ver=POLY_VERSION;
	if (pbad==NULL) save_pbad = pbad = calloc(sizeof(table_type),TABLESZ);
	if (pgood==NULL) save_pgood = pgood = calloc(sizeof(table_type),TABLESZ);
	f = fopen("poly_good.dat","wb");
	if (f!=NULL) {
		fwrite(&ver,sizeof(int),1,f);
		fwrite(&goodsz,sizeof(int),1,f);
		fwrite(pgood,sizeof(table_type),TABLESZ,f);
		fclose(f);
	}
	f = fopen("poly_bad.dat","wb");
	if (f!=NULL) {
		fwrite(&ver,sizeof(int),1,f);
		fwrite(&badsz,sizeof(int),1,f);
		fwrite(pbad,sizeof(table_type),TABLESZ,f);
		fclose(f);
	}
	return TRUE;
}
int poly_scan_data(Chan *chan,char *msg, int isbad,double *ret_score, int istrain)
{
	/* Tokenize it into words, probably not critical how we do this.  
	*/
	int wlen=0;
	int i;
	int nmiss,nhit;
	int nr = 0;
	char wrd[BFSZ];
	double score=0;
	int inword = FALSE;
	unsigned char c;
	unsigned char *umsg=(unsigned char *) msg;
	double score_good,score_bad;
	int ns_good,ns_bad;
	ns_good = ns_bad = 0;
	score_good = score_bad = 0;

	if (!done_init) return TRUE;
	nmiss = nhit = 0;
	if (istrain) if (isbad) badsz++; else goodsz++;
	*ret_score = 0;
	if (pbad==NULL) save_pbad = pbad = calloc(sizeof(table_type),TABLESZ);
	if (pgood==NULL) save_pgood = pgood = calloc(sizeof(table_type),TABLESZ);
	for (i=0; i<255; i++) {
		if (!sisalnum(i)) skip[i] = TRUE;
		if (i=='$') skip[i] = FALSE;
		if (i=='%') skip[i] = FALSE;
		if (i=='.') skip[i] = FALSE;
	}
	nwque = 0;
	msg_aspam_at("scane1");

	for (;TRUE;) {
		c = *umsg++;
		if (c==0) break;
		if (inword) {
			if (skip[c]) { /* end of the word */
				wrd[wlen] = 0;
				msg_aspam_at("scane2");
				
				poly_word(isbad,istrain,wrd,&score_good,&score_bad,&ns_good,&ns_bad);
				msg_aspam_at("scane3");
				wlen = 0;
				inword = FALSE;
			} else {
				if (wlen<100) wrd[wlen++] = c;
			}
		} else {
			if (!skip[c]) { /* end of the gap */
				wrd[0] = c; wlen = 1;
				inword = TRUE;
			}			
		}		
	}
	// Normalize probability.
	if (chan!=NULL) tcp_printf(chan," good %3.2g/%d bad %3.2g/%d \n",score_good,ns_good,score_bad,ns_bad);

	nr = ns_good + ns_bad;
	//if (ns_good>0) score_good = score_good / ns_good;
	//if (ns_bad>0) score_bad = score_bad / ns_bad;
	//if (chan!=NULL) tcp_printf(chan," good %3.2g bad %3.2g\n",score_good,score_bad);

	// Now I want to 'double' the significance of 'good' matches...

	*ret_score = 0.5;
	if (nr>0) *ret_score = (score_good + score_bad) / nr;
	if (chan!=NULL) tcp_printf(chan," Result normalized to %3.2g \n",*ret_score);
	
	// if 


/* Translation of poly score to 'spam' score....
	if (poly_score < 0.3) return -1.5;
	if (poly_score > 0.65) return 1.0;
	if (poly_score > 0.75) return 2.0;
*/
	return TRUE;
}
int poly_word(int isbad, int train, char *wrd, double *score_good, double *score_bad, int *ns_good, int *ns_bad)
{
	// Add this word to the queue, and if we have '5' then start processing it.
	int h = lib_str_hash(wrd,1000000123);
	unsigned int t;
	int nhit,nmiss;
	int ns=0;
	double score=0;
	int i;
	int k;
	double g,b;
	unsigned int tot=0;
	unsigned char *table=pgood;
	unsigned char *otable=pbad;

	nmiss = nhit = 0;

//printf("Poly word (%s)\n",wrd);
	if (isbad) {
		table = pbad;
		otable = pgood;
	}
	wque[nwque++] = h;
	if (nwque<QSIZE) return FALSE;
//	for (i=0; i<nwque;i++) printf("%d ",wque[i]);
//	printf("\n");
	for (i=0; i<16; i++) {
		tot = wque[0];
		for (k=0; k<4; k++) {
			if (1<<k & i) {
				tot += wque[k+1]+(k*23); 
			}
		}
		t = (unsigned int) tot % TABLESZ;
		if (train) {
			if (table[t]<TABLE_MAX) table[t]++;
			else { // Else take one off the other table
				if (otable[t]>0) {
					otable[t]--;		
				}
			}
		} else {
			double z;
			g = (double) pgood[t]/goodsz;
			b = (double) pbad[t]/badsz;
			if (g==0) if (b==0)  { (nmiss)++; continue;}
			(nhit)++;
			if (g==0) {
				z = 0.99;
			}
			else if (b==0) z = 0.01; 
			else z = (double) b/(g+b);
//printf("score %3.2g b=%g g=%g\n",z,b,g);
			if (z>.6) {(*score_good) += z; (*ns_good)++;}
			if (z<.4) {(*score_bad) += z; (*ns_bad)++;}
		}
	}	

	if (nhit>0) score = score / nhit;  // Average them. 
	if (nwque>=QSIZE) {
		for (i=0; i<QSIZE-1; i++ ) wque[i] = wque[i+1];
		nwque--;
	}
	return TRUE;
}

/*-----------------------------------------------------------------------------*/

/*

init()
Read in A list of 'key word' linked lists. 

scan()
Clear list of flags for them.

for each word()
	Find all matches in hash table, and set flags.

done scan()
	Go through and for any set that are all found, add to total score.
	return resulting score. 


Add a url fetch to fetch this data file from netwinsite.
Also load in a 'local' one. 



*/

/*-----------------------------------------------------------------------------*/

// File format
// ID score word word word word "two words" 
#define MAXWLIST 20
typedef struct RULE {int id; double score; Wordlist *wlist; int nwlist; int flags[MAXWLIST];int at[MAXWLIST]; struct  RULE *next;} Rule;
typedef struct PHRASE {char *name; char *phrase; int phrase_len; struct  PHRASE *next;} Phrase;
static Rule *conrules;
typedef struct RHASH {char *word; int wi; int word_len; Rule *rule; struct RHASH *next;} Rhash;
#define RHASHSZ 1999
static Rhash *conhash[RHASHSZ];
static Phrase *phrasehash[RHASHSZ];
int con_hash(char *s)
{
	char bf[BFSZ];
	ncpy(bf,s,8);
	for (s=bf;*s!=0;s++) {
		if (!sisalnum(*s)) {*s = 0; break;}
	}
	strlwr(bf);
	return lib_str_hash(bf,RHASHSZ);
}
int con_load_file(char *fname)
{
	Rule *r;
	FILE *f;
	Rhash *b;
	Wordlist *w;
	int i;
	int h;
	char bf[BFSZ];
	char *p[200];
	int np;
	f = fopen(fname,"r");
	if (f==NULL) return FALSE;
	for (;!feof(f);) {
		if (fgets(bf,BFSZ-1,f)==NULL) break;
		if (*bf=='#') continue;
		np = quoted_token_split_rm_quot(bf,p," \r\n",99);
		if (np<3) continue;
		if (atoi(p[0])==0) continue; // bad line 
		r = calloc(1,sizeof(Rule));
		r->id = atoi(p[0]);
		r->score = atof(p[1]);
		for (i=2;i<np;i++) {
//printf("con rule [%d] (%s)\n",i,p[i]);
			if (r->nwlist+1>=MAXWLIST) break; // discard rest of line.
			word_add(&r->wlist,strlwr(p[i]));
			r->nwlist++;
		}
		r->next = conrules;
		conrules = r;
		for (w=r->wlist,i=0; i<r->nwlist; i++,w=w->next) {
			h = con_hash(w->word);
			b = calloc(1,sizeof(Rhash));
			b->rule = r;
			b->word = w->word;
			b->word_len = lib_strlen(w->word);
			b->wi = i;
			b->next = conhash[h];	
			conhash[h] =  b;
		}
	}
	fclose(f);
	return TRUE;
}
int phrase_load_file(char *fname)
{
	Phrase *r;
	FILE *f;
	int i;
	int h;
	char bf[BFSZ];
	char *p[200];
	int np;
	f = fopen(fname,"r");
	if (f==NULL) return FALSE;
	for (;!feof(f);) {
		if (fgets(bf,BFSZ-1,f)==NULL) break;
		if (*bf=='#') continue;
		np = quoted_token_split_rm_quot(bf,p," \r\n",99);
		if (np<2) continue;

		for (i=1;i<np;i++) {
			r = calloc(1,sizeof(Phrase));
			r->name = strdup(p[0]);
			r->phrase = strdup(p[i]);
			r->phrase_len = lib_strlen(r->phrase);
			h = con_hash(r->phrase);
			r->next = phrasehash[h];
			phrasehash[h] =  r;
		}
	}
	fclose(f);
	return TRUE;
}
int phrase_exit(void)
{
	Phrase *r,*bnext;
	int h;
	for (h=0; h<RHASHSZ; h++) {
		for (r = phrasehash[h];r!=NULL;r=bnext) {
			bnext = r->next;
			free(r->name);
			free(r->phrase);
			free(r);
		}
		phrasehash[h] = NULL;
	}
	return TRUE;
}
int con_exit(void)
{
	Rule *r,*next;
	int h;
	Rhash *b,*bnext;
	phrase_exit();
	for (r=conrules;r!=NULL;r=next) {
		next = r->next;
		word_free(&r->wlist);
		free(r);
	}
	conrules = NULL;
	for (h=0; h<RHASHSZ; h++) {
		for (b = conhash[h];b!=NULL;b=bnext) {
			bnext = b->next;
			free(b);
		}
		conhash[h] = NULL;
	}
	return TRUE;
}
int con_load(void)
{
	con_exit(); // Free everything.
	con_load_file("aspam_content.txt");
	con_load_file("aspam_content_local.txt");
	phrase_load_file("aspam_phrase.txt");
	return TRUE;
}
int con_reset(void)
{
	// Reset all the scores before scanning a message.
	Rule *r;
	int i;

	for (r=conrules;r!=NULL;r=r->next) {
		for (i=0; i<r->nwlist; i++) {
			r->flags[i] = 0;  
			r->at[i] = 0;
		}
	}
	return TRUE;
}
int phrase_scan(char *word, int at, char *found, char *phrase)
{
	int h;
	Phrase *r;

	h = con_hash(word);	
	for (r = phrasehash[h];r!=NULL;r=r->next) {

		if (strncmpnc(r->phrase,word,r->phrase_len)==0) {
			if (!sisalnum(word[r->phrase_len])) {
				ncpy(found,r->name,BFSZ-1);
				ncpy(phrase,r->phrase,BFSZ-1);
				return TRUE;
			}
		}	
	}
	return FALSE;
}	
int con_scan(char *word, int at)
{
	int h;
	Rhash *b;
	Rule *r;
	int found=FALSE;

	h = con_hash(word);	
	for (b = conhash[h];b!=NULL;b=b->next) {
		if (strncmpnc(b->word,word,b->word_len)==0) {
			if (!sisalnum(word[b->word_len])) {
				r = b->rule;
				r->flags[b->wi] = TRUE;
//imsg("Found (%s) (%s) rid=%d\n",word,b->word,r->id);
				r->at[b->wi] = at;
				found = TRUE;
//	printf("Found (%.10s) \n",b->word);
			}
		}	
	}
//	printf("Not found (%.10s) \n",word);
	return TRUE;
}	
int con_score(double *val, char *reason)
{
	Rule *r;
	double v; 
	Wordlist *w;
	int nm=0;
	char bf[BFSZ];
	int i;
	int first;
	int dist;
	strcpy(bf,"");
	strcpy(reason,"");
	*val = 0;
	for (r=conrules;r!=NULL;r=r->next) {
		int nhit=0;
		v = 0;
		first = 0;
		nm = 0;
		for (i=0; i<r->nwlist; i++) {
			if (r->flags[i]) {
				nhit++;
				// Now we check if the word was 'near' the other words, this won't alwawys work.
				if (first==0) first = r->at[i];
				dist = first - r->at[i];
				if (dist<0) dist = -dist;
				if (dist<300) nm++;
			}
		}
		if (nhit==0) continue; // nothing matched.
		if (nm==r->nwlist) v = r->score;
		if (r->nwlist>7) {
			if (nm==r->nwlist-1) v = r->score * 0.7;
			if (nm==r->nwlist-2) v = r->score * 0.5;
		} else {
			if (nm==r->nwlist-1) v = r->score * 0.5;
		}
		if (r->nwlist>9) {
			if (nm==r->nwlist-1) v = r->score * 0.7;
			if (nm==r->nwlist-2) v = r->score * 0.6;
			if (nm==r->nwlist-3) v = r->score * 0.5;
			if (nm==r->nwlist-4) v = r->score * 0.4;
		}
		if (v>0) {
			char match[BFSZ];
			*val += v;
			lpcat(bf,BFSZ-1,"cid=%d(%2.1f) ",r->id,v);
			strcpy(match,"");
			for (i=0,w=r->wlist; i<r->nwlist && w!=NULL; i++,w=w->next) if (r->flags[i]) {
				ncat(match,w->word,BFSZ-1);
				ncat(match," ",BFSZ-1);
			}
//imsg("aspam: AContent match id=%d score=%2.1f/%2.1f Words (%s),nm=%d nwlist=%d\n",r->id,v,r->score,match,nm,r->nwlist);
		}
	}
	strcpy(reason,"");
	if (*val>0) {
		lprintf(reason,BFSZ-1,"X-AContent: Tot=%.1f %s",*val,bf);
		return TRUE;
	}
	return FALSE;
}
int con_scan_message(char *bin, int len)
{
	char *s;
	char c;
	double v;
	int at=0;
	char reason[BFSZ];
	int lastspace = TRUE;
	for (s=bin;(s-bin)<len-10;s++) {
		c = *s;
		if (lastspace) {
			if (sisalnum(c)) {
				at++;
				con_scan(s,at);
				lastspace = FALSE;
			}
		} else {
			if (!sisalnum(c)) lastspace = TRUE;
		}
	}
	if (con_score(&v,reason)) imsg("aspam: %2.1f %s\n",v,reason);
	return TRUE;
}
static char phrase_head[BFSZ];
int phrase_score(char *bf)
{
	ncpy(bf,phrase_head,BFSZ-1);
	if (lib_strlen(bf)>0) return TRUE;
	return FALSE;
}
int phrase_scan_message(char *bin, int len)
{
	char *s;
	char c;
	int nhits=0;
	int at=0;
	char thing[BFSZ];
	char foundphrase[BFSZ];
	char foundlist[BFSZ];
	Wordlist *wlist=NULL;
	Wordlist *w;
	int lastspace = TRUE;
	strcpy(foundphrase,"");
	strcpy(foundlist,"");
	for (s=bin;(s-bin)<len-10;s++) {
		c = *s;
		if (lastspace) {
			if (sisalnum(c)) {
				at++;
				if (phrase_scan(s,at,thing,foundphrase)) {
					//imsg("phrase: Found phrase (%s)\n",foundphrase);
					if (strstr(foundlist,foundphrase)==NULL) {
						if (lib_strlen(foundlist)>0) ncat(foundlist,", ",BFSZ-1); 
						ncat(foundlist,foundphrase,BFSZ-1);
					}
					word_add_nodup(&wlist,thing);
				}
				lastspace = FALSE;
			}
		} else {
			if (!sisalnum(c)) lastspace = TRUE;
		}
	}
	strcpy(phrase_head,"");
	for (w=wlist;w!=NULL;w=w->next) {
		ncat(phrase_head,w->word,BFSZ-1);
		ncat(phrase_head,": ",BFSZ-1);
		nhits++;
	}
	if (lib_strlen(foundlist)>0) {
		if (nhits>2) {	
			lpcat(phrase_head,BFSZ-1," hits=%d",nhits);
		}
		lpcat(phrase_head,BFSZ-1," [%s]",foundlist);
	}
	word_free(&wlist);
	return TRUE;
}
static Wordlist *surbl_list;
int surbl_remember(char *url)
{
	word_add_nodup(&surbl_list,url);
	return TRUE;
}
int surbl_reset(void)
{
	word_free(&surbl_list);
	return TRUE;
}
Wordlist *surbl_get(void)
{
	return surbl_list;
}
