#define MF_DECODE 1


#include <stdlib.h>
#ifdef TESTING

#include <io.h>
#endif
#include <stdio.h>
#include <stdarg.h>
#include <ctype.h>
#include <string.h>
#include <errno.h>

#include "vsnprintf.h"


#ifndef FALSE
#define FALSE 0
#define TRUE (!FALSE)
#endif
char *file_work(char *s);
char *file_config(char *s);
int dsmtp_send_file(char *fname, char *to);


// Marijn moved down (after inclusion of lib.h) as failed ot compile under solaris
#define DMID 18
#include "debug.h"
int  msgwrap(const char * arg_list, ...);
#include "lprintf.h"
#include <time.h>
#include "lib.h"
#include "pairs.h"
#include "lprintf.h"


#define file_config mfilter_file_config
char *file_config(char *s);
#ifndef strnicmp
#define strnicmp strncmpnc
#ifndef stricmp
#define stricmp strcmpnc
#endif
#endif

#	define mystrdup			strdup
#	define myrealloc			realloc
#	define mycalloc				calloc
#	define mymalloc(size)		malloc(size)
#	define myfree(ptr)			{if (ptr) { free(ptr); ptr = NULL; }}


int df_imsg(const char * arg_list, ...);

char* the_item = NULL;
int match_wild(char *p,char *text);

char *str_show2(char *s, char *bf);

// Wont allow undef as is different symbol??
//#define m->trace system_trace

#ifdef strcmpnc
#undef strcmpnc
#undef strncmpnc
#endif
#define strcmpnc mystrcmpnc
#define strncmpnc mystrncmpnc
int strncmpnc(const char *s1, const char *s2, size_t len);

int strcmpnc(const char *s1, const char *s2);
int df_concat(char *p[],int np, int maxp, char *bf);
#define DFILTER_MAIN 1
#include "pcre.h"
#include "lib_spawn.h"
#include "decode.h"
#include "mfilter.h"
#include "defines.h"
#include "rstring.h"
#include "mime.h"


// Only used in testing code so ok I think.
static int (*df_msgfn)(char *s);


enum {A_NULL,A_ACCEPT,A_REJECT,A_THEN,A_ELSE,A_END,A_SETFLAG,A_CLEARFLAG,A_PRINT,A_SET,A_ADD,A_SUB,A_SCORE_ONE,A_SCORE_ALL,A_RECIPIENTS,A_RECIPIENTS_END,A_FORWARD,A_CALL,A_DROP,A_REDIRECT_CC,A_DO_BOUNCE_FAST};



Mfilter *mfilter_new(void)
{
	Mfilter *m = calloc(1,sizeof(Mfilter));
	// setup any defaults here...
	m->max_len = 90000;
	return m;
}

int clock_disabled(void)
{
	return 0;
}
int dfilter_silent(Mfilter *m, int x)
{
	m->noisey = !x;
	return TRUE;
}
int comp_bounce_fast(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[]);
int run_bounce_fast(Exp *exp);
int run_time_min(Exp *exp);
void df_expr_free(FProg *curprog);

struct FNS {char *name; int narg; int (*compile)(FProg *fp, Exp *e,char *arg1, char *arg2, char *arg3[]); int (*run)(Exp *exp); } ;
static struct FNS flist[]={
	"constant",1,comp_constant,run_constant,
	"rexp",2,comp_rexp,run_rexp,
	"rexp_case",2,comp_rexp_case,run_rexp,
	"inpath",1,comp_inpath,run_rexp,
	"match",2,comp_match,run_match,
	"strcmp",2,comp_equals,run_equals,
	"size",0,comp_size,run_size,
	"nurls",0,comp_size,run_nurls,
	"crash",0,comp_crash,run_crash,
	"spawn",1,comp_spawn,run_spawn,
	"matchall",2,comp_matchall,run_matchall,
	"matchone",2,comp_matchone,run_matchone,
	"isin",2,comp_isin,run_isin,
	"istwo",2,comp_isin,run_istwo,
	"isflag",1,comp_ifflag,run_ifflag,
	"exists",1,comp_exists,run_exists,
	"isbinary",0,comp_size,run_isbinary,
	"isbinaryn",1,comp_isbinaryn,run_isbinaryn,
	"ishtml",0,comp_size,run_ishtml,
	"isencodedtext",0,comp_size,run_isencodedtext,
	"isencodedhtml",0,comp_size,run_isencodedhtml,
	"isencodedurl",0,comp_size,run_isencodedurl,
	"isuuencoded",0,comp_size,run_isuuencoded,
	"isbase64",0,comp_size,run_isbase64,
	"isimage",0,comp_size,run_isimage,
	"isclickimage",0,comp_size,run_clickimage,
	"isjpg",0,comp_size,run_isjpg,
	"ispdf",0,comp_size,run_ispdf,
	"image_size",0,comp_size,run_image_size,
	"pdf_size",0,comp_size,run_pdf_size,
	"nimage",0,comp_size,run_nimage,
	"islocal",0,comp_size,run_islocal,
	"isfriend",0,comp_size,run_isfriend,
	"isloggedin",0,comp_size,run_isloggedin,
	"isbounce",0,comp_size,run_isbounce,
	"lines",0,comp_size,run_lines,
	"body_lines",0,comp_size,run_body_lines,
	"body_words",0,comp_size,run_body_words,
	"allmod",1,comp_allmod,run_allmod,
	"get",1,comp_get,run_get,
	"forward_cc",1,comp_forward_cc,run_forward_cc,
	"attached",1,comp_attached,run_attached,
	"replace",3,comp_replace,run_replace,
	"ishidden",3,comp_ishidden,run_ishidden,
	"ismixedcase",1,comp_ismixedcase,run_ismixedcase,
	"report",2,comp_report,run_report,
	"report_full",2,comp_report,run_report_full,
	"report_brief",2,comp_report,run_report_brief,
	"head_len",1,comp_head_len,run_head_len,
	"spamdetect",2,comp_spamdetect,run_spamdetect,
	"feature_add",2,comp_feature_add,run_feature_add,
	"feature_manual",2,comp_feature_manual,run_feature_manual,
	"feature_header",2,comp_feature_header,run_feature_header,
	"feature_headerm",3,comp_feature_headerm,run_feature_headerm,
	"tomanual",3,comp_tomanual,run_tomanual,
	"isinc",2,comp_isin,run_isinc,
	"issimilar",2,comp_issimilar,run_issimilar,
	"wrongdomain",2,comp_wrongdomain,run_wrongdomain,
	"wronguser",2,comp_wronguser,run_wronguser,
	"forged",1,comp_forged,run_forged,
	"fromtwo",1,comp_fromtwo,run_fromtwo,
	"rexpstore",3,comp_rexpstore,run_rexpstore,
	"rexp_fast",3,comp_rexp_fast,run_rexp_fast,
	"bounce_fast",2,comp_bounce_fast,run_bounce_fast,
	"rexp_fast_flag",4,comp_rexp_fast_flag,run_rexp_fast,
	"evalflags",1,comp_evalflags,run_evalflags,
	"letterpairs",3,comp_invalidpairs,run_invalidpairs,
	"add_aspam",2,comp_add_aspam,run_add_aspam,
	"add_poly",2,comp_add_poly,run_add_poly,
	"add_header",1,comp_add_header,run_add_header,
	"hidden_word",2,comp_hidden_word,run_hidden_word,
	"folder_exists",1,comp_folder_exists,run_folder_exists,
	"date_offset",0,comp_date_offset,run_date_offset,
	"set_tohost",1,comp_set_tohost,run_set_tohost,
	"set_returnpath",1,comp_set_returnpath,run_set_returnpath,
	"unfriend",0,comp_unfriend,run_unfriend,
	"whitelist",1,comp_set_tohost,run_set_whitelist,
	"store_folder",1,comp_store_folder,run_store_folder,
	"is_dayofweek",1,comp_is_dayofweek,run_is_dayofweek,
	"time_hour",0,comp_time_hour,run_time_hour,
	"time_min",0,comp_time_hour,run_time_min,
NULL,0,NULL,NULL
};
char *item_read(char *fname)
{
	static char bf[2000];
	static char fbf[90000];
	FILE *f;
	char *s;

	strcpy(fbf,"");
	f = fopen(fname,"r");
	if (f==NULL) {
		return NULL;
	}
	for (;!feof(f);) {
		if (fgets(bf,1000,f)==NULL) break;
		s = strchr(bf,'\n');
		if (s!=NULL) *s = 0;
		strcat(fbf,bf);
		strcat(fbf,"\r\n");
		if(lib_strlen(fbf)>79000)
			break;
	}
	fclose(f);
	return fbf;
}
char *df_original(Fwddata *f, char *tmp2);
int df_add_fwd(Fwddata **flist, char *email)
{
	Fwddata *f;
	f = calloc(1,sizeof(Fwddata));
	f->next = *flist;
	f->cur = 0; /* Change to '1' to bounce message */
	f->path = mystrdup(email);
	*flist = f;
	return TRUE;
}
char *df_trim_email(char *s, char *bf2)
{
	char bf[BFSZ];
	
	char *lcl;

	lcl=NULL;
	ncpy(bf,s,200);
	s = mystrtok(bf,"<>, \t\r\n",&lcl);
	for (;s!=NULL;) {
		if (strchr(s,'@')!=NULL) {
			lprintf(bf2,BFSZ-1,"<%s>",s);
			return bf2;
		}
		s = mystrtok(NULL,"<>, \t\r\n",&lcl);
	}
	return "Novalidemailfound";
}


int mfilter_reload(Mfilter *m, char *fname)
{
#ifndef SURGEMAIL
#ifndef TESTING
	m->imsg=msgwrap;
	m->fmsg=msgwrap;

	fname=file_config(fname);
#endif
#endif
	ncpy(m->mfilter_fname,fname,100);
	if (m->fp!=NULL) dfilter_free(m,m->fp);
	m->fp = dfilter_compile(m,m->mfilter_fname);
	if (m->fp==NULL) {
		m->imsg("Compile failed or file doesn't exist (%s)\n",m->mfilter_fname);
		return FALSE;
	}
	return TRUE;
}
int mfilter_free(Mfilter *m)
{
	if (m->fp!=NULL) {
		dfilter_show_stats(m,m->fp,"mfilter.stats");
		dfilter_free(m,m->fp);
	}
	m->fp = NULL;
	return TRUE;
}


int mfilter_setipfrom(Mfilter *m, char *ip, char *sender )
{
	ncpy(m->cur_ip, ip,BFSZ-1);
	ncpy(m->cur_sender, sender,BFSZ-1);
	return TRUE;
}
int mfilter_setipname(Mfilter *m, char *ipname)
{
	ncpy(m->cur_ipname, ipname,BFSZ-1);
	return TRUE;
}

int mfilter_get_spamdetect(Mfilter *m, char *s, int sz)
{
	ncpy(s,m->spamdetect_save,sz-1);
	return TRUE;
}
int mfilter_reject(Mfilter *m, int chan, char *fname, struct fwddata **fwdlist, char *reason, char *ip, char *sender, char *ipname)
{
	int r;
	strcpy(m->spamdetect_save,"");
	if (m->fp==NULL) {
		m->imsg("mfilter: fprog is null\n");
		return FALSE;
	}

	strcpy(m->cur_ip, ip);
	if (ipname!=NULL) strcpy(m->cur_ipname, ipname);
	strcpy(m->cur_sender, sender);
	r = dfilter_do(chan,m->fp,fname,fwdlist,reason);
	return !r;
}
int mfilter_reject_prog(FProg *fprog, int chan, char *fname, struct fwddata **fwdlist, char *reason, char *ip, char *sender, char *ipname)
{
	int r;
	Mfilter *m=NULL;
	if (fprog==NULL) return FALSE;
	m = fprog->m;
	if (m==NULL) return FALSE;
	strcpy(m->spamdetect_save,"");
	if (ipname!=NULL) strcpy(m->cur_ipname, ipname);
	strcpy(m->cur_ip, ip);
	strcpy(m->cur_sender, sender);
	r = dfilter_do(chan,fprog,fname,fwdlist,reason);
	return !r;
}



#define MAXILEN 100000
int mfilter_set_real_len(Mfilter *m, int len)
{
	m->true_len = len;
	return TRUE;
}
int dfilter_setmem(Mfilter *m, char *s, int len)
{
	m->inmem = s;
	m->inmem_len = len;
	return TRUE;
}
int dfilter_setmax(Mfilter *m, int len)
{
	m->max_len = len;
	return TRUE;
}

void dfilter_free_statics(Mfilter *m)
{
	if (m->item) free(m->item);
	m->item=NULL;
	if (m->item_decoded) free(m->item_decoded);
	m->item_decoded=NULL;
	if (m->item_cleaned) free(m->item_cleaned);
	m->item_cleaned=NULL;
	if (m->urls) free(m->urls);
	m->urls=NULL;
}
int dfilter_do(int chan, FProg *fp, char *fname, Fwddata **fwdlist, char *reason)
{
	int t1;
	FILE *f;
	int len,r;
	Mfilter *m = fp->m;

	if(fwdlist==NULL) {
		strcpy(reason, "Rejected, No recipients");
		return FALSE;
	}
	if(*fwdlist==NULL) {
		strcpy(reason, "Rejected, No recipients");
		return FALSE;
	}

	if (m->item==NULL) m->item = malloc(MAXILEN);
#ifdef MF_DECODE
	if (m->item_decoded==NULL) m->item_decoded = malloc(MAXILEN);
	if (m->item_cleaned==NULL) m->item_cleaned = malloc(MAXILEN);
#endif
	if (m->inmem!=NULL) {
		len = m->inmem_len;
		m->real_len = len;
		if (len>=MAXILEN-2) len = MAXILEN-2;
		safe_memmove(m->item,m->inmem,len+1,m->item,MAXILEN-1);
		if (len>0) m->item[len] = 0;
		m->imsg("mfilter: read from MEMORY\n");
	} else {
		f = fopen(fname,"r");
		if (f==NULL) {
			m->imsg("Error reading (%s) %s\n",strerror(errno),fname);
			return TRUE;
		}
		m->real_len = lib_file_osize(f);
		len = (int)fread(m->item,1,MAXILEN-1,f);
		fclose(f);
		//m->imsg("mfilter: read from DISK\n");
	}
	if (len<=0) {
		m->imsg("Read zero bytes from temp file %s\n",fname);
		return TRUE;
	}
	if (len>=m->max_len) len = m->max_len-1;
	if (len>0) m->item[len] = 0;
	strcpy(reason,"");
	t1 = lstat_lib_msec();
	r = dfilter_run(m,chan,fp,m->item,m->item_decoded,m->item_cleaned,len,fwdlist,fname,reason,NULL);
	//m->imsg("mfilter: dfilter_run %d msec\n",lstat_lib_msec()-t1);
	if (m->noisey) m->imsg("Running filter and result was %d (%.100s)\n",r,reason);
	m->real_len = 0;
	m->true_len = 0;
	return r;
}

enum {O_AND,O_LESSTHAN,O_GREATERTHAN,O_EQUALS};
static char *operators[]={"and","lessthan","greaterthan","equals"};
#define MAX_LINE 10000



void dfilter_emsg(Mfilter *m, int (*msgfn)( const char *format, ... ))
{
	m->fmsg = msgfn;
}
void dfilter_imsg(Mfilter *m, int (*msgfn)( const char *format, ... ))
{
	m->imsg = msgfn;
}
void dfilter_trace(Mfilter *m,int x)
{
	m->trace = x;
}
char *gp(Mfilter *m, int n)
{
	if (n>=m->global_np) return "(NULL_TOKEN)";
	return m->global_p[n];
}

Linex *df_comp_if_true (FProg *fp, char *was);
FProg *dfilter_compile(Mfilter *m, char *fname)
{
	char *bf = calloc(1,MAX_LINE);
	char *tbf = calloc(1,MAX_LINE);
	int nin=0;
	char *upto;
	FILE *instack[20];
	char *p[1000];
	int np;
	FILE *f;
	int xupto;
	char *s;
	int qq[2000];
	FProg *fp=NULL;
	char  bfmax[MAX_LINE];
	char bfline[MAX_LINE+10];
	Linex *l,*last=NULL;
	char xbf[BFSZ];
	int k;
	int nif=0;
	int inif[1000];
	int i;

	for (i=0; i<1000; i++) inif[i] = FALSE;

	upto=NULL;
	m->lineno = 0; m->nthis = 0;
	if (lib_strlen(fname)==0) goto failed;
	ncpy(m->compile_fname,fname,BFSZ-1);
	m->imsg("Compiling %s\n",fname);
	//m->fmsg("Compiling %s\n",fname);
	mf_hidden_free(m);
	df_var_free(m);
	strcpy(m->derr,"No Error");
	f = fopen(fname,"r");
	if (f==NULL) {lprintf(m->derr,BFSZ-1,"Can't read (%s) (%s)\n",fname,strerror(errno)); goto failed;}
	/* Improve to handle \ on end of line as continuation */
	fp = mymalloc(sizeof(FProg));  memset(fp,0,sizeof(FProg));
	fp->m = m;
	m->curprog = fp;
	for (;!feof(f);) {
		xupto = 0;
		strcpy(bf,"");
again:		if (fgets(tbf,MAX_LINE-1,f)==NULL) break;
		m->lineno++; m->nthis++;
		s = strchr(tbf,'\n'); if (s!=NULL) *s = 0;
		strcat(bf,tbf);

		k = (int) strspn(tbf," \t");
		if (tbf[k]!='#')
		{
			if ((lib_strlen(bf)<10000)&&(lib_strlen(bf)>0))
				if (tbf[lib_strlen(tbf)-1]=='\\')
				{
					bf[lib_strlen(bf)-1] = 0;
					goto again;
				}
		}
		strncpy(xbf,bf,100); xbf[100] = 0;
		if (m->trace) m->imsg("dfilter: Compile %.55s%s\n",xbf,(lib_strlen(xbf)>55) ? "..." : "");
//m->imsg("dfilter: Compile %.55s%s\n",xbf,(lib_strlen(xbf)>55) ? "..." : "");
		if (nin<10) if (strncmp(bf,"#include ",9)==0) {
			m->nthis = 0;
			s = mystrtok(bf," \r\n",&upto);
			if (s!=NULL) s = mystrtok(NULL," \r\n",&upto);
			if (s!=NULL) {
				char *sx;
				FILE *ftemp;
				if (*s=='"') s++;
				sx = strchr(s,'"'); if (sx!=NULL) *sx = 0;
				if (m->trace) m->imsg("dfilter: include %.55s\n",s);
				m->imsg("dfilter: include %.55s\n",s);
				ftemp = fopen(s,"r");
				if (ftemp==NULL) {
					if (m->trace) m->imsg("dfilter: include  failed %.55s %s\n",s,strerror(errno));
				} else {
					if (m->trace) m->imsg("dfilter: include (%s)\n",s);
					m->imsg("dfilter: include (%s) open worked\n",s);
					instack[nin++] = f;
					f = ftemp;
				}
				strcpy(bf,"");
				goto again;
			}
		}
		np = df_token(bf,p,990,qq,bfline); m->global_np = np; m->global_p = p;
		if (np<0) { m->fmsg("Terminating quote missing, on line %d,  %.40s\n",m->lineno,xbf); continue;	}
		if (np<1) continue;
		if (strncmp(gp(m,0),"#",1)==0) continue;

		df_var_replace(m,p,np,qq);
		np = df_concat(p,np,999,bfmax);

//#define TESTINGV 1
#ifdef TESTINGV
		{int t;
		printf("Tok: ");
		for (t=0; t<np; t++) printf("{%s} ",gp(t));
		printf("\n");
		}
#endif
		l = NULL;
		if (df_find_action(m,&l,p,np,&xupto)) {
			if (strcmp(gp(m,0),"recipients")==0) {
				nif++;
				inif[nif] = TRUE; // not strictly true...
			}
		} else {
			if (strcmp(gp(m,0),"else")==0) {
				l = df_comp_else(p,np);
				if (!inif[nif]) m->fmsg("mfilter: WARNING ELSE when not in if block line=%d (%.40s)\n",m->lineno,gp(m,0));
				inif[nif] = FALSE;			
			} else {
			if (strcmp(gp(m,0),"call")==0) {
				l = df_comp_call(fp,p,np,qq);
				// Changed so it keeps going I hope.
				if (l==NULL) continue; // goto giveup;
			} else if (strcmp(gp(m,0),"endif")==0) {
				m->fmsg("mfilter: WARNING endif used instead of END IF  line=%d (%.40s)\n",m->lineno,gp(m,0));
				l = df_comp_end(p,np);
				if (nif==0) m->fmsg("mfilter: END found when not in block\n",m->lineno);
				else nif--;
			} else {
			if (np<2) {
				m->fmsg("mfilter: Invalid statement line=%d %.40s, try 'call ...'\n",m->lineno,gp(m,0));
				continue;
			}
			if (strcmp(gp(m,1),"=")==0) {
				df_comp_assign(m,p,np,qq);
				continue;
			} else if (strcmp(gp(m,0),"if")==0) {
				l = df_comp_if(fp,p,np,qq);
//				if (l==NULL) goto giveup;
				if (l==NULL) {
					l = df_comp_if_true(fp,tbf);
					if (l==NULL) continue;
				}
				if (l->action==A_THEN) nif++;
				inif[nif] = TRUE;
			} else if (strcmp(gp(m,0),"call")==0) {
				l = df_comp_call(fp,p,np,qq);
//				if (l==NULL) goto giveup;
				if (l==NULL) continue;
			} else if (strcmp(gp(m,0),"end")==0) {
				l = df_comp_end(p,np);
				if (nif==0) m->fmsg("mfilter: END found when not in block\n",m->lineno);
				else nif--;
			} else if (strcmp(gp(m,0),"endif")==0) {
				m->fmsg("mfilter: WARNING endif used instead of END IF  line=%d (%.40s)\n",m->lineno,gp(m,0));
				l = df_comp_end(p,np);
				if (nif==0) m->fmsg("mfilter: END found when not in block\n",m->lineno);
				else nif--;
			} else {
				m->fmsg("mfilter: Invalid statement line=%d (%.40s)\n",m->lineno,gp(m,0));
				continue;
			}
			}
		}
		}
		l->lineno = m->lineno;
		l->src = mystrdup(xbf);
		if (fp->lines==NULL) fp->lines = l;
		else last->next = l;
		last = l;
	}
	fclose(f);
	if (nin>0) {
		nin--;
		f = instack[nin];
		strcpy(bf,"");
		if (m->trace) m->imsg("dfilter: Finished including file\n");
		m->imsg("mfilter: Include finished %d \n",m->lineno);
		goto again;
	}

	m->skip_filteri = df_flag_findadd(m,"skip_filter");
	m->skip_fromi = df_flag_findadd(m,"skip_from");
	m->skip_dupi = df_flag_findadd(m,"skip_dup");
	df_var_free(m);

	if (nif!=0) m->fmsg("mfilter: WARNING MISSING END IF %d\n",m->lineno);

	m->imsg("mfilter: Compile finished, %d lines, %d nflags, %d=iflevel \n",m->lineno,fp->nflags,nif);
	
	free(bf);
	free(tbf);
	return fp;
//giveup:
	fclose(f);
	df_var_free(m);
	dfilter_free(m,fp);

failed:
	free(bf);
	free(tbf);
	return NULL;
}
// DODGY CODE HERE!!!
int df_concat(char *p[],int np, int maxp, char *bf)
{
	int i;
	char *out=bf;
	int maxline = MAX_LINE - 100;
	int outlen = 0;
	int dst;
	dst=0;
	maxp -= 3;
	for (i=0; i<(np); ) {
		if (dst>=maxp) break;
		if (i>=maxp) break;
		if (i>=(np-1)) goto rest;
		if (strcmp(p[i+1],"+")==0) {
			if ((int) (lib_strlen(out)+outlen+lib_strlen(p[i])) >= maxline) break;
			strcpy(out,p[i]);
			for (;i<(np-1);) {
				if (strcmp(p[i+1],"+")!=0) break;
				if ((int) (lib_strlen(out)+outlen+lib_strlen(p[i+2])) >= maxline) break;
				strcat(out,p[i+2]);
				i += 2;
				if (i>=maxp) break;
			}
			p[dst++] = out;
			outlen += lib_strlen(out)+1;
			out += lib_strlen(out)+1;
			i += 1;
		} else {
rest:			p[dst++] = p[i++];
		}
	}
	return dst;
}
int skip_filter(Mfilter *m)
{
	return df_flag_get(m,m->skip_filteri);
}
int skip_from(Mfilter *m)
{
	return df_flag_get(m,m->skip_fromi);
}
int skip_dup(Mfilter *m)
{
	return df_flag_get(m,m->skip_dupi);
}
static char *actions[]={"A_NONE","A_ACCEPT","A_REJECT","A_THEN","A_ELSE","A_ENDIF","SETFLAG","CLEARFLAG","PRINT","SET","ADD","SUB","SCORE_ONE","SCORE_ALL","RECIPIENTS","RECIPIENTS_END","FORWARD","CALL","DROP","REDIRECT_CC","DO_BOUNCE_FAST"};
char *df_fix_email(char *s, char *bf2)
{

	if (*s=='"') {
			ncpy(bf2,s+1,BFSZ-1);
			s = strchr(bf2,'"'); 
			if (s!=NULL) *s = 0;
			s = bf2;
	}
	if (*s=='<') return s;
	lprintf(bf2,BFSZ-1,"<%.200s>",s);
	return bf2;
}
void df_forward(Fwddata *fwd, char *to, int lineno)
{
	char tmp2[BFSZ];
	myfree(fwd->path);
	fwd->lineno = lineno;
	fwd->path = mystrdup(df_fix_email(to,tmp2));
	if (fwd->uinfo!=NULL) free(fwd->uinfo); /* Incase it's a local delivery */
	fwd->uinfo = NULL;
#ifdef FWDDATA_RCPT
	if(fwd->rcpt!=NULL)
		free(fwd->rcpt);
	fwd->rcpt=NULL;
#else
	if(fwd->env!=NULL)
		free(fwd->env);
	fwd->env=NULL;
#endif
}
void df_setflag(Mfilter *m, int p, int t)
{
	if ((p<0) || (p>=MAX_FLAGS)) return;
	m->flags[p] = t;
}

void df_addflag(Mfilter *m, int p, int t)
{
    if ((p<0) || (p>=MAX_FLAGS)) return;
    m->flags[p] += t;
}

void df_subflag(Mfilter *m, int p, int t)
{
    if ((p<0) || (p>=MAX_FLAGS)) return;
    m->flags[p] -= t;
}
void df_scoreoneflag(Mfilter *m, int p)
{
    if ((p<0) || (p>=MAX_FLAGS)) return;
#ifdef TESTING
    m->flags[p] += 999; /* brian add score_one function call here */
#else
    if (the_item==NULL) { m->imsg("score: scoreone - the m->item pointer was null\n"); return; }
    m->flags[p] += score_one(the_item,NULL);
#endif
}

void df_scoreallflag(Mfilter *m, int p)
{
    if ((p<0) || (p>=MAX_FLAGS)) return;
#ifdef TESTING
    m->flags[p] += 888; /* brian add score_all function call here */
#else
    if (the_item==NULL) { m->imsg("score: scoreall - the item pointer was null\n"); return; }
    m->flags[p] += score_all(the_item);
#endif
}

typedef struct _mf_counts
{
	int comments;
	int images;
	int letters;
	int caps;
	int encalpha;
	int spanmisuse;
} mf_counts;

void mf_count_letters(char *bf, int len, mf_counts *counts)
{
	if(*bf==0)
		return;
	while(len>0)
	{
		if(*bf==0)
			return;
		if((*bf>='a')&&(*bf<='z'))
			counts->letters++;
		if((*bf>='A')&&(*bf<='Z'))
		{
			counts->letters++;
			counts->caps++;
		}

		bf++;
		len--;
	}
}

int df_strip_html(Mfilter *m,char *item_inx, int olen, int istexthtml, mf_counts *counts,char *item_scratch,char *item_out,char *item_in)
{
	char *src, *dst, *scanner, *lookat, *skip, *lookend, *s;
	int state;
	int lookahead;
	int declen=0;
	char *lastend;
	int inimage=FALSE;
	int intable=0, inanchor=0, ishref=FALSE;

	lastend=NULL;
	counts->spanmisuse=0;
	counts->comments=0;
	counts->images=0;
	counts->letters=0;
	counts->caps=0;
	if (olen>=MAXILEN-2) olen = MAXILEN -2;
	safe_memmove(item_in, item_inx, olen,item_in,MAXILEN); // CHECK
	item_in[olen]=0;
	lookend=item_in+olen;

	lookat=item_in;
	src=item_in;
	dst=item_out;
	state=istexthtml;
	while(1)
	{
		if(lookat>=lookend)
		{
			lookat=lookend;
			break;
		}

		if(state==0)
		{
			scanner=strstr(lookat, "content-");
			if(scanner==NULL)
				scanner=strstr(lookat, "Content-");
			if(scanner==NULL)
				scanner=strstr(lookat, "CONTENT-");
			if(scanner==NULL)
				break;

			if(strnicmp(scanner, "Content-Type:", 13)!=0)
			{
				lookat=scanner+13;
				continue;
			}

			scanner+=13;

			while((*scanner==' ')||(*scanner=='\t'))
				scanner++;

			if(strnicmp(scanner, "text/html", 9)!=0)
			{
				lookat=scanner+9;
				continue;
			}

			state=1;
			intable=0;
			inanchor=0;
			ishref=FALSE;
			lookat=scanner+9;
			continue;
		}
		if(state==1)
		{
			char *taghit;
			int usetag;

			taghit=NULL;
			usetag=0;
			for(scanner=lookat;*scanner!=0;scanner++)
			{
				if((*scanner=='\r')||(*scanner=='\n')||(*scanner==0x09))
					break;
				if(strncmp(scanner, "  ", 2)==0)
					break;
				if((*scanner=='>')&&(taghit!=NULL))
				{
					usetag=1;
					inimage=FALSE;
					break;
				}

				// This code fails and double counts from unknown reason...
				if (inimage) if (*scanner=='"') if (strnicmp(scanner+1,"cid:",4)==0) {
					m->item_nimage++;
				}

				if(*scanner!='<')
					continue;
				if(taghit==NULL)
				{
					taghit=scanner;
					continue;
				}
				if(scanner-taghit>=6)
				{
					usetag=1;
					break;
				}
			}

			if(*scanner==0)
			{
				lookat=scanner;
				continue;
			}

			if(usetag)
				scanner=taghit;

			lookahead=(int) (scanner-lookat);

			if((*scanner=='\r')||(*scanner=='\n')||(*scanner==0x09)||(strncmp(scanner, "  ", 2)==0))
			{
				skip=scanner;
				while((*skip=='\r')||(*skip=='\n')||(*skip==0x09)||(*skip==' '))
					skip++;

				if(skip>lookend)
					skip=skip;

				strcpy(item_scratch, " ");
				safe_memmove(item_scratch+1, skip,(int) ( lookend+1-skip),item_scratch,MAXILEN-1); // CHECK
				safe_memmove(scanner, item_scratch,(int) ( lookend+2-skip),item_in,MAXILEN-1); // CHECK 

				lookend-=(skip-1-scanner);

				continue;
			}

			if(!istexthtml)
			{
				if(strnicmp(scanner, "Content-Type:", 13)==0)
				{
					lookat=scanner;
					state=0;
					continue;
				}
			}

			if(strncmp(scanner+1, "!--", 3)!=0)
			{
				skip=strstr(scanner+1, ">");

				if(strnicmp(scanner+1, "table ", 6)==0)
					intable++;
				if(strnicmp(scanner+1, "/table ", 7)==0)
					intable--;
				if(strnicmp(scanner+1, "a ", 2)==0)
				{
					s=strstr(scanner, "href");
					if(s==NULL)
						s=strstr(scanner, "HREF");
					if((s!=NULL)&&(s<skip))
						ishref=TRUE;

					inanchor++;
				}
				if(strnicmp(scanner+1, "/a ", 3)==0)
				{
					ishref=FALSE;
					inanchor--;
				}

				if(strnicmp(scanner+1, "span ", 5)==0)
				{
					if (strstrnc(scanner,"display: none")!=NULL) counts->spanmisuse++;
				}

				if(strnicmp(scanner+1, "img ", 4)==0)
				{
					if((ishref)&&(intable))
						counts->images+=4;
					if(ishref)
						counts->images++;
					inimage = TRUE;

				}

				mf_count_letters(src, (int) (scanner-src), counts);
				safe_memmove(dst, src, (int) (scanner-src),item_out,MAXILEN); // CHECK
				declen+=(int) (scanner-src);
				dst+=(int) (scanner-src);

				if(skip==NULL)
				{
					lookat=lookend;
					break;
				}
				skip++;
				src=skip;
				lookat=src;
				continue;
			}

			if(lastend!=NULL)
			{
				if(scanner-lastend<10)	/* Small gap between comments. Very suspicious */
					counts->comments+=2;
			}
			counts->comments++;

			skip=strstr(scanner+2, "-->");

			mf_count_letters(src, (int) (scanner-src), counts);
			safe_memmove(dst, src, (int) (scanner-src),item_out,MAXILEN); // CHECK 
			declen+=(int) (scanner-src);
			dst+=(int) (scanner-src);

			if(skip==NULL)
			{
				lookat=lookend;
				break;
			}

			skip+=3;
			src=skip;
			lookat=src;

			lastend=skip;
		}
	}

	mf_count_letters(src, lib_strlen(src)+1, counts);
	safe_memmove(dst, src, lib_strlen(src)+1,item_out,MAXILEN); // CHECK 
	declen+=lib_strlen(src);
	item_out[declen]=0;
	safe_memmove(item_inx, item_out, olen,item_inx,MAXILEN);

	return declen;
}
#ifdef MF_DECODE
void df_create_decoded(Mfilter *m, char *item, char **item_decoded, int *upto, char boundaries[20][128], int n_bound, int *istexthtml, int *is_encoded_text)
{
/*
<HOMER>
Mmmmm... recursion... *droooool*
</HOMER>

Notes:
	This routine does *not* modify the headers, and will therefore cause
	MAJOR HAVOC if the result is run through a MIME parser again. YHBW.

	NUL chars have been replaced by 0x20. Do not use the decoded data for
	virus scanning or anything like that.
*/
	char *bf, *line, *decoded, *s, *t;
	int gothdr, encoding, reading, type, linelen, oldupto;
	int dumphdr, i, blen, softbreak;
	int len;
	int curtext, qpcr;
	int drop=FALSE; // Drop application *.gif etc.

	bf = (char *) calloc(MAX_LINE, 1);
	line = (char *) calloc(MAX_LINE, 1);
	decoded = (char *) calloc(MAX_LINE, 1);
	if (n_bound==17) {
		len = 0;
	}
	if (n_bound>10) goto ret;
	if(n_bound>0)
		dumphdr=1;
	else
		dumphdr=0;

	reading=FALSE;		/* Do not add wrapped lines to the current header. */
	gothdr=FALSE;		/* Of course we don't have the header... maybe later */
	encoding=0;			/* Some messages are non-multipart, but still encoded. Eww */
	linelen=0;			/* I got nothin' */
	type=0;				/* We don't even know it's MIME yet */
	curtext=0;

	/* This loop only processes header lines */
	while(df_getline(item,upto,bf,MAX_LINE-100))
	{
		if(dumphdr) {
			if ((lib_strlen(bf)+m->dec_len) >= (MAXILEN-10)) goto ret;
			len = sprintf(*item_decoded, "%s\n", bf);
			(*item_decoded)+=len; m->dec_len += len;
			if (m->dec_len>MAXILEN/2) goto ret;
		}

		if((bf[0]==' ')||(bf[0]=='\t'))		/* Line-wrapped continuation */
		{
			if(!reading)
				continue;

			for(s=bf;(*s==' ')||(*s=='\t');s++);
			if(*s==0)	/* It could happen. Maybe. */
				continue;

			if(linelen+lib_strlen(s)>=MAX_LINE-2)	/* Oops. Header buffer overflow averted. Bit-bucketing further wraps */
			{
				reading=FALSE;
				continue;
			}

			if (linelen+lib_strlen(s) >= MAX_LINE-3) goto ret;
			if(linelen>0)	/* Concatenate the new stuff and the old stuff */
				line[linelen++]=' ';
			strcpy(line+linelen, s);
			linelen+=lib_strlen(s);
			continue;
		}
		/* We've either hit a new header line or the end of the header block */

		if(linelen!=0)	/* Examine the header line we've just finished */
		{
			if(strnicmp(line, "Content-Type:", 13)==0)
			{
				curtext=0;
				type=0;
				drop = FALSE;
				for(s=line+13;(*s==' ')||(*s=='\t');s++);
				if(strnicmp(s, "multipart/", 10)==0)
				{
					s=strstr(line, "boundary=");
					if(s!=NULL)	/* This should always happen */
					{
						for(t=s+9;(*t!=0)&&(*t!='\"');t++);
						if(*t!=0)
						{
							type=1;
							s=t;
							for(t=s+1;(*t!=0)&&(*t!='\"');t++);
							len = (int) (t-s-1);
							if (len<0) len = 0;
							if (len>120) len = 120;
							ncpy(boundaries[n_bound], s+1, len);
						}
					}
				}
				else if(strnicmp(s, "message/rfc822", 14)==0)
					type=2;
				else if(strnicmp(s, "image/", 6)==0) {drop = TRUE; } // dowsn't work as message is truncatd
				else if(strnicmp(s, "application/", 12)==0) drop = TRUE;
				else if(strnicmp(s, "audio/", 6)==0) drop = TRUE;
				else if(strnicmp(s, "video/", 6)==0) drop = TRUE;
				else if(strnicmp(s, "text/", 5)==0)
				{
					curtext=1;
					if((n_bound==0)&&(strnicmp(s+5, "html", 4)==0))
						*istexthtml=1;
				}
			}
			if(strnicmp(line, "Content-Transfer-Encoding:", 26)==0)
			{
				s=line+26;
				for(s=line+26;(*s==' ')||(*s=='\t');s++);
				if(strnicmp(s, "base64", 6)==0)
					encoding=1;
				else if(strnicmp(s, "quoted-printable", 16)==0)
					encoding=2;
			}

			linelen=0;
			strcpy(line, "");
		}

		if(lib_strlen(bf)==0)	/* Blank line. The header is finished. */
		{
			gothdr=TRUE;
			break;
		}
		if (linelen+lib_strlen(bf) >= MAX_LINE-3) goto ret;
		strcpy(line+linelen, bf);
		linelen+=lib_strlen(bf);
		reading=TRUE;
	}

	if(!gothdr)	/*df_getline returned FALSE before the headers officially ended */
		goto ret;

	if((curtext)&&(*is_encoded_text==0))
	{
		if(encoding==1)
			*is_encoded_text=1;
		else
			*is_encoded_text=-1;
	}

	if(type==2)
	{
		df_create_decoded(m,item, item_decoded, upto, boundaries, n_bound, istexthtml, is_encoded_text);
		goto ret;
	}

	oldupto=*upto;
	linelen=0;
	strcpy(line, "");
	softbreak=0;	/* The last line wasn't softbroken */
	while(df_getline(item,upto,bf,MAX_LINE-100))
	{
		if(strncmp(bf, "--", 2)==0)	/* We've hit something that might be a MIME boundary */
		{
			for(i=0;i<n_bound;i++)
			{
				blen=lib_strlen(boundaries[i]);
				if(strnicmp(bf+2, boundaries[i], blen)!=0)
					continue;
				if(bf[blen+2]==0)	/* Exact match. New part */
					break;
				if(strcmp(bf+blen+2, "--")==0) /* Terminator match */
					break;
			}
			if(i<n_bound)	/* We've found a boundary for a parent multipart */
			{
				*upto=oldupto;
				goto ret;
			}
		}

		oldupto=*upto;
		qpcr=FALSE;

		if(encoding==1)
		{
			if (!drop) {
				memset(decoded, 0, MAX_LINE);
				blen=decode_base64(bf, decoded, MAX_LINE-2);
				safe_memmove(bf, decoded, MAX_LINE-1,bf,MAX_LINE-1);

				for(i=0;i<blen;i++)
				{
					if(bf[i]==0x00)	 // kill nulls
						bf[i]=0x20;
				}
			} else bf[0] = 0;
		}
		else if(encoding==2)
		{
			qpcr=TRUE;
			if(lib_strlen(bf)>0)
			{
				if(bf[lib_strlen(bf)-1]=='=')
				{
					qpcr=FALSE;
					bf[lib_strlen(bf)-1]=0;
				}
				else
					qpcr=TRUE;
			}
			linelen=decode_quoted(bf, line, MAX_LINE-10);
			safe_memmove(bf, line, linelen+1,bf,MAX_LINE-10);
			strcpy(line, "");
		}

		if ((lib_strlen(bf)+m->dec_len) >= (MAXILEN-10)) goto ret;

		if(encoding==1)
			len = sprintf(*item_decoded, "%s", bf);
		else if((encoding==2)&&(!qpcr))
			len = sprintf(*item_decoded, "%s", bf);
		else
			len = sprintf(*item_decoded, "%s\n", bf);

		(*item_decoded)+=len;  m->dec_len += len;

		if (m->dec_len>MAXILEN/2) goto ret;

		while((strncmp(bf, "--", 2)==0)&&(type==1))	/* We've hit something that might be a MIME boundary */
		{
			blen=lib_strlen(boundaries[n_bound]);
			if(strnicmp(bf+2, boundaries[n_bound], blen)!=0)
				break;
			if(bf[blen+2]==0)	/* Exact match. New part */
			{
				if(n_bound<18) {
					df_create_decoded(m,item, item_decoded, upto, boundaries, n_bound+1, istexthtml, is_encoded_text);
				}
				break;
			}

			if(strcmp(bf+blen+2, "--")==0) /* Terminator match */
				goto ret;
			/* Break added by chris to fix bug, may be wrong */
			break;
		}
	}
ret:;
	myfree(bf);
	myfree(line);
	myfree(decoded);

	if(encoding==2)
	{
		strcpy(*item_decoded, "\n\n");
		(*item_decoded)+=2;  m->dec_len += 2;
	}
}
#endif /* MF_DECODE */

char is_dec_encoded(char *src)
{
	char *s;

	if(*src=='&' && *(src+1)=='#') {
		for(s=src+2;*s && *s>='0' && *s<='9' && s<src+5;s++);
		if(s>src+2 && *s==';') {
			return (char)atoi(src+2);
		}
	}
	return 0;
}

char *df_decode_url(char *src, char *dst,mf_counts *counts,int maxlen)
{
	char c;
	char ldst;
	char *start=dst;


	if(!dst) dst=&ldst;

	for(;*src!=0;src++)
	{
		if (maxlen && (dst-start>maxlen)) {
			break;
		}
		if(*src=='%') {
			if(!*++src)	break;
			*dst=decode_hexbyte(src);
			src++;
		} else if(c=is_dec_encoded(src)) {
			*dst=c;
			src=strchr(src,';');
		} else {
			*dst = *src;
			if(start) dst++;
			continue;
		}
		if(counts) if(mfisalpha(*dst) || (*dst>='0' && *dst<='9')) counts->encalpha++;
		if(start) dst++;
		if(*src==0) break;
	}
	*dst=0;
//if (counts) printf("decode url %d\n",counts->encalpha);

	return dst;
}

void df_scan_urls(Mfilter *m, char *item, char *urls,mf_counts *counts)
{
	char *src, *s, *dst;
	char scratch[BFSZ+100];
	int quoted=FALSE;
	int len;

	m->nurls = 0;

	counts->encalpha=0;
	strcpy(urls, "");
	strcpy(m->firsturl,"");
	dst=urls;
	src=strstr(item, "http");
	while(src!=NULL)
	{
		if(src[4]!=':')
		{
			if((src[4]!='s')||(src[5]!=':'))
			{
				item=src+4;
				src=strstr(item, "http:");
				continue;
			}
		}
		if(src!=item)
		{
			if(src[-1]=='\"')
				quoted=TRUE;
			else
				quoted=FALSE;
		}

		if(quoted)
			for(s=src;(*s!=0)&&(*s!='>')&&(*s!='\"')&&(*s!='\r')&&(*s!='\n');s++);
		else
			for(s=src;(*s!=0)&&(*s!='>')&&(*s!='\"')&&(*s!=' ')&&(*s!='\t')&&(*s!='\r')&&(*s!='\n');s++);
		len = (int) (s - src);
		if (len>=BFSZ-2) len = BFSZ-2;
		if (len<0) len = 0;
		if(len>10)
		{
			safe_memmove(scratch, src, len,scratch,BFSZ-1);
			scratch[len]=0;

			dst=df_decode_url(scratch, dst,counts,5000);
			*dst++='\r';
			*dst++='\n';
			*dst=0;
			m->nurls++;
		}
		item=s;
		src=strstr(item, "http");
		if (dst-urls > 5000) break;
	}
}

char *df_get_addr(char *string)
{
	char *s, *t;

	s=string;
	if(*s=='\"')
	{
		s=strchr(s+1, '\"');
		while(s!=NULL)
		{
			if(*(s-1)!='\\')
				break;
			s=strchr(s+1, '\"');
		}
	}
	if(s==NULL)
		return strdup("");

	s=strchr(s, '<');
	if(s==NULL)
		return strdup("");

	t=++s;

	if(*t=='\"')
	{
		t=strchr(t+1, '\"');
		while(t!=NULL)
		{
			if(*(t-1)!='\\')
				break;
			t=strchr(t+1, '\"');
		}
	}

	if(t!=NULL)
		t=strchr(t, '>');
	if(t!=NULL)
		*t=0;
	s=strdup(s);
	if(t!=NULL)
		*t='>';

	return s;
}
static char *specials[]={"body","head","cleanbody","rawbody","full","urls","recipient","paragraph1","paragraph2","paragraph3",NULL};
//int new_decode(char *item,char *out, int outlen, int *istexthtml,int *encoded_html)
//{
//
//}

#ifdef SURGEMAIL
int mfilter_set_extra(Mfilter *m, Wordlist *w)
{
	m->hdr_add = w;
	for (;w!=NULL;w=w->next) {
		//m->imsg("mfilter: DEBUGGING CODE set extra got (%s)\n",w->word);
	}
	return TRUE;
}
#endif
char *df_get_equals(char *msg, char *thing)
{
	char *s = strstr(msg,thing);
	if (s==NULL) return "";
	s = s + lib_strlen(thing);
	if (*s =='"') {
		s++;
		return s;
	}
	return s;
}
int df_clickimage(char *msg,int *isclick)
{
	int i;
	char *width;
	char bf[BFSZ];
	char *at = msg;
	char *s;
	*isclick = FALSE;

	for (i=0; i<10; i++) {
		at = strstrnc(at+1,"<a "); if (at==NULL) break;
		ncpy(bf,at,BFSZ-1);
		s = strstrnc(bf,"</a>"); if (s!=NULL) *s = 0;
		s = strstrnc(bf,"<img "); if (s==NULL) continue;
		width = df_get_equals(s,"width=");
		if (width) if (atoi(width)<100) continue; // too small
		width = df_get_equals(s,"height=");
		if (width) if (atoi(width)<100) continue; // too small
		*isclick = TRUE;
		break;
	}
	return TRUE;
}

int df_hdrs_process(Mfilter *m, char *item, char *item_decoded, char *item_cleaned, char *urls)
{
	int lens[MAX_HEADERS];
	char bf2[MAX_LINE];
	int i;
	int n;
	int slen;
#ifdef MF_DECODE
	int declen;
#endif
	int t;
	int h= -1;
	char *hadd;
	char bf[MAX_LINE];
	int upto = 0;
	int gotbegin;
	int this_text = FALSE;
	int this_base64 = FALSE;
	int this_html = FALSE;
	char *item_scratch;
	char *item_out;
	char *item_in;

	item_scratch = malloc(MAXILEN);
	item_out = malloc(MAXILEN);
	item_in = malloc(MAXILEN);

	the_item = item;
	m->nbin = 0;
	/* Step thru item and define each header if found in
		curprog->headers[]
		Also define 'body' and 'head'
		And 'size'
	*/
	m->fix_len = -2;
	df_setflag(m,df_flag_findadd(m,"html_encoded"), 0);
	m->item_isuuencoded = m->item_isbinary = m->item_ishtml = 0;
	m->item_isbase64 = m->item_isimage = m->item_isclickimage = m->item_istext = 0;
	m->item_nimage = 0;

	m->item_isencodedhtml = m->item_isencodedtext = m->item_isencodedurl = 0;


	for (i=0; i<m->curprog->nflags; i++) {
		m->flags[i] = FALSE;
	}

	m->item_head_len = 0;
	for (i=0; i<m->curprog->nheaders; i++) {
		lens[i] = lib_strlen(m->curprog->headers[i]);
		m->hdrs_fake[i] = FALSE;
		m->hdrs_changed[i] = FALSE;
	}

	df_setflag(m,df_flag_findadd(m,"__REAL_IMAGE"), TRUE); // so we know it has real image checking code. 

#ifdef MF_DECODE
	{
		char *s;
		int decode_upto=0, istexthtml=0;
		char *item_dc_start;
		char boundaries[20][128];
		char *headend;
		int bodylen, decbodylen, percent, isec;
		int prelen=0;
		mf_counts counts;

		memset(&counts,0,sizeof(counts));

		bodylen=0;
		isec=0;
		headend=strstr(item, "\n\n");
		if(headend!=NULL)
			bodylen=lib_strlen(headend+2);
		else
		{
			headend=strstr(item, "\n\r\n");
			if(headend!=NULL)
				bodylen=lib_strlen(headend+3);
		}

		df_setflag(m,df_flag_findadd(m,"__UNUSABLE_MSGID"), FALSE);

		item_dc_start=item_decoded;
		// item_decoded is pre allocated array of maxilen
		// boundaries is not reused
		// istexthtml, not sure what that means
		// isec = encoded html
		//new_decode(item,item_decoded,&istexthtml,&isec);
		df_create_decoded(m,item, &item_decoded, &decode_upto, boundaries, 0, &istexthtml, &isec);
		decbodylen=lib_strlen(item_dc_start);
//printf("Got %d bytes of message \n",decbodylen,item_dc_start);

		df_scan_urls(m,item_dc_start, urls,&counts);
//printf("nurls %d\n",nurls);
		df_clickimage(item_dc_start,&m->item_isclickimage);
		ncpy(m->firsturl,urls,BFSZ-1);
		s = strchr(m->firsturl,'\n'); if (s!=NULL) *s = 0;
		s = strchr(m->firsturl,'\r'); if (s!=NULL) *s = 0;

		prelen = decbodylen;
		m->item_nimage = 0;
		declen=df_strip_html(m,item_dc_start,(int) ( item_decoded-item_dc_start), istexthtml, &counts,item_scratch,item_out,item_in);
		decbodylen=lib_strlen(item_dc_start);
//printf("nimage %d\n",m->item_nimage);

//printf("Got %s\n",item_dc_start);
		if(counts.encalpha) {
			df_setflag(m,df_flag_findadd(m,"url_alpha_encoded"), 1);
		}
		if(counts.comments>20)
			df_setflag(m,df_flag_findadd(m,"many_html_comments"), 1);
		else if(counts.comments>10)
			df_setflag(m,df_flag_findadd(m,"some_html_comments"), 1);

		if(isec>0)
			df_setflag(m,df_flag_findadd(m,"encoded_html"), 1);

		if(counts.spanmisuse>6)
		{
				df_setflag(m,df_flag_findadd(m,"__SPAN_MISUSED"), 1);
		}

		if(counts.images>0)
		{
			if(counts.images>10)
				df_setflag(m,df_flag_findadd(m,"some_suspicious_images"), 1);
			else if(counts.images>25)
				df_setflag(m,df_flag_findadd(m,"many_suspicious_images"), 1);

			if((counts.letters<150)&&(counts.images>0))
				df_setflag(m,df_flag_findadd(m,"low_text_images_ratio"), 1);
			if((counts.letters<300)&&(counts.images>5))
				df_setflag(m,df_flag_findadd(m,"low_text_images_ratio"), 1);
			if((counts.letters<800)&&(counts.images>5))
				df_setflag(m,df_flag_findadd(m,"medium_text_images_ratio"), 1);
		}

		if((bodylen>0)&&(decbodylen>0))
		{
			if(strstr(item, "urn:schemas-microsoft-com:office")==NULL)
			{
//				percent=100-((100*decbodylen)/bodylen);
				percent=100-((100*decbodylen)/prelen);
				//m->imsg("mfilter: tagstotext %d %d %d%%\n",prelen,decbodylen,percent);
				if(percent>=90)
					df_setflag(m,df_flag_findadd(m,"extreme_tags_to_text"), 1);
				else if(percent>=70)
					df_setflag(m,df_flag_findadd(m,"high_tags_to_text"), 1);
				else if(percent>=48)
					df_setflag(m,df_flag_findadd(m,"medium_tags_to_text"), 1);
				else {
					if (prelen>2000) {
						if (percent>40)	df_setflag(m,df_flag_findadd(m,"some_tags_to_text"), 1);
					}
				}
			}
		}

		if(counts.letters>0)
		{
			percent=(100*counts.caps)/counts.letters;

			if(percent>50)
				df_setflag(m,df_flag_findadd(m,"high_caps"), 1);
			if(percent>25)
				df_setflag(m,df_flag_findadd(m,"medium_caps"), 1);
		}

		item_decoded=item_dc_start;
	}
#endif

	hadd = m->item_head;
	for (;TRUE;) {
		if (!df_getline(item,&upto,bf,MAX_LINE-100)) goto skip_body;
		/* m->imsg("data %d->%s\n",m->item_ishtml,bf);*/
		if (h>=0) {
			if ((bf[0]==' ') || (bf[0]=='\t')) {
				char *s;

				s=bf;
				while((*s==' ')||(*s=='\t'))
					s++;
				//sprintf(bf2,"%.400s\n%.400s",hdrs[h],s);
				lprintf(bf2,MAX_LINE-1,"%s\n %s",m->hdrs[h],s);
				myfree(m->hdrs[h]);
				m->hdrs[h] = mystrdup(bf2);
				m->hdrs_len[h]=lib_strlen(m->hdrs[h]);
				goto cont;
			}
		}
		h = -1;
		for (i=0; i<m->curprog->nheaders; i++) {
			if (strnicmp(bf,m->curprog->headers[i],lens[i])==0 && bf[lens[i]]==':' ) {
				char *s;
				int skip=FALSE;
				int z;

				for (z=0; specials[z]!=NULL; z++) {
					if (stricmp(m->curprog->headers[i],specials[z])==0) {
						m->imsg("Ignore special header in message (%s)\n",bf);
						skip = TRUE;
						break;
					}
				}
				// Skip received spf header if we already have a real local one :-)
				if (m->hdrs[i]!=NULL) if (strnicmp(bf,"Received-SPF",12)==0) skip = TRUE;
				if (!skip) {
					char xbf[BFSZ];
					strcpy(xbf,"");
					if (m->hdrs[i]!=NULL) ncpy(xbf,m->hdrs[i],BFSZ-1);
					
					if (m->hdrs[i]!=NULL) myfree(m->hdrs[i]); m->hdrs[i] = NULL;

					s = bf + lens[i] + 1;
					if (*s==' ') s++;
					if(stricmp(m->curprog->headers[i], "body")!=0)
					{
						while((*s==' ')||(*s=='\t'))
						s++;
					}
					if (lib_strlen(xbf)>0) {
						ncat(xbf," ",BFSZ-1);
						ncat(xbf,s,BFSZ-1);	
						m->hdrs[i] = mystrdup(xbf);
					} else {
						m->hdrs[i] = mystrdup(s);
					}
					m->hdrs_len[i]=lib_strlen(m->hdrs[i]);
					h = i;
				}
			}
			if((stricmp(m->curprog->headers[i], "to-addr")==0) && (strnicmp(bf, "to:", 3)==0))
			{
				char *s;
				if (m->hdrs[i]!=NULL) myfree(m->hdrs[i]); m->hdrs[i] = NULL;

				s = bf + 4;

				m->hdrs[i] = df_get_addr(s);
				m->hdrs_len[i]=lib_strlen(m->hdrs[i]);
				m->hdrs_fake[i] = FALSE;
			}
			if((stricmp(m->curprog->headers[i], "from-addr")==0) && (strnicmp(bf, "from:", 5)==0))
			{
				char *s;
				if (m->hdrs[i]!=NULL) myfree(m->hdrs[i]);

				s = bf + 6;

				m->hdrs[i] = df_get_addr(s);
				m->hdrs_len[i]=lib_strlen(m->hdrs[i]);
				m->hdrs_fake[i] = FALSE;
			}
			if((stricmp(m->curprog->headers[i], "cleansubject")==0) && (strnicmp(bf, "subject:", 8)==0))
			{
				char *s;
	//			char stemp[BFSZ];

				if (m->hdrs[i]!=NULL) myfree(m->hdrs[i]);

				s = bf + 9;

				m->hdrs[i] = mystrdup(s);

				if(m->hdrs[i]) {
					df_decode_words(m,m->hdrs[i]);
					cleanstr(m->hdrs[i],m->hdrs[i]);
				}
				m->hdrs_len[i]=lib_strlen(m->hdrs[i]);
				m->hdrs_fake[i] = FALSE;
			}
		}
		if (strncmpnc(bf,"Content-Transfer-Encoding: base64",33)==0) {m->item_isbase64 = TRUE; };
		if (strncmpnc(bf,"Content-Type: image",19)==0) {m->item_isimage = TRUE;};
		if (strncmpnc(bf,"Content-Type: text/html",23)==0) {m->item_ishtml = this_html = TRUE; };
		if (strncmpnc(bf,"Content-Type: text",18)==0) {m->item_istext = this_text = TRUE;};
		m->item_isencodedtext = m->item_isbase64 && this_text;
		m->item_isencodedhtml = m->item_isbase64 && this_html;
cont:
		if (strcmp(bf,"")==0) break;
		n = lib_strlen(bf);
		if (hadd-m->item_head+n+1000>MAX_HEAD) continue;
		strcpy(hadd,bf); strcat(hadd,"\n");
		hadd+= n+1;
	}
	this_text = this_html = FALSE;

	for (i=0; i<m->curprog->nheaders; i++) {
			if (m->hdrs[i]==NULL) continue;
			if((stricmp(m->curprog->headers[i], "subject")==0))
			{
//				char *s;
				char stemp[BFSZ];
				mime_header_decode(m->hdrs[i],stemp,BFSZ-1);
				myfree(m->hdrs[i]);
//printf("Decoded subject %s\n",stemp);
				m->hdrs[i] = mystrdup(stemp);
				m->hdrs_len[i]=lib_strlen(m->hdrs[i]);
				m->hdrs_fake[i] = FALSE;
			}
	}

	for (i=0; i<m->curprog->nheaders; i++) {
#ifndef MF_DECODE
		if (strcmp("body",m->curprog->headers[i])==0) {
			m->hdrs[i] = item + upto;
			m->hdrs_len[i]=lib_strlen(m->hdrs[i]);
			m->hdrs_fake[i] = TRUE;
		}
#else
		if(strcmp("body", m->curprog->headers[i])==0) {
			m->hdrs[i] = item_decoded;
			m->hdrs_len[i]=lib_strlen(m->hdrs[i]);
//printf("len=%d %s\n", m->hdrs_len[i],item_decoded);
			m->hdrs_fake[i] = TRUE;
		}
		if(strcmp("paragraph1", m->curprog->headers[i])==0) {
			m->hdrs[i] = m->item_paragraph1;
			m->hdrs_len[i]=lib_strlen(m->hdrs[i]);
			m->hdrs_fake[i] = TRUE;
		}
		if(strcmp("paragraph2", m->curprog->headers[i])==0) {
			m->hdrs[i] = m->item_paragraph2;
			m->hdrs_len[i]=lib_strlen(m->hdrs[i]);
			m->hdrs_fake[i] = TRUE;
		}
		if(strcmp("paragraph3", m->curprog->headers[i])==0) {
			m->hdrs[i] = m->item_paragraph3;
			m->hdrs_len[i]=lib_strlen(m->hdrs[i]);
			m->hdrs_fake[i] = TRUE;
		}
		if(strcmp("cleanbody", m->curprog->headers[i])==0 && item_cleaned) {
			df_decode_url(item_decoded,item_cleaned,0,lib_strlen(item_decoded));
			cleanstr(item_cleaned,item_cleaned);
//printf("cleanbody (%s)\n",item_cleaned);
			m->hdrs[i] = item_cleaned;
			m->hdrs_len[i]=lib_strlen(m->hdrs[i]);
			m->hdrs_fake[i] = TRUE;
		}
		if (strcmp("rawbody",m->curprog->headers[i])==0) {
			m->hdrs[i] = item + upto;
			m->hdrs_len[i]=lib_strlen(m->hdrs[i]);
			m->hdrs_fake[i] = TRUE;
		}
#endif
		if (strcmp("full",m->curprog->headers[i])==0) {
			m->hdrs[i] = item;
			m->hdrs_len[i]=lib_strlen(item);
			m->hdrs_fake[i] = TRUE;
		}
		if (strcmp("urls",m->curprog->headers[i])==0) {
			m->hdrs[i] = urls;
			m->hdrs_len[i]=lib_strlen(urls);
			m->hdrs_fake[i] = TRUE;
//printf("urls: (%s)\n", urls);
		}
		if (strcmp("firsturl",m->curprog->headers[i])==0) {
			m->hdrs[i] = m->firsturl;
			m->hdrs_len[i]=lib_strlen(m->firsturl);
			m->hdrs_fake[i] = TRUE;
		}
		if (strcmp("recipient",m->curprog->headers[i])==0) {
			m->hdrs[i] = m->cur_recipient;
			m->hdrs_len[i]=lib_strlen(m->hdrs[i]);
			m->hdrs_fake[i] = TRUE;
			m->fix_len = i;
		}
		if (stricmp("return-path",m->curprog->headers[i])==0) {
			if((m->hdrs[i]==NULL)&&(lib_strlen(m->cur_sender)>0)) {
				m->hdrs[i] = m->cur_sender;
				m->hdrs_len[i]=lib_strlen(m->hdrs[i]);
				m->hdrs_fake[i] = TRUE;
			}
		}
		if (stricmp("sender-ip",m->curprog->headers[i])==0) {
			if((m->hdrs[i]==NULL)&&(lib_strlen(m->cur_ip)>0)) {
				m->hdrs[i] = m->cur_ip;
				m->hdrs_len[i]=lib_strlen(m->hdrs[i]);
				m->hdrs_fake[i] = TRUE;
			}
		}
		if (stricmp("sender-ipname",m->curprog->headers[i])==0) {
			if((m->hdrs[i]==NULL)&&(lib_strlen(m->cur_ipname)>0)) {
				m->hdrs[i] = m->cur_ipname;
				m->hdrs_len[i]=lib_strlen(m->hdrs[i]);
				m->hdrs_fake[i] = TRUE;
			}
		}
#ifdef SURGEMAIL
		{
			Wordlist *w;
			char name[BFSZ];
			char *s;
			for (w=m->hdr_add; w!=NULL; w=w->next) {
				ncpy(name,w->word,BFSZ-1);
				s = strchr(name,':'); if (s!=NULL) *s = 0;
				if (stricmp(name,m->curprog->headers[i])==0) {
					if((m->hdrs[i]==NULL)) {
						m->hdrs[i] = mystrdup(s+1);
						m->hdrs_len[i]=lib_strlen(m->hdrs[i]);
						m->hdrs_fake[i] = FALSE; // cause it's malloced.
					} else { // I Think we add them togehter ? 
						char bf[BFSZ];
						ncpy(bf,m->hdrs[i],BFSZ-1);
						ncat(bf," ",BFSZ-1);
						ncat(bf,s+1,BFSZ-1);
						if (!m->hdrs_fake[i]) myfree(m->hdrs[i]);
						m->hdrs[i] = mystrdup(bf);
						m->hdrs_len[i]=lib_strlen(m->hdrs[i]);
						m->hdrs_fake[i] = FALSE; // cause it's malloced.
					}
				}
			}
		}
#endif

	}

	gotbegin = FALSE;
	for (t=0; t<100; t++) {
		if (!df_getline(item,&upto,bf,MAX_LINE-100)) goto skip_body;
		/*m->imsg("body %d->%s\n",m->item_ishtml,bf);*/
		slen = lib_strlen(bf);
		/* Is it uuencoded data */
		if (strncmpnc(bf,"=ybegin",7)==0) {
			m->item_isbinary = TRUE;
			m->nbin += 30;
		}
		if (strncmpnc(bf,"begin ",6)==0) {
			gotbegin = TRUE;
			/* Allow signatures the outlook puts all over the place */
			if (strstr(bf,".vcf")!=NULL) gotbegin = FALSE;
			strlwr(bf);
			if (strstr(bf,".jpg")!=NULL) m->item_isimage = TRUE;
			if (strstr(bf,".jpeg")!=NULL) m->item_isimage = TRUE;
			if (strstr(bf,".bmp")!=NULL) m->item_isimage = TRUE;
			if (strstr(bf,".tif")!=NULL) m->item_isimage = TRUE;
			if (strstr(bf,".gif")!=NULL) m->item_isimage = TRUE;

			if (strstr(bf,".txt")!=NULL) m->item_isencodedtext = TRUE;
			if (strstr(bf,".text")!=NULL) m->item_isencodedtext = TRUE;
			if (strstr(bf,".htm")!=NULL) m->item_isencodedhtml = TRUE;
			/*if (strstr(bf,".html")!=NULL) m->item_isencodedhtml = TRUE;*/
			if (strstr(bf,".url")!=NULL) m->item_isencodedurl = TRUE;
		}
		if (gotbegin) {
			if (bf[0]=='M') if (slen==61) {
				m->item_isuuencoded = TRUE;
				m->nbin++;
			}
		}
		if (m->item_isbase64) {
                  if (slen>40) if (strchr(bf,' ')==NULL) m->nbin++;
		}

		/* Does it look like html */
		strlwr(bf);
		if (t<3) if (strstr(bf,"<html>")!=NULL) m->item_ishtml = TRUE;

		/* Does it look like base64 stuff and is it an image */
		if (strncmpnc(bf,"Content-Transfer-Encoding: base64",33)==0) {m->item_isbase64 = this_base64 = TRUE; };
		if (strncmpnc(bf,"Content-Type: image",19)==0) {m->item_isimage = TRUE; };
		if (strncmpnc(bf,"Content-Type: text",18)==0) {m->item_istext = this_text = TRUE;};
		if (strncmpnc(bf,"Content-Type: text/html",23)==0) {m->item_ishtml = this_html = TRUE;};
		if (strcmp(bf,"")==0)
		{
			m->item_isencodedtext = this_base64&&this_text;
			m->item_isencodedhtml = this_base64&&this_html;
			this_base64 = this_text = this_html = FALSE;
		}

	}
skip_body:
	if (m->item_isuuencoded) m->item_isbinary = TRUE;
	if (m->item_isbase64) m->item_isbinary = TRUE;
	if (m->item_isimage) m->item_isbinary = TRUE;
	for (i=0; i<m->curprog->nheaders; i++) {
		if (strcmp("head",m->curprog->headers[i])==0) {
			m->hdrs[i] = m->item_head;
			m->hdrs_len[i]=lib_strlen(m->hdrs[i]);
			m->hdrs_fake[i] = TRUE;
		}
	}
	for (i=0; i<m->curprog->nheaders; i++) {
/*		m->fmsg("Header (%.40s) found (%.40s)\n",curprog->headers[i],m->hdrs[i]);*/
	}

	hadd=df_hdrs_get(m,df_header_findadd(m,"to"));

	m->item_lines = count_lines(item,&m->body_lines,&m->body_words);

	free(item_scratch);
	free(item_out);
	free(item_in);


	return TRUE;
}
int df_hdrs_exists(Mfilter *m, int x)
{
	if (m->hdrs[x]==NULL) return FALSE;
	return TRUE;
}

int df_hdrs_getlen(Mfilter *m, int x)
{
	if(m->hdrs[x]==NULL)
		return 0;

	if (x==m->fix_len) m->hdrs_len[x] = lib_strlen(m->hdrs[x]);
	return m->hdrs_len[x];
}

char *df_hdrs_get(Mfilter *m, int x)
{
	if (m->hdrs[x]==NULL) return "";
	return m->hdrs[x];
}

void df_hdrs_set(Mfilter *m, int i, char *s)
{

	if (i<0) return;
	if (m->hdrs_fake[i]) return;
	if (m->hdrs[i]!=NULL) myfree(m->hdrs[i]);
	m->hdrs[i] = mystrdup(s);
	m->hdrs_len[i]=lib_strlen(m->hdrs[i]);
	m->hdrs_changed[i] = TRUE;
}



void mfilter_set_hdrs_add(Mfilter *m, void (*hdrs_add)(char *s))
{
	m->hdrs_external = hdrs_add;
}
int mfilter_set_aspam(Mfilter *m, double score)
{
	m->aspam_score = score;
	return TRUE;
}
int mfilter_set_poly(Mfilter *m, double score)
{
	m->poly_score = score;
	return TRUE;
}
void df_hdrs_add(Mfilter *m, char *s)
{
//m->imsg("df_hdrs_add(%s)\n",s);
	if (m->hdrs_external!=NULL) {
		m->hdrs_external(s);
		return;
	}
	if (m->nadd<90) m->hdrs_add[m->nadd++] = mystrdup(s);
}

int df_rewrite_message(Mfilter *m)
{
	FILE *fin,*fout;
	int skip = FALSE;
	int i;
	int failed=FALSE;
	int inhead = TRUE;
	/* Write message with new modified headers */
	char msgfiletmp[BFSZ];
	char bf[BFSZ];
	int lens[MAX_HEADERS];

	m->imsg("Rewriting (%s)\n",m->msgfile);
	lprintf(msgfiletmp,BFSZ-1,"%.100s_new",m->msgfile);

	for (i=0; i<m->curprog->nheaders; i++) lens[i] = lib_strlen(m->curprog->headers[i]);
	fin = fopen(m->msgfile,"rb");
	fout = fopen(msgfiletmp,"wb");
	if (fin==NULL) {
		m->imsg("df_rewrite_failed %s %s\n",m->msgfile,strerror(errno));
		return FALSE;
	}
	if (fout==NULL) {
		m->imsg("df_rewrite_failed %s %s\n",msgfiletmp,strerror(errno));
		fclose(fin);
		return FALSE;
	}
	for (;!feof(fin);) {
		if (fgets(bf,BFSZ-1,fin)==NULL) break;
		if (inhead) {
			if (skip) if ((bf[0]==' ') || (bf[0]=='\t')) continue;
			skip = FALSE;
			for (i=0; i<m->curprog->nheaders; i++) {
				if (strncmpnc(bf,m->curprog->headers[i],lens[i])==0 && bf[lens[i]]==':' ) {
					if (m->hdrs_changed[i]) {
						bf[lens[i]] = 0;
						if (fprintf(fout,"%.100s: %.800s\r\n",bf,m->hdrs[i])<=0) failed=TRUE;
						skip = TRUE;
						goto next_line;
					}
				}
			}
			if (strcmp(bf,"\r\n")==0) inhead = FALSE;
			if (strcmp(bf,"\n")==0) inhead = FALSE;
			if (strcmp(bf,"\r\r\n")==0) inhead = FALSE;
			if (!inhead) {
				for (i=0; i<m->nadd; i++) {
					if (fprintf(fout,"%.300s\r\n",m->hdrs_add[i])<=0) failed=TRUE;
				}
			}

		}
		if (fprintf(fout,"%s",bf)<=0) failed=TRUE;
next_line:	;
	}
	fclose(fin);
	if (fclose(fout)!=0) failed=TRUE;
	if (!failed) {
		if (remove(m->msgfile)!=0) {
			m->imsg("Remove (%s) failed %s\n",m->msgfile,strerror(errno));
			return FALSE;
		}
		if (rename(msgfiletmp,m->msgfile)!=0) {
			m->imsg("rename failed (%s) (%s)\n",m->msgfile,strerror(errno));
			return FALSE;;
		}
		m->imsg("File rewritten (%s) %d\n",m->msgfile,inhead);
	} else {
		remove(msgfiletmp);
		m->imsg("Unable to rewrite file\n");
	}
	return TRUE;
}


int df_hdrs_free(Mfilter *m)
{
	int i;
	df_spamdetect_free(m);
	df_bounce_fast_clear(m);


	for (i=0; i<m->curprog->nheaders; i++) {
//m->hdrs[i] = 3;
		if (!m->hdrs_fake[i]) if (m->hdrs[i]!=NULL) {
			if (m->hdrs[i]< (char *) 2000) {
				m->imsg("df_hdrs_free: Signal panic Dubious free nheaders=%d [%d] %p\n",m->curprog->nheaders,i,m->hdrs[i]);
				m->imsg("df_hdrs_free: Dubious free nheaders=%d [%d] %p = %s\n",m->curprog->nheaders,i,m->hdrs[i],m->curprog->headers[i]);
			}
			if (m->hdrs[i]< (char *) 255) {
				FILE *f;
				m->imsg("df_hdrs_free: invalid address [%d] %p %s\n",i,m->hdrs[i],m->curprog->headers[i]);
				f = fopen("mfilter_hdr_crash.txt","wb");
				if (f!=NULL) {
					if (m->run_item!=NULL) fwrite(m->run_item,1,m->run_len,f);
					fclose(f);
				}
				m->imsg("ABORT NOW, AS HDRS WAS INVALID signal Signal \n");
				lib_abort();
			}
			myfree(m->hdrs[i]); // I've seen two crashes here.
		}
		m->hdrs[i] = NULL;
	}
	for (i=0; i<m->nadd; i++) {
		myfree(m->hdrs_add[i]);
		m->hdrs_add[i] = NULL;
	}
	m->nadd = 0;
	return TRUE;
}
char *df_strip_brackets(char *s, char *tmp2);
char *df_original(Fwddata *f,char *tmp2)
{
	if (f->rcpt==NULL) return df_strip_brackets(f->path,tmp2);
	return df_strip_brackets(f->rcpt,tmp2);
}
char *df_strip_brackets(char *s, char *bf)
{
	ncpy(bf,s,BFSZ-1);
	if (*s=='<') {
		ncpy(bf,s+1,200);
		s = strchr(bf,'>');
		if (s!=NULL) *s = 0;
		return bf;
	}
	return bf;
}
int df_forward_cc_free(Mfilter *m)
{
	Fwddata *fwd;
	Fwddata *next;
	for (fwd = m->forward_cc; fwd!=NULL; fwd=next) {
		next = fwd->next;
		myfree(fwd->path);
		free(fwd);
	}
	m->forward_cc = NULL;
	return TRUE;
}
int df_set_bounce(Mfilter *m, Fwddata *fwd, char *s)
{
	if (fwd==NULL) {
		m->imsg("Cannot bounce as not in recipients block\n");
		return TRUE;
	}
	fwd->cur = 1;
	if (fwd->uinfo!=NULL) myfree(fwd->uinfo);
	fwd->uinfo = mymalloc(512);
	ncpy(fwd->uinfo,s,200);
	return TRUE;
}
int df_already_sending(Fwddata *list, char *to)
{
	char bf[BFSZ];
	char tmp[BFSZ];
	char tmp2[BFSZ];
	ncpy(bf,df_strip_brackets(to,tmp),200);
	for (;list!=NULL;list=list->next) {
		if (strcmpnc(bf,df_original(list,tmp2))==0) return TRUE;
	}
	return FALSE;
}

int df_cmpfromto(Mfilter *m)
{
	char *s,*s2;
	char bto[BFSZ];
	char bfrom[BFSZ];
	char *lcl;
	ncpy(bto,df_hdrs_get(m,df_header_findadd(m,"to")),BFSZ-1);
	ncpy(bfrom,df_hdrs_get(m,df_header_findadd(m,"from")),BFSZ-1);

	lcl=NULL;
	s=mystrtok(bfrom,"(<>)",&lcl); s2=0;
	if(s) do {
		if(strchr(s,'@')) s2=s;
	} while(s=mystrtok(NULL,"(<>)",&lcl));

	if(!s2) return FALSE;

	strlwr(s2); strlwr(bto);

	if(strstr(bto,s2)) {
		df_setflag(m,df_flag_findadd(m,"FROMEQTO"),TRUE);
		return TRUE;
	}
	return FALSE;
}
int mf_min(int a, int b)
{
	if (a<b) return a;
	return b;
}
int mfilter_add_detect(Mfilter *m, char *s, double v)
{
	Spamlist *l;
	l = calloc(1,sizeof(Spamlist));
	l->next = m->add_splist;
	l->reason = strdup(s);
	l->score = (float) v;
	m->add_splist = l;
	return TRUE;
}
char *copy_paragraph(char *dest, char *src)
{
	char *was=dest;
	for (;*src!=0;) {
		if (dest-was>BFSZ-10) break;
		*dest++ = *src++;
		if (*src=='\n') if (src[1]=='\r') break;
		if (*src=='\n') if (src[1]=='\n') break;
	}
	*dest++= 0;
	if (*src!=0) src++;
	if (*src!=0) src++;
	if (*src=='\n') src++;
	if (*src=='\r') src++;
	if (*src=='\n') src++;
	if (*src=='\r') src++;
	return src;
}
int df_paragraphs(Mfilter *m, char *msg)
{
	// 	Find the first 3 paragraphs, most likely place for spam.
	char *s;

	strcpy(m->item_paragraph1,"");
	strcpy(m->item_paragraph2,"");
	strcpy(m->item_paragraph3,"");
	if (msg==NULL) return TRUE;

	// skip white space.
	for (s=msg;*s!=0;s++) {
		if (*s==' ') continue;
		if (*s=='\r') continue;
		if (*s=='\n') continue;
		if (*s=='\t') continue;
		break;
	}
	// copy until we hit a paragraph.
	s = copy_paragraph(m->item_paragraph1,s);
	s = copy_paragraph(m->item_paragraph2,s);
	s = copy_paragraph(m->item_paragraph3,s);

/*	m->fmsg("paragraph: raw msg %.200s\n",msg);
	m->fmsg("paragraph: %.100s\n",item_paragraph1);
	m->fmsg("paragraph: %.100s\n",item_paragraph2);
	m->fmsg("paragraph: %.100s\n",item_paragraph3);
*/

	return TRUE;
}
int mfilter_free_add(Mfilter *m)
{
	Spamlist *p,*next;
	for (p=m->add_splist;p!=NULL;p=next) {
		next = p->next;
		free(p->reason);
		free(p);
	}
	m->add_splist = NULL;
	return TRUE;
}
#include "rstring.h"
#include "mime.h"
int df_item_mime(Mfilter *mm, char *item, int len) // setup isimage and imagesize variables based on content...
{
	int inimage=FALSE;
	Mime *m;
	int k;
	char raw[BFSZ];
	char bin[BFSZ];
	FILE *f=NULL;
	int nlines=0;
	int inpdf=FALSE;

	mm->mitem_nimage = 0;
	mm->mitem_image_size = mm->mitem_isjpg = mm->mitem_isimage = FALSE;
	mm->mitem_ispdf = FALSE;
	mm->mitem_pdf_size = 0;
	ncpy(mm->attach_name,"",BFSZ-1);

	mime_new(&m);
	mime_setmem(m,item,len);
	m->loose_header = TRUE;
	for (k=TRUE;k;) {
		if (nlines>2000) break;
		nlines++;
		k = mime_line(m,f,raw,bin,&len,BFSZ-1);
		switch (k) {
		case MIME_END:
			break;
		case MIME_HEADER:
			if (lib_strlen(m->fname)>0) ncpy(mm->attach_name,m->fname,BFSZ-1);
			break;
		case MIME_CONTENT:
			break;
		case MIME_HEADER_END:
			inimage = FALSE;
			inpdf=FALSE;
			strlwr(m->content);
			if (strstr(m->content,"image")!=NULL) {
				mm->mitem_isimage = TRUE;
				if (strstrnc(m->content,"jpeg")!=NULL) mm->mitem_isjpg = TRUE;
				if (strstrnc(m->content,"jpg")!=NULL) mm->mitem_isjpg = TRUE;
				mm->mitem_nimage++;
				inimage = TRUE;
			}
			if (strstr(m->content,"application/pdf")!=NULL) {
				mm->mitem_ispdf = TRUE;
				inpdf = TRUE;
			}
			break;		
		case MIME_BODY:
			// Should buffer up 10K so we get multi line urls'
			if (inimage) mm->mitem_image_size += len; 
			if (inpdf) mm->mitem_pdf_size += len;			
			break;
		case MIME_BOUNDARY:
			break;
		}
	}
	if (lib_strlen(m->fname)>0) ncpy(mm->attach_name,m->fname,BFSZ-1);
	mime_free(&m);
	return TRUE;	
}
int dfilter_runx(int chan, FProg *fp, char *item, char *item_decoded, char *item_cleaned, int len, Fwddata **fwdlist, char *tfile, char *reason, Spamlist *startspam)
{
	Linex *l,*loop_point=NULL;
	char bf[BFSZ];
	int i;
	int r,r1,r2;
	int op;
	int notin = 0;
	Mfilter *m = fp->m;
	char tmp2[BFSZ];
	Exp *e;
	Fwddata *fwd;
#ifdef DEBUG_MFILTER
	Fwddata *ff;
#endif
	int gotone;
	int drop=FALSE;
	int msec;
	float spamscore;
	Fwddata *fwdlist_copy;

#ifdef MF_DECODE
	mf_counts bodycnts;
#endif

	msec = lstat_lib_msec();
	if (m->urls==NULL) m->urls = malloc(MAXILEN);
	m->run_item = item;
	m->run_len = len;
	fwdlist_copy =*fwdlist;
	m->curprog = fp;
	df_hdrs_free(m);
	m->ishidden = FALSE;
	m->is_rexp_fast = FALSE;
	strcpy(m->mfilter_to,"");
	strcpy(m->return_path,"");
	m->mfilter_unfriend = false;
	strcpy(m->mfilter_whitelist,"");
	strcpy(m->store_folder,"");
	m->dec_len=0;
	m->passed_splist=startspam;
	// Chrisp added, it should be cleared before we start
	strcpy(m->spamdetect_save,"");
	if (*fwdlist==NULL) {
		m->fmsg("drun: No 'To:' addresses specified, cannot run filter\n");
		m->imsg("drun: No 'To:' addresses specified, cannot run filter\n");
		df_hdrs_free(m);
		return FALSE;
	}
	// debugging to be removed
//	for (cur_fwd = fwdlist; cur_fwd!=NULL; cur_fwd=cur_fwd->next ) {
//		m->fmsg("Recipient env=(%.90s) path=(%.90s) info=(%.90s) \n",cur_fwd->env,cur_fwd->path,cur_fwd->uinfo);
//	}

#ifdef DEBUG_MFILTER
	m->imsg("Before chan %d\n",chan);
	for (ff=*fwdlist;ff!=NULL;ff=ff->next) {
		m->imsg("fwddata path (%s) uinfo (%s) %d\n",ff->path,ff->uinfo,ff->cur);
	}
	dfilter_trace(TRUE);
#endif

	//Check linked list is valid here.
	for (m->cur_fwd = *fwdlist; m->cur_fwd!=NULL; m->cur_fwd=m->cur_fwd->next ) {
		int i=0;
		if (m->cur_fwd->cur==0) i++;
	}


	ncpy(m->bounce_reason,"",10);
	strcpy(reason,"No Rule Applied");
	m->curprog = fp;
	m->item_size = len;
	item[len]='\0';
	df_forward_cc_free(m);
	df_item_mime(m,item,len); // setup isimage and imagesize variables based on content...
	df_hdrs_process(m,item, item_decoded, item_cleaned, m->urls);
	/*{
		FILE *f = fopen("decoded.txt","wb");
		printf("CREATING DECODED.TXT\n");
		if (f!=NULL) {
			fwrite(item_decoded,1,lib_strlen(item_decoded),f);
			fclose(f);
		}
	}*/

	df_paragraphs(m,item_decoded);
/*	mf_fast_scan(item,len);*/
	m->dec_len=mf_min(lib_strlen(item_decoded), m->dec_len);

#ifdef MF_DECODE
//printf("decoe_url here\n");
	bodycnts.encalpha=0;
	df_decode_url(*item_decoded?item_decoded:item,0,&bodycnts,0);
//	if(bodycnts.encalpha-(cntdx=df_flag_get(df_flag_findadd(m,"url_alpha_encoded")))>0) df_setflag(m,df_flag_findadd(m,"body_alpha_encoded"), bodycnts.encalpha-cntdx);
#endif

	m->cur_fwd = NULL;
	ncpy(m->msgfile,tfile,BFSZ-1);

	df_cmpfromto(m);
	

	for (l=fp->lines;l!=NULL;l=l->next) {
/*m->fmsg("drun:%d %saction=%s Line: %.40s...\n",notin,notin>0 ? "SKIP " : "",actions[l->action],l->src);*/
		if (m->trace) if (lib_strlen(l->src) < 1000) {
			char sbf[BFSZ];
			lprintf(sbf,BFSZ-1,"SKIP%d ",notin);
			m->imsg("drun: %.90saction=%.90s Line: %.40s...\n",notin>0 ? sbf : "",actions[l->action],l->src);
		}
		if (notin>0) {
			if (l->action==A_THEN) notin++;
			if (l->action==A_END) {notin--; continue;}
			if (notin==1) if (l->action==A_ELSE) {notin--; continue;}
			continue;
		} else {
			if (l->action==A_ELSE) {notin++; continue;}
		}
		e = l->elist;
		if (e==NULL) {
			goto doit;
		}
		for (; e!=NULL; e=e->next) {

			r = FALSE;
#ifdef TESTING
			if (m->trace) m->imsg("dfilter: %.90s%.90s header:%d p2=%.90s op=(%.40s)\n"
				,e->isnot ? "!" : "",flist[e->fn].name,e->header,e->p2,operators[e->binop]);
#endif
			if (e->fn>=0) r = (*flist[e->fn].run)(e);
			if (e->isnot) r = !r;
			if (m->trace) if (r) m->imsg("drun: Expression Result=%s %d\n",r ? "TRUE" : "FALSE" , r);
			if (e->binop!=O_AND) if (e->next!=NULL) {
				op = e->binop;
				e = e->next;
				r1 = r;
				if (e->fn>=0) r2 = (*flist[e->fn].run)(e);
				if (e->isnot) r2 = !r2;
				if (m->trace) m->imsg("drun: Result %s %d %.90s %d\n",r1 ? "TRUE" : "FALSE" ,r1,operators[op],r2);
				r = 0;
				if (op==O_GREATERTHAN) {if (r1>r2) r = 1;}
				else if (op==O_LESSTHAN) {if (r1<r2) r = 1;}
				else if (op==O_EQUALS) {if (r1==r2) r = 1;}
			}
			if (l->action!=A_THEN) if (e->next!=NULL) if (!r) break; /* Next Line*/
			if (l->action==A_THEN) {
				if (!r) {
					notin++;
					break;
				}
			} else if (r && (e->next==NULL)) {
doit:				if (l->action==A_ACCEPT) {
					sprintf(reason,"line %d: %.200s",l->lineno,l->reason);
					//Not sensible accept must mean accept all cases
					//if (cur_fwd!=NULL) goto recipients_accept;
					goto return_true;
				}
				if (l->action==A_REJECT) {
					sprintf(reason,"line %d: %.200s",l->lineno,l->reason);
					if (m->cur_fwd!=NULL) goto recipients_bounce;
					goto return_false;
				}
				if (l->action==A_DO_BOUNCE_FAST) {
					char tmp[BFSZ];
					if (df_bounce_fast_find(m,item_decoded,m->dec_len,tmp)) {
						sprintf(reason,"line %d: %.200s",l->lineno,tmp);
						if (m->cur_fwd!=NULL) goto recipients_bounce;
						goto return_false;
					}
				}
				if (l->action==A_DROP) {
					sprintf(reason,"line %d: %.200s",l->lineno,l->reason);
					drop = TRUE;
					if (m->cur_fwd!=NULL) goto recipients_bounce;
					goto return_false;
				}
				if (l->action==A_PRINT) {
					m->imsg("PRINT line %d:%.90s\n",l->lineno,l->reason);
				}
				if (l->action==A_SETFLAG) df_setflag(m,l->p1,TRUE);
				if (l->action==A_CLEARFLAG) df_setflag(m,l->p1,FALSE);
				if (l->action==A_SET) df_setflag(m,l->p1,l->p2);
				if (l->action==A_ADD) df_addflag(m,l->p1,l->p2);
				if (l->action==A_SUB) df_subflag(m,l->p1,l->p2);
				if (l->action==A_SCORE_ONE) df_scoreoneflag(m,l->p1);
				if (l->action==A_SCORE_ALL) df_scoreallflag(m,l->p1);
				if (l->action==A_FORWARD) {
					sprintf(reason,"line %d: forward %.200s",l->lineno,l->reason);
					if (m->cur_fwd==NULL) { /* If we are not in recipient block then forward to this single user */
						m->imsg("Warning, undefined results, forward should be in a recipient block in %s\n",m->mfilter_fname);
						df_forward(*fwdlist,l->reason,l->lineno);
						m->cur_fwd = (*fwdlist)->next;
						/* And drop to any others */
						for (;m->cur_fwd!=NULL;m->cur_fwd=m->cur_fwd->next) {
							sprintf(bf,"999 %.100s %.100s",m->cur_recipient,reason);
							df_set_bounce(m,m->cur_fwd,bf);
						}
					} else {
						df_forward(m->cur_fwd,l->reason,l->lineno);
					}
					goto return_true;
				}
				if (l->action==A_REDIRECT_CC) {
					Exp dummy;
					dummy.p2 = l->reason;
					run_forward_cc(&dummy); // Reuse existing cc code
				}
				if (l->action==A_RECIPIENTS) {
					/* Loop from here thru recipient list processing lines */
					loop_point = l;
					m->cur_fwd = *fwdlist;
					ncpy(m->cur_recipient,df_original(m->cur_fwd,tmp2),BFSZ-1);
					if (m->noisey) m->imsg("recipient %s\n",m->cur_recipient);
				}

			}
			if (l->action==A_RECIPIENTS_END) {
				drop = FALSE;
				/* Loop from here thru recipient list processing lines */
end_again:			m->cur_fwd = m->cur_fwd->next;
				if (m->cur_fwd!=NULL) {
					ncpy(m->cur_recipient,df_original(m->cur_fwd,tmp2),BFSZ-1);
					if (m->noisey) m->imsg("recipient %s\n",m->cur_recipient);
					if (loop_point!=NULL) l = loop_point;
					drop = FALSE;
				} else {
					/* Find recipients end if not already there */
					for (;l!=NULL;l=l->next) if (l->action==A_RECIPIENTS_END) break;
				}
				break;
/*
recipients_accept:		m->imsg("Accepted %.100s %.100s\n",cur_recipient,reason);
				goto end_again;
*/
recipients_bounce:		m->imsg("Bounce mfilter %.100s %.100s\n",m->cur_recipient,reason);
				sprintf(bf,"570 %.100s %.100s",m->cur_recipient,reason);
				if (drop) sprintf(bf,"999 %.100s %.100s",m->cur_recipient,reason);
				df_set_bounce(m,m->cur_fwd,bf);
				ncpy(m->bounce_reason,reason,BFSZ-1);
				if (m->cur_fwd) goto end_again;
			}
			if (e==NULL) break;
		}
		// un commented for cpu timing

		if (l==NULL) break;
	}
	if (m->trace) m->imsg("dfilter: No rules applied, so accepting\n");
return_true:
	
	mf_hidden_scan(m,item_decoded,m->dec_len);
	mf_fast_scan(fp,item_decoded, m->dec_len);


	/* Check that someone is still a valid recipieint*/
	gotone = FALSE;
/*m->imsg("fwdlist at end is %p\n",fwdlist);*/

	if (fwdlist_copy !=*fwdlist) {
		m->imsg("PANIC, FWDLIST CHANGED %p %p\n",fwdlist_copy,fwdlist);
		lib_abort();
	}

	for (m->cur_fwd = *fwdlist; m->cur_fwd!=NULL; m->cur_fwd=m->cur_fwd->next ) {
		if (m->cur_fwd->cur==0) gotone = TRUE;
		if (m->trace) m->imsg("Recipient (%.90s) %.90s %.50s\n",m->cur_fwd->path,m->cur_fwd->cur ? "Bounced" : "Accepted",m->cur_fwd->uinfo==NULL ? "" : m->cur_fwd->uinfo);
/*		m->imsg("Recipient (%.90s) %.90s %.50s\n",cur_fwd->path,cur_fwd->cur ? "Bounced" : "Accepted",cur_fwd->uinfo==NULL ? "" : cur_fwd->uinfo); */
	}
	for (fwd = m->forward_cc; fwd!=NULL; fwd=fwd->next) {
		gotone = TRUE;
	}
	if (!gotone) {
		m->imsg("Boucing message immediately as all recipients were bounced\n");
		if (lib_strlen(m->bounce_reason)>0) ncpy(reason,m->bounce_reason,100);
		goto return_false;
	}
	for (fwd = m->forward_cc; fwd!=NULL; fwd=fwd->next) {
		if (df_already_sending(*fwdlist, fwd->path)) continue;
		add_fwd(fwdlist,chan,fwd->path);
	}
	spamscore=df_spamdetect_do(m);

	

	if(!m->hdrs_external) {
		if (m->noisey) m->imsg("checking for changed headers\n");
		for (i=0; i<=m->curprog->nheaders; i++) {
			if ((m->hdrs_changed[i]) || (m->nadd>0)) {
				df_rewrite_message(m);
				break;
			}
		}
	}	else {
		m->nadd=0;
		if (m->noisey) m->imsg("hdrs_external set so not rewriting message\n");

	}

	df_hdrs_free(m);

	df_forward_cc_free(m);

	/*
	m->imsg("After \n");
	for (ff=fwdlist;ff!=NULL;ff=ff->next) {
		m->imsg("fwddata path (%s) uinfo (%s) \n",ff->path,ff->uinfo);
	}
	*/
	
	return TRUE;
return_false:

	// Do spam scoring even on failures. CHRISP bug fix for drop rules in local.rul
	mf_hidden_scan(m,item_decoded,m->dec_len);
	mf_fast_scan(fp,item_decoded, m->dec_len);
	spamscore=df_spamdetect_do(m);

	if (m->cur_fwd==NULL) m->cur_fwd = *fwdlist;
	ncpy(m->cur_recipient,df_original(m->cur_fwd,tmp2),BFSZ-1);
	m->imsg("Bounced mfilter %.100s %.100s\n",m->cur_recipient,reason);
	if (drop) {
		sprintf(bf,"999 mfilter Drop %.200s",reason);
	} else {
		sprintf(bf,"570 mfilter Bounced %.200s",reason);
	}

	for (m->cur_fwd = *fwdlist; m->cur_fwd!=NULL; m->cur_fwd=m->cur_fwd->next ) {
		df_set_bounce(m,m->cur_fwd,bf);
	}
	df_hdrs_free(m);
	df_forward_cc_free(m);
	
	return FALSE;
}
int dfilter_show_stats(Mfilter *m, FProg *fp, char *fname)
{
	FILE *f=NULL;
	Linex *l=NULL;
	f = fopen(fname,"w");
	if (f!=NULL) {
		for (l=fp->lines;l!=NULL;l=l->next) {
			fprintf(f,"%8d %s\n",l->cpu,l->src);
		}

		fclose(f);
	}
	return TRUE;
}

int dfilter_run(Mfilter *m, int chan, FProg *fp, char *item, char *item_decoded, char *item_cleaned, int len, Fwddata **fwdlist, char *tfile, char *reason, Spamlist *startspam)
{
	int r;
	m->nact++;
	m->item_string = item;
	r=dfilter_runx(chan, fp, item, item_decoded, item_cleaned, len, fwdlist, tfile, reason, startspam);
	m->item_string = NULL;

	return r;
}

void dline_free(Linex *l)
{
	Exp *e,*enext;
	for (e = l->elist; e!=NULL; e=enext) {
			enext = e->next;
			if (e->p2!=NULL) myfree(e->p2);
			if (e->p3!=NULL) myfree(e->p3);
			if (e->pcreg!=NULL) pcre_freex(e->pcreg);
			myfree(e);
	}
}

void dfilter_free(Mfilter *m, FProg *fp)
{

	int i;
	Linex *l,*lnext;
	mf_fast_free(fp);
	word_free(&fp->feature);
	for (l=fp->lines;l!=NULL;l=lnext) {
		lnext = l->next;

		dline_free(l);
		if (l->src!=NULL) myfree(l->src);
		if (l->reason!=NULL) myfree(l->reason);
		myfree(l);
	}

	for (i=0; i<fp->nheaders; i++) {
		myfree(fp->headers[i]); fp->headers[i] = NULL;
	}
	fp->nheaders = 0;

	for (i=0; i<fp->nflags; i++) {
		myfree(fp->flags[i]); fp->flags[i] = NULL;
	}
	fp->nflags = 0;
	df_expr_free(fp);

	myfree(fp);
}
char *dfilter_error(Mfilter *m)
{
	return m->derr;
}


/* Handle   $a = "xxx" + "yyy" */
int  df_comp_assign(Mfilter *m, char *p[], int np, int qq[])
{
	char bf[MAX_LINE];
	int i;
	int len=0;
	df_var_replace(m,p,np,qq);
	for (i=2; i<np; i+=2) {
		if (len+lib_strlen(gp(m,i))+100>MAX_LINE) {
			m->fmsg("dfilter: Line too long in assign line=%d\n",m->lineno);
			return FALSE;
		}
		strcpy(bf+len,gp(m,i));
		len += lib_strlen(gp(m,i));
		if (np>=i+2) if (strcmp(gp(m,i+1),"+")!=0) {
			m->fmsg("dfilter: Missing '+' in assignment line=%d\n",m->lineno);
			return FALSE;
		}
	}
	df_var_set(m,gp(m,0),bf);
	return TRUE;

}


void df_var_set(Mfilter *m, char *var, char *val)
{
	Pairs *v;
	if (strncmp(var,"$",1)==0) {
	    for (v=m->vlist;v!=NULL;v=v->next) if (strcmp(var,v->var)==0) {
		    m->fmsg("Duplicate definition of variable (%.90s) line %d\n",var,m->lineno);
		    myfree(v->val);
		    v->val = mystrdup(val);
		    return;
		}
	}
	else
	{
		m->fmsg("Variable name must begin with $ (%.90s) line %d\n",var,m->lineno);
		return;
	}
	v = mymalloc(sizeof(Pairs));
	v->next = m->vlist;
	m->vlist = v;
	v->var = mystrdup(var);
	v->val = mystrdup(val);
}

void df_var_free(Mfilter *m )
{
	Pairs *v,*vnext;
	for (v=m->vlist;v!=NULL;v=vnext) {
		vnext = v->next;
		myfree(v->val);
		myfree(v->var);
		myfree(v);
	}
	m->vlist = NULL;
}
char *df_var_get(Mfilter *m, char *var)
{
	Pairs *v;
	for (v=m->vlist;v!=NULL;v=v->next) {
		if (strcmp(var,v->var)==0) {
			return v->val;
		}
	}
	return NULL;
}
int df_var_replace(Mfilter *m, char *p[], int np, int qq[])
{
	int i;
	char *v;
	for (i=1; i<np; i++) {
		if (qq[i]) continue;
		if (strncmp(gp(m,i),"$",1)==0) {
			v = df_var_get(m,gp(m,i));
			if (v==NULL) {
				m->fmsg("dfilter: Undefined variable, line %d variable %.90s\n",m->lineno,gp(m,i));
			} else p[i] = v;
		}
	}
	return TRUE;
}
/*
Process if to, Replace variables
	if rexp ( subject , [Ss][Ee][Xx] ) and !isbinary() reject Naughty Subject

if exp [and exp...] action


------------------------------------------------------------------------
if !match("nntp-posting-host",$local_hosts) then
	if rexp("subject",$bad_subjects) reject "Naughty Subject"
else
	if rexp("subject","[Ss][Ee][Xx]") and match("domain","*.sex.*") reject "Naughty Subject in non naughty group"
	if inpath($bad_cancel_paths) reject "Came from spam domain"

------------------------------------------------------------------------
*/

Linex *df_comp_if (FProg *fp, char *p[], int np, int qq[])
{
	int upto = 1;
	Exp *e,*elast=NULL;
	Linex *eline;
	Mfilter *m = fp->m;
	df_var_replace(m,p,np,qq);
	eline = mymalloc(sizeof(Linex));
	memset(eline,0,sizeof(Linex));
	if (strcmp(gp(m,upto),"(")!=0) {
		m->fmsg("If must be followed by left bracket not (%.40s) line=%d\n",gp(m,upto),m->lineno);
		return eline;
	}
	upto++;
	for (;upto<np;) {
		e = df_comp_exp(fp,p,np,&upto);
		if (e==NULL) {
			dline_free(eline);
			return NULL;
		}
		e->binop = O_AND;
		if (e==NULL) break;
		if (eline->elist==NULL) eline->elist = e;
		else elast->next = e;
		elast = e;
		/* Expecting right bracket or operator <>= */
		if (strcmp(gp(m,upto),"<")==0) {
			e->binop = O_LESSTHAN;
		} else if (strcmp(gp(m,upto),">")==0) {
			e->binop = O_GREATERTHAN;
		} else if (strcmp(gp(m,upto),"=")==0) {
			e->binop = O_EQUALS;
		} else if (strcmp(gp(m,upto),")")==0) {
			upto++;
			if (strcmp(gp(m,upto),"and")==0) {
				upto++;
				if (strcmp(gp(m,upto),"(")!=0) {
					m->fmsg("If must be followed by left bracket not (%.40s) line=%d\n",gp(m,upto),m->lineno);
					return eline;
				}
			} else break;
		} else {
			m->fmsg("If exp must be followed by right bracket not (%.40s) line=%d\n",gp(m,upto),m->lineno);
			return eline;
		}
		upto++;
	}
	if (upto>=np) {
		m->fmsg("Expecting action, e.g. reject,accept,then.. (line=%d)\n",m->lineno);
	}
	if (!df_find_action(m,&eline,p,np,&upto)) {
		m->fmsg("Unknown action (%.40s) lineno=%s:%d\n",gp(m,upto),m->compile_fname,m->lineno);
	}
	return eline;
}
Linex *df_comp_if_true (FProg *fp,char *was)
{
	char bfline[MAX_LINE+10];
	int qq[1000];
	char *p[1000];
	Mfilter *m = fp->m;
	int np;
	char bf[BFSZ];
	if (strstr(was,"then")==NULL) return NULL;

	strcpy(bf,"if (0) then");
	np = df_token(bf,p,990,qq,bfline); m->global_np = np; m->global_p = p;
	return df_comp_if(fp,p,np,qq);
}

Linex *df_comp_call(FProg *fp, char *p[], int np, int qq[])
{
	int upto = 1;
	Exp *e,*elast=NULL;
	Linex *eline;
	Mfilter *m = fp->m;
	df_var_replace(m,p,np,qq);
	eline = mymalloc(sizeof(Linex));dmalloc_ignore_leak();
	memset(eline,0,sizeof(Linex));
	for (;upto<np;) {
		e = df_comp_exp(fp,p,np,&upto);
		if (e==NULL) {
			dline_free(eline);
			return NULL;
		}
		e->binop = O_AND;
		if (e==NULL) break;
		if (eline->elist==NULL) eline->elist = e;
		else elast->next = e;
		elast = e;
		/* Expecting right bracket or operator <>= */
		if (strcmp(gp(m,upto),"<")==0) {
			e->binop = O_LESSTHAN;
		} else if (strcmp(gp(m,upto),">")==0) {
			e->binop = O_GREATERTHAN;
		} else if (strcmp(gp(m,upto),"=")==0) {
			e->binop = O_EQUALS;
		} else if (strcmp(gp(m,upto),")")==0) {
			upto++;
			if (strcmp(gp(m,upto),"and")==0) {
				upto++;
				if (strcmp(gp(m,upto),"(")!=0) {
					m->fmsg("If must be followed by left bracket not (%.40s) line=%d\n",gp(m,upto),m->lineno);
					return eline;
				}
			} else break;
		} else {
			return eline;
		}
		upto++;
	}
	return eline;
}
Linex *df_comp_else(char *p[], int np)
{
	int upto = 1;
	Linex *eline;
	eline = mymalloc(sizeof(Linex));
	memset(eline,0,sizeof(Linex));
	eline->action = A_ELSE;
	return eline;
}
Linex *df_comp_end(char *p[], int np)
{
	int upto = 1;
	Linex *eline;
	eline = mymalloc(sizeof(Linex));
	memset(eline,0,sizeof(Linex));
	eline->action = A_END;
	if (np>1) if (strcmp(p[1],"recipients")==0) {
		eline->action = A_RECIPIENTS_END;
	}
	return eline;
}
/*
! match ( nntp-posting-host , x.y,z,a.b.c )
rexp ( subject , [Ss][Ee][Xx] )
inpath ( xxx,mci )
typedef struct EXP {int fn, int header, int isnot; char *p2} Exp;
*/

Exp *df_comp_exp(FProg *fp, char *p[], int np, int *upto)
{
	Mfilter *m = fp->m;
	int u = *upto;
	int start=*upto;
	int f,r;
	Exp *e;
	char *args[10];

	if (u>=np) {
		m->fmsg("No expression on line %d\n",m->lineno);
		return NULL;
	}
	e = mymalloc(sizeof(Exp));
	memset(e,0,sizeof(Exp));
	e->m = m;
	e->fn = -1;
	if (strcmp(gp(m,u),"!")==0) {
		u++;
		e->isnot = TRUE;
	}
	if (u>=np) goto failed;
/*

rexp ( subject , [Ss][Ee][Xx] )
inpath ( xxx,mci )

*/
	for (f = 0; flist[f].name!=NULL; f++) {
		if (strcmp(gp(m,u),flist[f].name)==0) {
			if (flist[f].narg==0) {
				if (strcmp(gp(m,u+1),"(")!=0) goto failed2;
				if (strcmp(gp(m,u+2),")")!=0) goto failed2;
			}
			if (flist[f].narg==1) {
				if (strcmp(gp(m,u+1),"(")!=0) goto failed2;
				if (strcmp(gp(m,u+3),")")!=0) goto failed2;
			}
			if (flist[f].narg==2) {
				if (strcmp(gp(m,u+1),"(")!=0) goto failed2;
				if (strcmp(gp(m,u+3),",")!=0) goto failed2;
				if (strcmp(gp(m,u+5),")")!=0) goto failed2;
			}
			if (flist[f].narg==3) {
				if (strcmp(gp(m,u+1),"(")!=0) goto failed2;
				if (strcmp(gp(m,u+3),",")!=0) goto failed2;
				if (strcmp(gp(m,u+5),",")!=0) goto failed2;
				if (strcmp(gp(m,u+7),")")!=0) goto failed2;
				args[0] = gp(m,u+6);
				args[1] = NULL;
			}
			if (flist[f].narg==4) {
				if (strcmp(gp(m,u+1),"(")!=0) goto failed2;
				if (strcmp(gp(m,u+3),",")!=0) goto failed2;
				if (strcmp(gp(m,u+5),",")!=0) goto failed2;
				if (strcmp(gp(m,u+7),",")!=0) goto failed2;
				if (strcmp(gp(m,u+9),")")!=0) goto failed2;
				args[0] = gp(m,u+6);
				args[1] = gp(m,u+8);
				args[2] = NULL;
			}
			r = (*flist[f].compile)(fp,e,gp(m,u+2),gp(m,u+4),args);

			e->fn = f;
			if (!r) goto failedc;
			u += 2 + 2*flist[f].narg;
			if (flist[f].narg==0) u++;
			*upto = u;
			return e;
		}

	}
	if (sisdigit(*gp(m,u))||((*gp(m,u)=='-')&&(*(gp(m,u)+1)!=0)&&(sisdigit(*(gp(m,u)+1))))) {
		f = 0;
		r = (*flist[f].compile)(fp,e,gp(m,u),gp(m,u),args);
		e->fn = f;
		if (!r) goto failedc;
		u += 1;
		*upto = u;
		return e;
	}
	m->fmsg("No such function (%.40s) on line %d\n",gp(m,u),m->lineno);

failed:
	myfree(e);
	m->fmsg("Invalid expression on line %d %d %.90s...\n",m->lineno,m->nthis,gp(m,start));
	return NULL;
failedc:
	myfree(e);
	m->fmsg("Cant compile expression on line %d %d %.90s...\n",m->lineno,m->nthis,gp(m,start));
	return NULL;
failed2:
	myfree(e);
	m->fmsg("Invalid expression braces on line %d %d %.90s...\n",m->lineno,m->nthis,gp(m,start));
	return NULL;
}
/*-----------------------------------------------------------------*/
/* Compile functions, fill out 'exp' structure */
/* typedef struct EXP {int fn, int header, int isnot; char *p2} Exp;*/
int comp_constant(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup("");
	exp->header = atoi(arg1);
	return TRUE;
}
int comp_match(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg2);
	exp->header = df_header_findadd(exp->m,arg1);
	return TRUE;
}

int comp_replace(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg2);
	exp->p3 = mystrdup(arg3[0]);
	exp->header = df_header_findadd(exp->m,arg1);
	return TRUE;
}
int comp_ishidden(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg2);
	exp->p3 = mystrdup(arg3[0]);
	exp->header = df_header_findadd(exp->m,arg1);
	return TRUE;
}
int comp_ismixedcase(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->header = df_header_findadd(exp->m,arg1);
	return TRUE;
}
int comp_report(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg1);
	exp->p3 = mystrdup(arg2);
	return TRUE;
}
int comp_spamdetect(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{

	exp->p2 = mystrdup(arg1);
	exp->p3 = mystrdup(arg2);
	return TRUE;
}
int comp_feature_header(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	char bf[BFSZ];
	lprintf(bf,BFSZ-1,"head|0|%s|%s|0|",arg1,arg2);
	word_add_nodup(&fp->feature,bf);
	exp->p2 = mystrdup(arg1);
	exp->p3 = mystrdup(arg2);
	return TRUE;
}
int comp_feature_headerm(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	char bf[BFSZ];
	lprintf(bf,BFSZ-1,"manual|0|%s|%s|%s|",arg1,arg2,arg3[0]);
	word_add_nodup(&fp->feature,bf);
	exp->p2 = mystrdup(arg1);
	exp->p3 = mystrdup(arg2);
	return TRUE;
}
int comp_feature_add(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	//head|8|X-myrbl|Color=black|0.5|sig=0
	char bf[BFSZ];
	exp->p2 = mystrdup(arg1);
	exp->p3 = mystrdup(arg2);
	lprintf(bf,BFSZ-1,"head|0|x-spamdetect|%s|0|",arg2);
	word_add_nodup(&fp->feature,bf);
	return TRUE;
}
int comp_feature_manual(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	//head|8|X-myrbl|Color=black|0.5|sig=0
	char bf[BFSZ];
	exp->p2 = mystrdup(arg1);
	exp->p3 = mystrdup(arg2);
	lprintf(bf,BFSZ-1,"manual|0|x-spamdetect|%s|%2.3f|",arg2,atof(arg1));
	word_add_nodup(&fp->feature,bf);
	return TRUE;
}
int feature_tomanual(Wordlist **words, double score, char *header, char *contains)
{
	// Scan through words, and replace any rule that matches with identical 'manual' rule... 
	Wordlist *w;
	char *p[100];
	char bf[BFSZ];
	char rule[BFSZ];
	int np;
//printf("tomanual header (%s) contains (%s) score %f\n",header,contains,score);
	for (w=*words;w!=NULL;w=w->next) {
		ncpy(bf,w->word,BFSZ-1);
		np = token_split(bf,p,"|",10);
		if (np>=5) {
			if ((strcmp(p[0],"head")==0) || (strcmp(p[0],"manual")==0) ) {
				if (strcmpnc(p[2],header)==0) {
					if (strcmpnc(p[3],contains)==0) {
						// Then replace the rule...
						lprintf(rule,BFSZ-1,"manual|0|%s|%s|%2.3f|",header,contains,score);
//printf("Replacing (%s) with (%s)\n",w->word,rule);
						free(w->word);
						w->word = strdup(rule);
					}
				}
			}
		}
	}
	return TRUE;
}
int comp_tomanual(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	//tomanual(score,header,color=black) head|8|X-myrbl|Color=black|0.5|sig=0
	feature_tomanual(&fp->feature,atof(arg1),arg2,*arg3);
	return TRUE;
}
int comp_exists(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->header = df_header_findadd(exp->m,arg1);
	return TRUE;
}
int comp_folder_exists(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg1);
	return TRUE;
}
int comp_is_dayofweek(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg1);
	return TRUE;
}
int comp_allmod(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->header = df_header_findadd(exp->m,arg1);
	return TRUE;
}
int comp_head_len(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->header = df_header_findadd(exp->m,arg1);
	return TRUE;
}
int comp_forward_cc(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg1);
	return TRUE;
}
int comp_attached(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg1);
	return TRUE;
}
int comp_matchall(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg2);
	exp->header = df_header_findadd(exp->m,arg1);
	return TRUE;
}
int comp_matchone(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg2);
	exp->header = df_header_findadd(exp->m,arg1);
	return TRUE;
}
int comp_date_offset(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg1);
	exp->header = df_header_findadd(exp->m,"Date");
	return TRUE;
}
int comp_set_tohost(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg1);
	return TRUE;
}
int comp_set_returnpath(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg1);
	return TRUE;
}
int comp_unfriend(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	return TRUE;
}
int comp_store_folder(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg1);
	return TRUE;
}
int comp_isin(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg2);
	exp->header = df_header_findadd(exp->m,arg1);
	 return TRUE;
}
int comp_bounce_fast(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg1);
	exp->p3 = mystrdup(arg2);
	strlwr(exp->p2);
	return TRUE;
}
int comp_rexp_fast(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	char bf[BFSZ];
	lprintf(bf,BFSZ-1,"head|0|x-spamdetect|%s|0|",arg3[0]);
	word_add_nodup(&fp->feature,bf);

	mf_fast_add(fp,atof(arg1),arg2,arg3[0],NULL);
	return TRUE;
}

int comp_hidden_word(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	Hidden *h;
	Mfilter *m = fp->m;
	char bf[BFSZ];
	h = calloc(1,sizeof(Hidden));
	mf_normalize(arg2,bf,BFSZ-1,lib_strlen(arg2));
	h->word = strdup(bf);
	h->original = strdup(arg2);
	h->score = atof(arg1);
	h->next = m->hlist;
	m->hlist = h;
	return TRUE;
}
int comp_add_aspam(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg1); // Scale factor 1.0
	exp->p3 = mystrdup(arg2); // reason to log
	return TRUE;
}
int comp_add_header(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg1); // Full header to add
	return TRUE;
}
int comp_add_poly(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg1); // Scale factor 1.0
	exp->p3 = mystrdup(arg2); // reason to log
	return TRUE;
}
int comp_rexp_fast_flag(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[]) // not used in aspam_mfilter.txt I think.
{
	mf_fast_add(fp,atof(arg1),arg2,arg3[0],arg3[1]);
	return TRUE;
}
int comp_isbinaryn(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->header = atoi(arg1);
	return TRUE;
}
int comp_spawn(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg1);
	return TRUE;
}
int comp_ifflag(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->header = df_flag_findadd(exp->m,arg1);
	return TRUE;
}
int comp_crash(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	return TRUE;
}
int comp_get(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	Mfilter *m = fp->m;
	exp->header = df_flag_findadd(m,arg1);
	return TRUE;
}
int comp_equals(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg2);
	exp->header = df_header_findadd(exp->m,arg1);
	return TRUE;
}
int comp_inpath(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	return FALSE;
}
int comp_rexp(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	Mfilter *m = fp->m;
	pcre *pcreg;
	pcre_extra *pcreg_hints;
	char *pcre_error;
	int pcre_errloc;
	exp->p2 = mystrdup(arg2);
	exp->header = df_header_findadd(exp->m,arg1);

	pcreg=pcre_compile(arg2, PCRE_CASELESS | PCRE_EXTENDED | PCRE_MULTILINE,  (const char **)&pcre_error, &pcre_errloc, NULL);

	if(pcre_error!=NULL)
	{
		m->fmsg("Regular expression failed at %d in (%.40s) (%.40s)\n",pcre_errloc, arg2, pcre_error);
		pcre_freex(pcreg);
		return FALSE;
	}
	exp->pcreg = pcreg;

	pcreg_hints=pcre_study(pcreg, 0, (const char **)&pcre_error);
	if(pcre_error==NULL)
		exp->hints=pcreg_hints;
	else
		myfree(pcreg_hints);

	return TRUE;
}
int mfilter_test_rexp(char *exp, char *sample, char *result)
{
	pcre *pcreg;
	pcre_extra *pcreg_hints;
	char *pcre_error;
	int pcre_errloc;
	int r;
	int offsets[99];

	strcpy(result,"none");
	pcreg=pcre_compile(exp, PCRE_CASELESS | PCRE_EXTENDED | PCRE_MULTILINE,  (const char **)&pcre_error, &pcre_errloc, NULL);

	if(pcre_error!=NULL)
	{
		lprintf(result,BFSZ-1,"Regular expression failed %d %s\n",pcre_errloc, pcre_error);
		pcre_freex(pcreg);
		return FALSE;
	}
	pcreg_hints=pcre_study(pcreg, 0, (const char **)&pcre_error);
	if(pcre_error!=NULL) {
		myfree(pcreg_hints);
		pcreg_hints = NULL;
	}
	r = pcre_exec(pcreg, pcreg_hints, sample, lib_strlen(sample), 0, 0, offsets, 99);
	lprintf(result,20,"r=%d",r);
	if(r<0) return FALSE;
	return TRUE;
}
int util_rexp(char *exp, char *sample, char *result, int *len)
{
	pcre *pcreg;
	pcre_extra *pcreg_hints;
	char *pcre_error;
	int pcre_errloc;
	int r;
	int offsets[99];
	strcpy(result,"none");
	*len = 0;
	pcreg=pcre_compile(exp, PCRE_CASELESS | PCRE_EXTENDED | PCRE_MULTILINE,  (const char **)&pcre_error, &pcre_errloc, NULL);

	if(pcre_error!=NULL)
	{
		lprintf(result,BFSZ-1,"Regular expression failed %d %s\n",pcre_errloc, pcre_error);
		pcre_freex(pcreg);
		return -1;
	}
	pcreg_hints=pcre_study(pcreg, 0, (const char **)&pcre_error);
	if(pcre_error!=NULL) {
		myfree(pcreg_hints);
		pcreg_hints = NULL;
	}
	r = pcre_exec(pcreg, pcreg_hints, sample, lib_strlen(sample), 0, 0, offsets, 99);
	if(r<0) return -1;
	*len = offsets[1]-offsets[0];
	return offsets[0];
}
int comp_issimilar(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup(arg2);
	exp->header = df_header_findadd(exp->m,arg1);
	return TRUE;
}
int comp_wrongdomain(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->header2 = df_header_findadd(exp->m,arg2);
	exp->header = df_header_findadd(exp->m,arg1);
	return TRUE;
}
int comp_wronguser(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->header2 = df_header_findadd(exp->m,arg2);
	exp->header = df_header_findadd(exp->m,arg1);
	return TRUE;
}
int comp_forged(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->header = df_header_findadd(exp->m,arg1);
	return TRUE;
}
int comp_fromtwo(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->header = df_header_findadd(exp->m,arg1);
	return TRUE;
}

int comp_rexpstore(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p3=mystrdup(arg3[0]);

	return comp_rexp(fp,exp,arg1,arg2,arg3);
}

int comp_evalflags(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->header = df_expr_findadd(fp->m,arg1);

	if(exp->header>=0)
		return TRUE;
	return FALSE;
}

/*
Fake lookahead operator

  xxx(bbbb|ccc)xxx|free(?!bsd|cats)xx(b)x|pic(?!ton)ture


  /(/)
output=  xxxxxxx|freexxx|picture
mask  =  free(bsd|cats)xxx|pic(ton)ure

  (free(?!bsd|cats)xxx|pic(?!ton)ture)


*/

int lookahead_mask(char *outbf,char *maskbf)
{
	char *s;
	char *p;
	char amask[BFSZ];
	int depth;
	char *start;
	char *out;
	int blen;

	strcpy(maskbf,"");
	p = outbf;
	for (;TRUE;) {
		p = strstr(p,"(?!"); if (p==NULL) break;
		depth = 0;
		for (s=p;TRUE;s--) {
			if (s==outbf) break;
			if (depth==0) {
				if (s[-1]=='(') break;
				if (s[-1]=='|') break;
			}
			if (s[-1]==')') depth++;
			if (s[-1]=='(') depth--;
		}
		start = s;
		for (s=p;*s!=0;s++) if (*s==')') break;
		if (*s!=0) s++;
		depth = 0;
		for (;*s!=0;s++) {
			if (depth==0) {
				if (*s==')') break;
				if (*s=='|') break;
			}
			if (*s==')') depth--;
			if (*s=='(') depth++;
		}
		blen =(int) ( s - start);
		if (blen>BFSZ-1) blen = BFSZ-1;
		ncpy(amask,start,(int) (s-start));
		p = s;
		for (s=amask,out=amask;*s!=0;) {
			*out++ = *s++;
			if (strncmp(s-1,"(?!",3)==0) s+=2;
		}
		*out++ = 0;
		if (lib_strlen(maskbf)>0) strcat(maskbf,"|");
		strcat(maskbf,amask);
	}
	return TRUE;
}

int lookahead_out(char *outbf)
{
	char *s;
	char *out;
	int gotone = FALSE;
	out = outbf;
	for (s = outbf;*s!=0;s++) {
		if (strncmp(s,"(?!",3)==0) {
		 if (strchr(s,')') !=NULL) {
		  gotone = TRUE;
		  s+=3;
		  for (;*s!=0;s++) if (*s==')') break;
		 } else goto xxx;
		} else {
xxx:		*out++ = *s;
		}
		if (*s==0) break;
	}
	*out++ = 0;
	return gotone;
}

int comp_rexp_case(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	Mfilter *m = fp->m;
	pcre *pcreg;
	pcre_extra *pcreg_hints;
	char *pcre_error;
	int pcre_errloc;
	exp->p2 = mystrdup(arg2);
	exp->header = df_header_findadd(exp->m,arg1);

	pcreg=pcre_compile(arg2, PCRE_EXTENDED | PCRE_MULTILINE,  (const char **)&pcre_error, &pcre_errloc, NULL);

	if(pcre_error!=NULL)
	{
		m->fmsg("Regular expression failed at %d in (%.40s) (%.40s)\n",pcre_errloc, arg2, pcre_error);
		pcre_freex(pcreg);
		return FALSE;
	}
	exp->pcreg = pcreg;

	pcreg_hints=pcre_study(pcreg, 0, (const char **)&pcre_error);
	if(pcre_error==NULL)
		exp->hints=pcreg_hints;
	else
		myfree(pcreg_hints);

	return TRUE;
}
int comp_size(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup("noargs");
	exp->header = 0;
	return TRUE;
}
int comp_time_hour(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[])
{
	exp->p2 = mystrdup("noargs");
	exp->header = 0;
	return TRUE;
}
/* Add matchany(header,value) and matchall() */

/*-----------------------------------------------------------------*/
/* Run expressions */
char *str_show(char *s, char *bf)
{
	ncpy(bf,s,99);
	for (s=bf;*s!=0;s++) {
		if (*s=='\r') *s = '.';
		if (*s=='\n') *s = '.';
	}
	return bf;
}
char *str_show2(char *s, char *bf)
{
	ncpy(bf,s,99);
	for (s=bf;*s!=0;s++) {
		if (*s=='\r') *s = '.';
		if (*s=='\n') *s = '.';
	}
	return bf;
}

char rexp_res[BFSZ];

void rexp_result(char *sstart,char *send) /* copy_dst fptr in regex.c*/
{
	*rexp_res='\0';
	if(sstart && send && (send>sstart)) {
		if(send-sstart>50) send=sstart+50;
		safe_memmove(rexp_res,sstart,(int) (send-sstart),rexp_res,52);
		rexp_res[send-sstart]='\0';
	}
}

/*
void reg_result_dst(void (*dst)(Mfilter *m,char *sstart,char *send))
{
	m->copy_dst = dst;
}
*/
int run_rexp(Exp *exp)
{
	int r=FALSE;
	int offsets[99];
	Mfilter *m = exp->m;
	char *string=df_hdrs_get(exp->m,exp->header);
	char tmp3[BFSZ];
	int doingbody;

#ifdef TESTING
	if (m->trace) m->imsg("drun: exp: rexp(%.90s,%.90s)\n",str_show(string),exp->p2);
#else
	if (m->trace) m->imsg("drun: exp: rexp(%.40s,%.200s)\n",str_show(string,tmp3),exp->p2);
#endif

	doingbody=1;
	if(m->curprog->headers[exp->header]!=NULL)
	{
		if(stricmp(m->curprog->headers[exp->header], "body")!=0)
			doingbody=0;
	}



	r = pcre_exec(exp->pcreg, exp->hints, string, df_hdrs_getlen(m,exp->header), 0, 0, offsets, 99);


	if(r<0)
		r=FALSE;
	else
	{
		if(m->copy_dst)
			m->copy_dst(string+offsets[0],string+offsets[1]);
		r=TRUE;
	}

	return r;
}

#define REXPKEEPMAX 10000
Rexps *rexproot=0;
Rexps *find_rexp(char *id,char *value,int add)
{
	Rexps *curr,*prev=NULL;
	int n;

	curr=rexproot;
	for(n=0;curr;n++) {
		if(!strcmp(id,curr->id) && !strcmp(value,curr->value)) {
			curr->count++;
			return curr;
		}
		if(curr->next) prev=curr;
		curr=curr->next;
	}

	if(add) {
		if(n>=REXPKEEPMAX) {
			free(prev->next);
			prev->next=0;
		}
		curr=mymalloc(sizeof(Rexps));
		curr->id=strdup(id);
		curr->value=mystrdup(value);
		curr->next=rexproot;
		curr->count=1;
		return rexproot=curr;
	}
	return 0;
}

void reg_result_dst(Mfilter *m,void (*dst)(char *sstart,char *send))
{
	m->copy_dst = dst;
}
int run_rexpstore(Exp *exp)
{
	int res;
	Mfilter *m = exp->m;
	Rexps *r;
	reg_result_dst(m,rexp_result);
	res=run_rexp(exp);
	reg_result_dst(m,0);
	if(!res) return 0;
	r=find_rexp(exp->p3,rexp_res,1);
	return r?r->count:0;
}

int run_constant(Exp *exp)
{
	int r=FALSE;
	Mfilter *m = exp->m;
	if (m->trace) m->imsg("drun: exp: constant(%d)\n",exp->header);
	return exp->header;
}
int run_size(Exp *exp)
{
	Mfilter *m = exp->m;
	if (m->trace) m->imsg("drun: exp: size(%d/%d/%d)\n",m->true_len,m->real_len,m->item_size);
	if (m->true_len>0) return m->true_len;
	if (m->real_len>0) return m->real_len;
	return m->item_size;
}
int run_nurls(Exp *exp)
{
	Mfilter *m = exp->m;
	if (m->trace) m->imsg("drun: exp: nurls(%d)\n",exp->m->nurls);
	return exp->m->nurls;
}
int run_time_hour(Exp *exp)
{
	Mfilter *m = exp->m;
	if (m->trace) m->imsg("drun: time_hour (%d)\n",lib_hour());
	return lib_hour();
}
int run_time_min(Exp *exp)
{
	Mfilter *m = exp->m;
	if (m->trace) m->imsg("drun: time_min (%d)\n",lib_minutes());
	return lib_minutes();
}
int run_lines(Exp *exp)
{
	Mfilter *m = exp->m;
	return m->item_lines;
}
int run_body_lines(Exp *exp)
{
	return exp->m->body_lines;
}
int run_body_words(Exp *exp)
{
	return exp->m->body_words;
}
int run_isencodedtext(Exp *exp)
{
	Mfilter *m = exp->m;
	return m->item_isencodedtext;
}
int run_isencodedhtml(Exp *exp)
{
	Mfilter *m = exp->m;
	return m->item_isencodedhtml;
}
int run_isencodedurl(Exp *exp)
{
	Mfilter *m = exp->m;
	return m->item_isencodedurl;
}
int run_isuuencoded(Exp *exp)
{
	Mfilter *m = exp->m;
	return m->item_isuuencoded;
}
int run_isbinary(Exp *exp)
{
	Mfilter *m = exp->m;
	if (m->item_isbinary) if (m->nbin>=20) return TRUE;
	return FALSE;
}
int run_spawn(Exp *exp)
{
#ifdef SURGEMAIL
	int r;
	Mfilter *m = exp->m;
	char cmd[BFSZ];
	char reason[BFSZ];
	char tmp[BFSZ];
	FILE *f;
	IHANDLE phandle=0;
	/* Spawn it and then check result code */
	if (the_item==NULL) { m->imsg("run_spawn: run spawn failed\n"); return FALSE; }

	f = fopen("spawn.tmp","wb");
	if (f==NULL) {m->imsg("run_spawn: Unable to write (spawn.tmp) %s\n",strerror(errno)); return FALSE;}
	fwrite(the_item,1,lib_strlen(the_item),f);
	fclose(f);

	ncpy(tmp,exp->p2,BFSZ-1);
	lib_var_replace(tmp,cmd,BFSZ-1,"$FILE$","spawn.tmp");

	m->imsg("run_spawn: (%s)\n",cmd);

	if (!lib_detach_handle(cmd,reason,&phandle)) {
		m->imsg("run_spawn: Unable to open (%s) %s\n",cmd,reason);
		return 0;
	}
	/* Wait for 30 seconds at most */
	if (!lib_detach_wait(phandle,30,&r)) {
		m->imsg("run_spawn: no response for 30 seconds, killing robot\n");
		lib_detach_kill(phandle);
		return 0;
	}
	lib_detach_free(phandle);
	m->imsg("run_spawn: NEW Spawned program returned %d\n",r);
	return r;
#else
	return FALSE;
#endif
}
int run_isbinaryn(Exp *exp)
{
	Mfilter *m = exp->m;
	if (m->item_isbinary) if (m->nbin>=exp->header) return TRUE;
	return FALSE;
}
int run_ishtml(Exp *exp)
{
	Mfilter *m = exp->m;
	return m->item_ishtml;
}
int run_isbase64(Exp *exp)
{
	Mfilter *m = exp->m;
	return m->item_isbase64;
}

int run_isimage(Exp *exp)
{
	Mfilter *m = exp->m;
	if (m->trace) m->imsg("drun: exp: mitem_ismage=%d\n",m->mitem_isimage); 
	if (m->mitem_isimage) return TRUE;
	return m->item_isimage;
}
int run_clickimage(Exp *exp)
{
	Mfilter *m = exp->m;
	return m->item_isclickimage;
}
int run_isjpg(Exp *exp)
{
	Mfilter *m = exp->m;
	if (m->trace) m->imsg("drun: exp: isjpg=%d\n",m->mitem_isjpg); 
	return m->mitem_isjpg;
}
int run_ispdf(Exp *exp)
{
	Mfilter *m = exp->m;
	if (m->trace) m->imsg("drun: exp: ispdf=%d\n",m->mitem_ispdf); 
	return m->mitem_ispdf;
}
int run_image_size(Exp *exp)
{
	Mfilter *m = exp->m;
	if (m->trace) m->imsg("drun: exp: image_size=%d\n",m->mitem_image_size); 
	return m->mitem_image_size;
}
int run_pdf_size(Exp *exp)
{
	Mfilter *m = exp->m;
	if (m->trace) m->imsg("drun: exp: pdf_size=%d\n",m->mitem_pdf_size); 
	return m->mitem_pdf_size;
}
int run_nimage(Exp *exp)
{
	Mfilter *m = exp->m;
	if (m->trace) m->imsg("drun: exp: nimage()=%d\n",m->item_nimage); 
	// doesn't work as message is truncated
	return m->item_nimage;
}
int mfilter_set_loggedin(Mfilter *m, int x)
{
	m->item_isloggedin = x;
	return TRUE;
}

int mfilter_set_local(Mfilter *m, int x)
{
	m->item_islocal = x;
	return TRUE;
}
int mfilter_set_friend(Mfilter *m, int x)
{
	m->item_isfriend = x;
	return TRUE;
}
int mfilter_clear(Mfilter *m)
{
	m->item_isloggedin = FALSE;
	m->item_islocal = FALSE;
	m->item_isfriend = FALSE;
	return TRUE;
}
int run_islocal(Exp *exp)
{
	Mfilter *m = exp->m;
	return m->item_islocal;
}
int run_isfriend(Exp *exp)
{
	Mfilter *m = exp->m;
	return m->item_isfriend;
}
int run_isbounce(Exp *exp)
{
	Mfilter *m = exp->m;
	if (lib_strlen(m->cur_sender)==0) return TRUE;
	if (strcmp(m->cur_sender,"<>")==0) return TRUE;
	return FALSE;
}
int run_isloggedin(Exp *exp)
{
	Mfilter *m = exp->m;
	return m->item_isloggedin;
}
int run_ifflag(Exp *exp)
{
	Mfilter *m = exp->m;
	int r=FALSE;
	if (m->trace) m->imsg("drun: exp: ifflag(%d)\n",df_flag_get(exp->m,exp->header));
	r = df_flag_get(exp->m,exp->header);
	return r;
}
int run_crash(Exp *exp)
{
	Mfilter *m = exp->m;
	int r=FALSE;
	char *s = NULL;
	if (m->trace) m->imsg("drun: exp: crash\n",df_flag_get(exp->m,exp->header));
	*s = 0;
	return r;
}
int run_get(Exp *exp)
{
	Mfilter *m = exp->m;
	int r=FALSE;
	if (m->trace) m->imsg("drun: exp: get(%d)\n",df_flag_get(exp->m,exp->header));
	r = df_flag_get(exp->m,exp->header);
	return r;
}
int run_exists(Exp *exp)
{
	Mfilter *m = exp->m;
	char tmp3[BFSZ];
	/*int r=FALSE;

	if (m->trace) m->imsg("drun: exp: exists(%.40s)\n",str_show(df_hdrs_get(m,exp->header),tmp3));
	if (lib_strlen(df_hdrs_get(m,exp->header))>0) r = TRUE;

	return r;*/
	if (m->trace) m->imsg("drun: exp: exists(%.40s)\n",str_show(df_hdrs_get(m,exp->header),tmp3));
	return df_hdrs_exists(m,exp->header);
}
int wildrep(char *in, char *wild, char *tpl, char *result);
int run_replace(Exp *exp)
{
	char result[BFSZ];
	Mfilter *m = exp->m;
	char *s;
	s = df_hdrs_get(m,exp->header);
//printf("run_replace: headerwas %s\n",s);
	if (m->trace) m->imsg("drun: exp: replace(%.40s,%.40s,%.40s)\n",s,exp->p2,exp->p3);
	if (wildrep(s,exp->p2,exp->p3,result)) {
		if (m->trace) m->imsg("replace matched result (%.200s)\n",result);
//printf("run_replace: headernow %s\n",result);
		df_hdrs_set(m,exp->header,result);
	}
	return TRUE;
}
int run_ishidden(Exp *exp)
{
	Mfilter *m = exp->m;
	char *s;
	char bf[BFSZ];
	char *out;
	s = df_hdrs_get(m,exp->header);
	if (m->trace) m->imsg("drun: exp: ishidden(%.40s,%.40s,%.40s)\n",s,exp->p2,exp->p3);
	// remove exp->p3 from 's' then compare to exp->p2...
	if (strstrnc(s,exp->p2)!=NULL) {
		return FALSE; // we found it so it's 'not' hidden...
	}

	out = bf;
	for (;*s!=0;s++) {
		if (strchr(exp->p3,*s)==NULL) *out++ = *s;
		if (out-bf>BFSZ-10) break;
	}
	*out = 0;
	strlwr(bf);
	if (strstr(bf,exp->p2)!=NULL) {
		return TRUE;
	}
	return FALSE;
}
int run_ismixedcase(Exp *exp)
{
	char *s;
	char bf[BFSZ];
	char *p[100];
	int np;
	int i;
	int islow,isup;
	int mixed=0;
	Mfilter *m = exp->m;
	s = df_hdrs_get(m,exp->header);
	if (m->trace) m->imsg("drun: exp: ismixedcase(%.40s)\n",s);
	ncpy(bf,s,BFSZ-1);
	np = token_split(bf,p," \r\n\t",50);
	for (i=0; i<np; i++) {
		char *w;
		w = p[i];
		if (*w) w++; // skip first letter.
		islow = isup = FALSE;
		for (;*w!=0;w++) {
			if (sisalpha(*w)) {
				if (sisupper(*w)) isup = TRUE;
				if (sislower(*w)) islow = TRUE;
			}
		}
		if (isup && islow) {
			mixed++;
			if (lib_strlen(p[i])>30) mixed = 4;
		}
	}
	if (mixed>3) return TRUE;
	return FALSE;
}
char *mfilter_get_tohost(Mfilter *m)
{
	return m->mfilter_to;
}
char *mfilter_get_returnpath(Mfilter *m)
{
	return m->return_path;
}
int mfilter_get_unfriend(Mfilter *m)
{
	return m->mfilter_unfriend;
}
char *mfilter_get_whitelist(Mfilter *m)
{
	return m->mfilter_whitelist;
}
char *mfilter_get_store_folder(Mfilter *m)
{
	return m->store_folder;
}
int run_set_tohost(Exp *exp)
{
	ncpy(exp->m->mfilter_to,exp->p2,BFSZ-1);
	return 1;
}
int run_set_returnpath(Exp *exp)
{
	ncpy(exp->m->return_path,exp->p2,BFSZ-1);
//	printf("run_setreturnpath %s\n",m->return_path);
	return 1;
}
int run_unfriend(Exp *exp)
{
	exp->m->mfilter_unfriend = TRUE;
	return 1;
}
int run_set_whitelist(Exp *exp)
{
	ncpy(exp->m->mfilter_whitelist,exp->p2,BFSZ-1);
	return 1;
}
int run_attached(Exp *exp)
{
	Mfilter *m = exp->m;
	int r;
	if (m->trace) m->imsg("drun: attached (%.40s,%.40s)\n",exp->p2,m->attach_name);
	r = lib_split_and_match_wild(exp->p2,m->attach_name);
	if (r>0) r = TRUE; else r = FALSE;
	return r;
}
int run_store_folder(Exp *exp)
{
	ncpy(exp->m->store_folder,exp->p2,BFSZ-1);
	return 1;
}
int run_date_offset(Exp *exp)
{
	int r=FALSE;
	char *s;
	int diff;
	Mfilter *m = exp->m;
	time_t x;
	char tmp3[BFSZ];
	time_t now = time(NULL);
	if (m->trace) m->imsg("drun: exp: date(%.40s,%.40s)\n",str_show(df_hdrs_get(m,exp->header),tmp3),exp->p2);
	s = df_hdrs_get(m,exp->header);
	x = web_date_gmt(s);
	if (x==0) x = now;
	diff = (int) (now - x)/(60*60);
	if (m->trace) m->imsg("drun: Diff is %d hours\n",diff);
	return diff;
}
int run_istwo(Exp *exp)
{
	int r=FALSE;
	char *s;
	int found=0;
	char bf[BFSZ];
	char *p[100];
	char tmp3[BFSZ];
	int np;
	int i;
	Mfilter *m = exp->m;
	if (m->trace) m->imsg("drun: exp: istwo(%.40s,%.40s)\n",str_show(df_hdrs_get(m,exp->header),tmp3),exp->p2);
	s = df_hdrs_get(m,exp->header);


	// Looking for 'two' of the things in exp->p2.... 
	ncpy(bf,exp->p2,BFSZ-1);
	np = token_split(bf,p,",",90);
	for (i=0; i<np;i++) {
		if (strstrnc(s,p[i])!=NULL) found++;
	}
//printf("Found %d\n",found);
	if (found>1) return TRUE;
	return FALSE;
}
int run_isin(Exp *exp)
{
	Mfilter *m = exp->m;
	int r=FALSE;
	char *s;
	char tmp3[BFSZ];
	char tmp2[BFSZ];
	//,*s2;
	if (m->trace) m->imsg("drun: exp: isin(%.40s,%.40s)\n",str_show(df_hdrs_get(m,exp->header),tmp3),str_show2(exp->p2,tmp2));
//	s = mystrdup(df_hdrs_get(m,exp->header));
	s = df_hdrs_get(m,exp->header);
//	s2 = mystrdup(df_hdrs_get(m,exp->header));
//	strlwr(s);
//	strlwr(exp->p2);
	if (strstrnc(s,exp->p2)!=NULL) r = TRUE;
//	if (strstrnc(s,exp->p2)!=NULL) r = TRUE;
//	myfree(s);
//	myfree(s2);
	return r;
}
int run_rexp_fast(Exp *exp)
{
	exp->m->is_rexp_fast = TRUE;
	return FALSE;
}
int run_hidden_word(Exp *exp)
{
	exp->m->ishidden = TRUE;
	return FALSE;
}
int run_add_aspam(Exp *exp)
{
	Mfilter *m = exp->m;
	df_spamdetect(m,(float) (m->aspam_score * atof(exp->p2)),exp->p3);
	if (m->aspam_score>=3) return TRUE;
	return FALSE;
}
int run_add_header(Exp *exp)
{
	df_hdrs_add(exp->m,exp->p2);
	return FALSE;
}
int run_add_poly(Exp *exp)
{
	Mfilter *m = exp->m;
	df_spamdetect(m,(float) (m->poly_score * atof(exp->p2)),exp->p3);
	if (m->poly_score>=2) return TRUE;
	return FALSE;
}
int run_isinc(Exp *exp)
{
	char tmp3[BFSZ];
	int r=FALSE;
	Mfilter *m = exp->m;
	char *s,*s2;
	if (m->trace) m->imsg("drun: exp: isin(%.40s,%.40s)\n",str_show(df_hdrs_get(m,exp->header),tmp3),exp->p2);
	s = mystrdup(df_hdrs_get(m,exp->header));
	strlwr(s);
	strlwr(exp->p2);
	if((s2=mystrdup(exp->p2))==NULL) return TRUE;
	if (strstr(cleanstr(s,s),cleanstr(exp->p2,s2))!=NULL) r = TRUE;
	myfree(s);
	myfree(s2);
	return r;
}
int run_forward_cc(Exp *exp)
{
	/* Forward this message to another user, Add entry to linked list of forwards.*/
	char tmp2[BFSZ];
	Mfilter *m = exp->m;
	Fwddata *f;
	f = calloc(1,sizeof(Fwddata));
	f->path = mystrdup(df_fix_email(exp->p2,tmp2));
	m->imsg("mfilter: Forwarding to (%s)\n",f->path);
	f->next = m->forward_cc;
	m->forward_cc = f;
	return TRUE;
}
int run_matchall(Exp *exp)
{
	char tmp3[BFSZ];
	Mfilter *m = exp->m;
	int r;
	if (m->trace) m->imsg("drun: exp: matchall(%.40s,%.40s)\n",str_show(df_hdrs_get(m,exp->header),tmp3),exp->p2);
	r = df_group_match_list(m,exp->p2,df_hdrs_get(m,exp->header));
	if (r==1) r = TRUE; else r = FALSE;
	return r;
}
int run_report(Exp *exp)
{
	Mfilter *m = exp->m;
	if (m->trace) m->imsg("drun: exp: report(%.40s,%.40s)\n",exp->p2,exp->p3);
	df_send_report(m,exp->p2,exp->p3,100);
	return TRUE;
}
int run_report_full(Exp *exp)
{
	Mfilter *m = exp->m;
	if (m->trace) m->imsg("drun: exp: report(%.40s,%.40s)\n",exp->p2,exp->p3);
	df_send_report(m,exp->p2,exp->p3,1000000);
	return TRUE;
}
int run_report_brief(Exp *exp)
{
	Mfilter *m = exp->m;
	if (m->trace) m->imsg("drun: exp: report(%.40s,%.40s)\n",exp->p2,exp->p3);
	df_send_report(m,exp->p2,exp->p3,0);
	return TRUE;
}
int mfilter_set_mdir(Mfilter *m, char *path)
{
	int ns = 0;
	char *s;
	ncpy(m->user_mdir,path,BFSZ-1);
	// Now strip off the '/tmp/xxxx' to get the mdir part.
	for (s = m->user_mdir+lib_strlen(m->user_mdir)-1;s>m->user_mdir;s--) {
		if (*s=='/') ns++;
		if (*s=='\\') ns++;
		if (ns==2) {
			*s = 0; break;
		}	
	}
	return TRUE;
}
int run_folder_exists(Exp *exp)
{
	Mfilter *m = exp->m;
	char bf[BFSZ];
	int r;
	lprintf(bf,BFSZ-1,"%s/%s.uidl",m->user_mdir,exp->p2);
	r = lib_exists(bf);
	if (!r) {
		lprintf(bf,BFSZ-1,"%s/%s",m->user_mdir,exp->p2);
		r = lib_exists(bf);
	}
	m->imsg("mfilter: check if (%s) exists %d\n",bf,r);
	return r;
}
int run_is_dayofweek(Exp *exp)
{
	Mfilter *m = exp->m;
	if (m->trace) m->imsg("drun: exp: is_dayofweek(\"%.40s\", \"%.40s\")\n",exp->p2,lib_dayofweek());
	if (strstrnc(exp->p2,lib_dayofweek())!=NULL) return TRUE;
	return FALSE;
}
int run_spamdetect(Exp *exp)
{
	Mfilter *m = exp->m;
	if (m->trace) m->imsg("drun: exp: spamdetect(%.40s,%.40s)\n",exp->p2,exp->p3);
	df_spamdetect(m,(float) atof(exp->p2),exp->p3);

	return TRUE;
}
int run_feature_header(Exp *exp)
{
	return TRUE;
}
int run_feature_headerm(Exp *exp)
{
	return TRUE;
}
int run_tomanual(Exp *exp)
{
	return TRUE;
}
int run_feature_add(Exp *exp)
{
	df_spamdetect(exp->m,(float) atof(exp->p2),exp->p3);
	return TRUE;
}
int run_feature_manual(Exp *exp)
{
	Mfilter *m = exp->m;
	if (m->trace) m->imsg("drun: feature_manual %s %s\n",exp->p2,exp->p3);
	df_spamdetect(m,(float) atof(exp->p2),exp->p3);
	return TRUE;
}
int run_bounce_fast(Exp *exp)
{
	Mfilter *m = exp->m;
	if (m->trace) m->imsg("drun: exp: bounce_fast(%.40s,%.40s)\n",exp->p2,exp->p3);
	df_bounce_fast_add(m,exp->score,exp->p2,exp->p3);
	return TRUE;
}
int run_matchone(Exp *exp)
{
	Mfilter *m = exp->m;
	char tmp3[BFSZ];
	int r;
	if (m->trace) m->imsg("drun: exp: matchany(%.40s,%.40s)\n",str_show(df_hdrs_get(m,exp->header),tmp3),exp->p2);
	r = df_group_match_list(m,exp->p2,df_hdrs_get(m,exp->header));
	if (r>0) r = TRUE; else r = FALSE;
	return r;
}
int run_match(Exp *exp)
{
	Mfilter *m = exp->m;
	char tmp3[BFSZ];
	int r=FALSE;
	if (m->trace) m->imsg("drun: exp: match(%.40s,%.40s)\n",str_show(df_hdrs_get(m,exp->header),tmp3),exp->p2);
	if (df_group_match(m,exp->p2,df_hdrs_get(m,exp->header))) r = TRUE;
	return r;
}
int run_head_len(Exp *exp)
{
	Mfilter *m = exp->m;
	char tmp3[BFSZ];
	if (m->trace) m->imsg("drun: exp: head_len(%.40s)\n",str_show(df_hdrs_get(m,exp->header),tmp3));
	return lib_strlen(df_hdrs_get(m,exp->header));
}
int run_allmod(Exp *exp)
{
#ifndef TESTING
	int r;
	Mfilter *m = exp->m;
	char tmp3[BFSZ];
	if (m->trace) m->imsg("drun: exp: allmod(%.40s)\n",str_show(df_hdrs_get(m,exp->header),tmp3));
    r = df_match_mod(df_hdrs_get(m,exp->header));
	if (r==1) r=TRUE; else r=FALSE;
	return r;
#else
	return FALSE;
#endif
}
int mf_domain_only(char *from)
{
	char tmp[BFSZ];
	char *s;
	char *upto;
	s = strchr(from,'@');
	if (s==NULL) return FALSE;
	ncpy(tmp,s+1,BFSZ-1);
	s = mystrtok(tmp," ,><\"",&upto);
	if (s!=NULL) ncpy(from,s,BFSZ-1);
	return TRUE;
}
int m_spf_trim(char *ip, char *out, int digits)
{
	char *p[130];
	char ip2[BFSZ];
	int np;
	int i=0;
	ncpy(ip2,ip,BFSZ-1);
	np = token_split(ip2,p,".",90);
	strcpy(out,"");
	if (np>digits) i = np - digits;
	for (; i<np; i++) {
		if (*out!=0) ncat(out,".",BFSZ-1);
		ncat(out,p[i],BFSZ-1);
	}
	return TRUE;
}
int run_fromtwo(Exp *exp)
{
	Mfilter *m = exp->m;
	char p1[BFSZ];
	ncpy(p1,df_hdrs_get(m,exp->header),BFSZ-1);
	if (lib_from_two(p1)) return TRUE;
	return FALSE;
}
int run_forged(Exp *exp)
{
	Mfilter *m = exp->m;
	char p1[BFSZ];
	char *upto;
	char *s;
	char u1[BFSZ];
	char u2[BFSZ];
	char dom1[BFSZ];
	char dom2[BFSZ];
	char user[BFSZ];
	char orig[BFSZ];
	ncpy(p1,df_hdrs_get(m,exp->header),BFSZ-1);
	ncpy(orig,p1,BFSZ-1);
	// Do we have two email addresses, one in <> brackets, that don't match...
	s = mystrtok(p1," \"",&upto); if (s==NULL) return FALSE;
	ncpy(u1,s,BFSZ-1);
	s = mystrtok(NULL," \"",&upto); if (s==NULL) return FALSE;
	ncpy(u2,s,BFSZ-1);
	if (strchr(u1,'@')==NULL) return FALSE;
	if (strchr(u2,'@')==NULL) return FALSE;
	lib_split_email(u1,user,dom1);
	lib_split_email(u2,user,dom2);
//printf("forged testa (%s) (%s) orig: %s\n",dom1,dom2,orig);
	if (!lib_compare_base(dom1,dom2)) {
//printf("forged testb (%s) (%s) orig: %s\n",dom1,dom2,orig);
		return TRUE;
	}
	return FALSE;
}
int run_wrongdomain(Exp *exp)
{
	Mfilter *m = exp->m;
	char tmp[BFSZ];
	char p1[BFSZ];
	char p2[BFSZ];
	int want;
	ncpy(p1,df_hdrs_get(m,exp->header),BFSZ-1);
	ncpy(p2,df_hdrs_get(m,exp->header2),BFSZ-1);
	mf_domain_only(p1);
	mf_domain_only(p2);
//printf("wrong domain (%s) (%s)\n",p1,p2);
	// abc.xyz.com  
	/// xxx.xyz.com  
	// Truncate both to same number of dots 1 or 2

	want = 3;
	if (strstrnc(p1,".com")!=NULL) want=2;
	if (strstrnc(p1,".bz")!=NULL) want=2;
	if (strstrnc(p1,".biz")!=NULL) want=2;
	if (strstrnc(p1,".net")!=NULL) want=2;
	if (strstrnc(p1,".info")!=NULL) want=2;
	if (strstrnc(p1,".ph")!=NULL) want=2;

	m_spf_trim(p1,tmp,want); ncpy(p1,tmp,BFSZ-1);
	m_spf_trim(p2,tmp,want); ncpy(p2,tmp,BFSZ-1);
//printf("Wrong domain (%s) (%s) \n",p1,p2);

//printf("compaire wrong domain (%s) (%s)\n",p1,p2);
	if (lib_strlen(p1)<3) return FALSE;
	if (lib_strlen(p2)<3) return FALSE;
	if (strcmpnc(p1,p2)==0) return FALSE;
	if (strstr(p2,p1)!=NULL) return FALSE;
	m->imsg("mfilter: wrong domain true (%s) (%s)\n",p1,p2);
//printf("mfilter: wrong domain true (%s) (%s)\n",p1,p2);
	return TRUE;
}
int run_wronguser(Exp *exp)
{
	Mfilter *m = exp->m;
	char p1[BFSZ];
	char p2[BFSZ];
	char p1raw[BFSZ];
	char p2raw[BFSZ];
	ncpy(p1,df_hdrs_get(m,exp->header),BFSZ-1);
	ncpy(p2,df_hdrs_get(m,exp->header2),BFSZ-1);
	rfc_raw_email(p1,p1raw,BFSZ-1);
	rfc_raw_email(p2,p2raw,BFSZ-1);
	if (strcmpnc(p1raw,p2raw)!=0) return TRUE;
	return FALSE;
}
int run_issimilar(Exp *exp)
{	/* Quick check of first character to see it this is possible an alphabetic spam list */
	Mfilter *m = exp->m;
	int same_p=0,same_d=0,req;
	char p,d=0;
	char *s,*s2;
	char *lcl;
	req=atoi(exp->p2);
	if(req<=0) req=5;
	s = mystrdup(df_hdrs_get(m,exp->header));

	lcl=NULL;
	s2=mystrtok(s,"\r\n,;<> \t",&lcl);
	if(s2) p=*s2;
	while(s2) {
		if(p==*s2) same_p++;
		else {
			p=*s2;
			same_p=1;
		}

		s2=strchr(s2,'@');
		if(s2) {
			s2++;
			if(d==*s2) same_d++;
			else {
				d=*s2;
				same_d=1;
			}
		}

		if(same_p==req || same_d==req) break;
		s2=mystrtok(NULL,"\r\n,;<> \t",&lcl);
	}

	myfree(s);
	return same_p==req || same_d==req;
}

int run_evalflags(Exp *exp)
{
	return df_expr_eval(exp->m,exp->header);
}

#ifndef TESTING
int df_ismod(char *group)
{
	return TRUE;
}
int df_match_mod(char *groups)
{
    char *p = groups;
	int any = FALSE;
	int all = TRUE;
	char bf[BFSZ];
	char *end;
	int len;

	for (;*p!=0;) {
		for (end=p;*end!=0;end++) {
			if (*end==',') break;
			if (*end=='!') break;
			if (*end=='|') break;
		}
		len = (int) (end-p);
		if (len>BFSZ-4) len = BFSZ-10;
		ncpy(bf,p,len);
		if (df_ismod(bf)) any = TRUE;
		else all = FALSE;
		if (*end==0) break;
		p=end+1;
	}
	if (all) return 1; /* all groups moderated */
	if (any) return 2; /* at least one moderated */
	return 0; /* no moderated groups */
}
#endif

int run_equals(Exp *exp)
{
	char tmp3[BFSZ];
	Mfilter *m = exp->m;
	int r=FALSE;
	if (m->trace) m->imsg("drun: exp: equals(%.40s,%.40s)\n",str_show(df_hdrs_get(m,exp->header),tmp3),exp->p2);
	if (strcmp(df_hdrs_get(m,exp->header),exp->p2)==0) r = TRUE;
	return r;
}
/*
if rexp ( subject , "This is\" \r\n \\  " ) reject Naughty Subject
*/
#define sep "\r\n \t"
int df_token(char *line, char *p[], int maxp, int qq[], char *bf)
{
    int n=0;
	int z;
	char *out = bf;
	char *s;
	maxp -= 3;
        for (s=line;*s!=0;) {
                z = (int) strspn(s,sep);  /* Skip spaces */
		s += z;
                if (*s==0) break;
		qq[n] = FALSE;
		p[n++] = out;
		if (n>=maxp) break;
		*out = 0;
		if (*s=='\n') break;
		if (*s=='"') {
		 qq[n-1] = TRUE;
		 for (s++;*s!=0;) {
			if (*s=='"') {s++; *out = 0; goto next_tok;}
			if (*s=='\\') {
				if (s[1]=='"') { *out++ = '"';	s+= 2; continue; }
				if (s[1]=='r') { *out++ = '\r';	s+= 2; continue; }
				if (s[1]=='n') { *out++ = '\n';	s+= 2; continue; }
				if (s[1]=='\\') { *out++ = '\\';	s+= 2; continue; }
			}
			*out++ = *s++;
		 }
		 return -1;
		} else {
		 for (;*s!=0;) {
			if (strchr(" \t\r\n",*s)!=NULL) { *out = 0;break; }
			if (strchr("()!,<>=+",s[1])!=NULL) { *out++ = *s++; break; }
			if (strchr("()!,<>=+",*s)!=NULL) {
				*out++ = *s++;
				*out = 0;
				break;
			}
			*out++ = *s++;
		 }
		}
next_tok:
		*out++ = 0;
        }
	*out++ = 0;
        p[n] = NULL;
        return n;
}
/*enum {A_ACCEPT=1,A_REJECT};*/
int df_find_action(Mfilter *m, Linex **peline,char *p[],int np,int *upto)
{
	Linex *eline = *peline;
	int dofree= FALSE;
	if (eline==NULL) {
		*peline = mymalloc(sizeof(Linex));
		eline = *peline;
		dofree = TRUE;
		memset(eline,0,sizeof(Linex));
	}
	if (strcmp(gp(m,*upto),"accept")==0) {
		eline->action = A_ACCEPT;
		eline->reason = mystrdup(gp(m,*upto + 1));
		*upto += 2;
		return TRUE;
	}
	if ((strcmp(gp(m,*upto),"forward")==0) || (strcmp(gp(m,*upto),"redirect")==0)) {
		eline->action = A_FORWARD;
		eline->reason = mystrdup(gp(m,*upto + 1));
		*upto += 2;
		return TRUE;
	}
	if ((strcmp(gp(m,*upto),"redirect_cc")==0) || (strcmp(gp(m,*upto),"cc")==0)) {
		eline->action = A_REDIRECT_CC;
		eline->reason = mystrdup(gp(m,*upto + 1));
		*upto += 2;
		return TRUE;
	}
	if (strcmp(gp(m,*upto),"then")==0) {
		if (*upto!=(np-1)) {
			m->fmsg("'then' must be the last statement on the line line=%d\n",m->lineno);
		} else {
			eline->action = A_THEN;
			*upto += 1;
			return TRUE;
		}
	}
	if (strcmp(gp(m,*upto),"reject")==0) {
		eline->action = A_REJECT;
		eline->reason = mystrdup(gp(m,*upto + 1));
		*upto += 2;
		return TRUE;
	}
	if (strcmp(gp(m,*upto),"do_bounce_fast")==0) {
		eline->action = A_DO_BOUNCE_FAST;
		eline->reason = mystrdup("block_fast");
		*upto += 1;
		return TRUE;
	}
	if (strcmp(gp(m,*upto),"drop")==0) {
		eline->action = A_DROP;
		eline->reason = mystrdup(gp(m,*upto + 1));
		*upto += 2;
		return TRUE;
	}
	if (strcmp(gp(m,*upto),"bounce")==0) {
		eline->action = A_REJECT;
		eline->reason = mystrdup(gp(m,*upto + 1));
		*upto += 2;
		return TRUE;
	}
	if (strcmp(gp(m,*upto),"recipients")==0) {
		eline->action = A_RECIPIENTS;
		*upto += 1;
		return TRUE;
	}
	if (strcmp(gp(m,*upto),"print")==0) {
		eline->action = A_PRINT;
		eline->reason = mystrdup(gp(m,*upto + 1));
		*upto += 2;
		return TRUE;
	}
	if (strcmp(gp(m,*upto),"setflag")==0) {
		eline->action = A_SETFLAG;
		eline->p1 = df_flag_findadd(m,gp(m,*upto+2));
		*upto += 4;
		return TRUE;
	}
	if (strcmp(gp(m,*upto),"set")==0) {
		eline->action = A_SET;
		eline->p1 = df_flag_findadd(m,gp(m,*upto+2));
		eline->p2 = atoi(gp(m,*upto+4));
		*upto += 6;
		return TRUE;
	}
	if (strcmp(gp(m,*upto),"add")==0) {
		eline->action = A_ADD;
		eline->p1 = df_flag_findadd(m,gp(m,*upto+2));
		eline->p2 = atoi(gp(m,*upto+4));
		*upto += 6;
		return TRUE;
	}
	if (strcmp(gp(m,*upto),"sub")==0) {
		eline->action = A_SUB;
		eline->p1 = df_flag_findadd(m,gp(m,*upto+2));
		eline->p2 = atoi(gp(m,*upto+4));
		*upto += 6;
		return TRUE;
	}
	if (strcmp(gp(m,*upto),"clearflag")==0) {
		eline->action = A_CLEARFLAG;
		eline->p1 = df_flag_findadd(m,gp(m,*upto+2));
		*upto += 4;
		return TRUE;
	}
	if (strcmp(gp(m,*upto),"score_one")==0) {
		eline->action = A_SCORE_ONE;
		eline->p1 = df_flag_findadd(m,gp(m,*upto+2));
		*upto += 4;
		return TRUE;
	}
	if (strcmp(gp(m,*upto),"score_all")==0) {
		eline->action = A_SCORE_ALL;
		eline->p1 = df_flag_findadd(m,gp(m,*upto+2));
		*upto += 4;
		return TRUE;
	}
	if (dofree) myfree(eline);
	return FALSE;
}
int df_header_findadd(Mfilter *m,char *header)
{
	int i;
	char bf[BFSZ];
	char *s;
	strncpy(bf,header,100); bf[100]=0; strlwr(bf);
	s = strchr(bf,':'); if (s!=NULL) *s = 0;
	for (i=0; i<m->curprog->nheaders; i++) {
		if (strcmp(m->curprog->headers[i],bf)==0) return i;
	}
	if (m->curprog->nheaders+10>MAX_HEADERS) {
		m->fmsg("Too many different headers in rules file");
		printf("Too many different headers in rules file");
		lib_abort();
	}
	m->curprog->headers[m->curprog->nheaders++] = mystrdup(bf);
	return m->curprog->nheaders-1;
}
int df_flag_findadd(Mfilter *m, char *header)
{
	int i;
	char bf[BFSZ];
	char *s;
	strncpy(bf,header,100); bf[100]=0; strlwr(bf);
	s = strchr(bf,':'); if (s!=NULL) *s = 0;
	for (i=0; i<m->curprog->nflags; i++) {
		if (strcmp(m->curprog->flags[i],bf)==0) return i;
	}
	if (m->curprog->nflags+10>MAX_FLAGS) {
		m->fmsg("Too many different flags in rules file");
		lib_abort();
	}
	m->curprog->flags[m->curprog->nflags++] = mystrdup(bf);
//m->imsg("mfilter: flag (%s) = %d\n",bf,curprog->nflags-1);
	return m->curprog->nflags-1;
}
int df_flag_get(Mfilter *m, int x)
{
	if ((x<0) || (x>=MAX_FLAGS)) return FALSE;
	return m->flags[x];
}
#ifdef TESTING
/*
char *ncpy(char *dst, char *src, int len)
{
        char *xdst = dst;
        int xlen = len;
        for (;(*src !=0) && (len>0); ) {*dst++  = *src++; len--;}
        *dst++ = 0;
        xdst[xlen] = 0;
        return xdst;
}
*/
#endif
int df_getline(char *item, int *upto, char *bf, int bfsz)
{
	char *s;
	char *s2;
	char *s3;
	int len;

	s = item;
	if (s==NULL) return FALSE;
	s += *upto;
	for (s2=s;*s2!=0;s2++) if (*s2=='\n') break;
	if (*s2==0) return FALSE;
	len = (int) (s2-s);
	*upto = (int) (s2 - item + 1);
	if (len<0) len = 0; if (len>=bfsz) len = bfsz-1;
	ncpy_ok(bf,s,len);
	s3 = strchr(bf,'\r');
	if (s3!=NULL) *s3 = 0;
	return TRUE;
}
static const unsigned char charmap[] = {
        0000, 0001, 0002, 0003, 0004, 0005, 0006, 0007,
        0010, 0011, 0012, 0013, 0014, 0015, 0016, 0017,
        0020, 0021, 0022, 0023, 0024, 0025, 0026, 0027,
        0030, 0031, 0032, 0033, 0034, 0035, 0036, 0037,
        0040, 0041, 0042, 0043, 0044, 0045, 0046, 0047,
        0050, 0051, 0052, 0053, 0054, 0055, 0056, 0057,
        0060, 0061, 0062, 0063, 0064, 0065, 0066, 0067,
        0070, 0071, 0072, 0073, 0074, 0075, 0076, 0077,
        0100, 0141, 0142, 0143, 0144, 0145, 0146, 0147,
        0150, 0151, 0152, 0153, 0154, 0155, 0156, 0157,
        0160, 0161, 0162, 0163, 0164, 0165, 0166, 0167,
        0170, 0171, 0172, 0133, 0134, 0135, 0136, 0137,
        0140, 0141, 0142, 0143, 0144, 0145, 0146, 0147,
        0150, 0151, 0152, 0153, 0154, 0155, 0156, 0157,
        0160, 0161, 0162, 0163, 0164, 0165, 0166, 0167,
        0170, 0171, 0172, 0173, 0174, 0175, 0176, 0177,
        0200, 0201, 0202, 0203, 0204, 0205, 0206, 0207,
        0210, 0211, 0212, 0213, 0214, 0215, 0216, 0217,
        0220, 0221, 0222, 0223, 0224, 0225, 0226, 0227,
        0230, 0231, 0232, 0233, 0234, 0235, 0236, 0237,
        0240, 0241, 0242, 0243, 0244, 0245, 0246, 0247,
        0250, 0251, 0252, 0253, 0254, 0255, 0256, 0257,
        0260, 0261, 0262, 0263, 0264, 0265, 0266, 0267,
        0270, 0271, 0272, 0273, 0274, 0275, 0276, 0277,
        0300, 0301, 0302, 0303, 0304, 0305, 0306, 0307,
        0310, 0311, 0312, 0313, 0314, 0315, 0316, 0317,
        0320, 0321, 0322, 0323, 0324, 0325, 0326, 0327,
        0330, 0331, 0332, 0333, 0334, 0335, 0336, 0337,
        0340, 0341, 0342, 0343, 0344, 0345, 0346, 0347,
        0350, 0351, 0352, 0353, 0354, 0355, 0356, 0357,
        0360, 0361, 0362, 0363, 0364, 0365, 0366, 0367,
        0370, 0371, 0372, 0373, 0374, 0375, 0376, 0377
};

int mystrcmpnc(const char *s1, const char *s2) {
        const unsigned char *cm = charmap,
             *us1 = (const unsigned char *)s1,
             *us2 = (const unsigned char *)s2;

        while (cm[*us1] == cm[*us2++])
                if (*us1++ == '\0')
                        return (0);
        return (cm[*us1] - cm[*--us2]);
}



int mystrncmpnc(const char *s1, const char *s2, size_t len)
{
        int i;
/*      char tmp1[200],tmp2[200];
        ncpy(tmp1,s1,len);
        ncpy(tmp2,s2,len);
        if (m->trace) m->imsg("Compare nocase {%s} {%s} %d \n",tmp1,tmp2,len);
*/
        for (i=0;i<len;i++,s1++,s2++) {
                if (tolower(*s1)<tolower(*s2)) return -1;
                if (tolower(*s1)>tolower(*s2)) return 1;
                if (*s1 == 0) return -1;
                if (*s2 == 0) return 1;
        }
        return 0;
}

int df_group_match_list(Mfilter *m, char *wild, char *grps)
{
	/* return 1 if all groups match, 2 if any match */
	char *p = grps;
	int any = FALSE;
	int all = TRUE;
	char bf[BFSZ];
	char tmp2[BFSZ];
	char *end;
	int len;

	for (;*p!=0;) {
		for (end=p;*end!=0;end++) {
			if (*end==',') break;
			if (*end=='!') break;
			if (*end=='|') break;
		}
		len = (int) (end-p);
		if (len>BFSZ-4) len = BFSZ-10;
		ncpy(bf,p,len);
		lib_trim_spaces(bf,tmp2);
		ncpy(bf,tmp2,BFSZ-1);
		if (df_group_match(m,wild,bf)) any = TRUE;
		else all = FALSE;
		if (*end==0) break;
		p=end+1;
	}
	if (!any) all = FALSE;
	if (all) return 1;
	if (any) return 2;
	return 0;
}
int df_group_match(Mfilter *mf,char *wild, char *grp2)
{
        char bf[1000];
        char grp[BFSZ];
        int m=TRUE,lastm;
        char *s,*p,*w;
        int done,len;
        int all;

        mf->lastall = FALSE;
        ncpy(grp,grp2,BFSZ-10); strlwr(grp);
        lastm = FALSE;
        p = wild;
        done = FALSE;
        for (;!done;) {
                s = strchr(p,',');
                if (s==NULL) {s = wild + lib_strlen(wild); done = TRUE;}
                if (*p==0) break;
                len = (int) (s - p);
                if (len>100) len = 100;
                strncpy(bf,p,len); bf[len] = 0;
                strlwr(bf); w = bf;

                m = TRUE;
                all = FALSE;
                if (*w=='!') {
                        w++;
                        m = FALSE;
                }
                if (*w=='@') {
                        w++;
                        m = FALSE;
                        all = TRUE;
                }
                if (match_wild(w,grp)) { lastm = m; mf->lastall = all;}

                p = s + 1;
        }
        return lastm;
}
#ifndef EOL
#define EOL "\r\n"
#endif
int count_lines(char  *item, int *xbody_lines, int *xbody_words)
{
        char *s = item;
        int n=0;
		int isbody=false;
		int bn=0;
		int nwords = 0;
	if (s==NULL) return 0;

        for (;*s!=0;s++) {
			if (*s=='\n') {
				n++;
				if (isbody) bn++;
				if (s[1]=='\n') isbody = true;
				if (s[2]=='\n') isbody = true;
			}
			if (isbody) if (*s==' ') nwords++;
        }
		*xbody_lines = bn;
		*xbody_words = nwords;
//printf("count_lines: lines=%d body_lines=%d body_words=%d\n",n,bn,nwords);
        return n;
}
int df_send_report(Mfilter *m, char *to, char *subj, int lines)
{
	FILE *f,*fin;
	int i;
	int drop=FALSE;
	char bf[BFSZ];
	char fname[BFSZ];
	int inhead=TRUE;
	lprintf(fname,BFSZ-1,"report.tmp_%d",(int) time(NULL));
	f = fopen(fname,"wb");
	if (f==NULL) {
		m->imsg("Could not write temp file %s\n",strerror(errno));
		return FALSE;
	}
	fprintf(f,"From: Dmail_filter@dmail.filter\n");
	fprintf(f,"X-MFilter-Report: This is a report generated by mfilter\n");
	fprintf(f,"Subject: %.300s\n",subj);
	fprintf(f,"To: %s\n",to);
	fprintf(f,"\n");
	if (lines>0) {
		fprintf(f,"Top %d lines of message attached below\n",lines);
		fprintf(f,"\n");
		fin = fopen(m->msgfile,"rb");
		if (fin==NULL) goto failed;
		for (i=0;(i<lines) & (!feof(fin));i++) {
			if (fgets(bf,BFSZ-1,fin)==NULL) break;
			if (inhead) if (strlen(bf)<3) {
				Wordlist *w;
				inhead = FALSE;
				for (w=m->hdr_add; w!=NULL; w=w->next) {
					fprintf(f,"%s\n",w->word); // add special headers.
				}
			}
			fprintf(f,"%s",bf);
			if (strstr(bf,"X-MFilter-Report:")!=NULL) drop = TRUE;
		}
		fclose(fin);
	} else {
		fprintf(f,"Message reported\n");
	}
failed:	fclose(f);
		if (drop) {
			m->imsg("Not sending report as that message is a report!!\n");
			remove(fname);
			return TRUE;
		}
	dsmtp_send_file(fname,to);
	//remove(fname);
	return TRUE;
}
int score_one(char *item, char *s)
{
	return 0;
}
int score_all(char *item)
{
	return 0;
}

char *file_config(char *s)
{
	return s;
}
int df_imsg(const char * arg_list, ...)
{
        va_list arg_ptr;
        char *format;
		char output[10000];
        int len;

        va_start(arg_ptr, arg_list);
        format = (char *) arg_list;
        len = vsnprintf(output,500, format, arg_ptr); output[400] = 0;
	va_end(arg_ptr);
	if (len>5000) {
		msgwrap("df_imsg fatal failed with long string %.100s\n",output);
		lib_abort();
	}
	msgwrap("%.500s",output);
	return 0;
}
int df_cmsg(const char * arg_list, ...)
{
        va_list arg_ptr;
        char *format;
		char output[10000];
        int len;

        va_start(arg_ptr, arg_list);
        format = (char *) arg_list;
        len = vsnprintf(output, 400,format, arg_ptr); output[400] = 0;
	va_end(arg_ptr);
	if (len>5000) {
		msgwrap("df_imsg fatal failed with long string %.100s\n",output);
		lib_abort();
	}
	if (df_msgfn!=NULL) df_msgfn(output);
	return 0;
}

int mfilter_test(int (*msgfn)( char *mtext),char *tfile, char *rfile)
{
	char *item;
	int r;
	char reason[BFSZ];
	FProg *fprog;
	char xrfile[BFSZ];
	Fwddata *fwdlist,*fwd,*fnext;
	Mfilter *m = mfilter_new();

	df_msgfn = msgfn;
	ncpy(reason,"None",100);
	item = item_read(tfile);
	if (item==NULL) {
		df_cmsg("Could not read (%s) failed (%s)\n",tfile,strerror(errno));
		mfilter_free(m);
		return 0;
	}
	fwdlist = makeup_fwd(m,item);

	dfilter_emsg(m,df_cmsg);
	dfilter_imsg(m,df_cmsg);
	dfilter_trace(m,TRUE);
	if (lib_strlen(rfile)==0) rfile = m->mfilter_fname;
	ncpy(xrfile,rfile,100);
	fprog = dfilter_compile(m,rfile);
	if (fprog==NULL) {
		df_cmsg("Compile of (%s) failed (%s)\n",xrfile,dfilter_error(m));
		goto failed;
	}

	r = dfilter_do(1,fprog,tfile,&fwdlist,reason);
	if (r) df_cmsg("ACCEPTED IT (%s)\n",reason);
	else df_cmsg("REJECTED IT (%s)\n",reason);
	dfilter_free(m,fprog);
	for (fwd=fwdlist;fwd!=NULL;fwd=fnext) {
		fnext = fwd->next;
		if (fwd->path!=NULL) myfree(fwd->path);
		if (fwd->uinfo!=NULL) myfree(fwd->uinfo);
		free(fwd);
	}
failed:
	dfilter_trace(m,FALSE);
	dfilter_emsg(m,df_imsg);
	dfilter_imsg(m,df_imsg);
	df_msgfn =NULL;
	mfilter_free(m);
	return TRUE;
}

Fwddata *makeup_fwd(Mfilter *m, char *item)
{
	Fwddata *flist=NULL;
	char tmp2[BFSZ];
	char bf[MAX_LINE];
	int upto=0;

	/* Pull out 'to' and 'cc' and 'bcc' headers and generate forward list */


	for (;TRUE;) {
		if (!df_getline(item,&upto,bf,MAX_LINE-100)) break;
		if (strncmpnc(bf,"To: ",4)==0) df_add_fwd(&flist,df_trim_email(bf+4,tmp2));
		if (strncmpnc(bf,"Cc: ",4)==0) df_add_fwd(&flist,df_trim_email(bf+4,tmp2));
		if (strncmpnc(bf,"Bcc: ",5)==0) df_add_fwd(&flist,df_trim_email(bf+5,tmp2));
		if (strncmpnc(bf,"From: ",6)==0) ncpy(m->cur_sender,bf+6,BFSZ-1);
		if (strncmpnc(bf,"Ip: ",4)==0) ncpy(m->cur_ip,bf+4,BFSZ-1);
		if (lib_strlen(bf)==0) break;
	}
	return flist;
}

int df_spamdetect(Mfilter *m,float score, char *reason)
{
	Spamlist *s;

	for(s=m->splist;s!=NULL;s=s->next)
	{
		if(stricmp(reason, s->reason)==0)
		{
			if (m->noisey) m->imsg("runspamdetect resetting %f (%s)\n", score, reason);

			s->score=score;
			return TRUE;
		}
	}

	if(score==0.0f)
		return TRUE;

	

	s = mymalloc(sizeof(*s));
	s->next = m->splist;
	s->reason = mystrdup(reason);
	s->score = score;
	m->splist = s;
	return TRUE;
}
int mfilter_do_not_add(Mfilter *m,int x)
{
	m->df_add_disabled = x;
	return TRUE;
}
int mf_set_always(Mfilter *m,int x)
{
	m->show_always = x;
	return TRUE;
}
float df_spamdetect_do(Mfilter *m)
{
	char reason[BFSZ];
	char bf[BFSZ];
	Spamlist *s;
	float tot=0.0f;
	int runner;

	runner=0;
	strcpy(reason,"");
	for (s=m->add_splist; s!=NULL; s=s->next) {
		if(s->score==0) continue;
		tot += s->score;
		if(runner<750) {
			if(runner>0) strcat(reason,", ");
			strcat(reason,s->reason);
			runner+=lib_strlen(s->reason);
			sprintf(reason+lib_strlen(reason),"=%1.1f",s->score);
			runner = lib_strlen(reason);
		}
	}
	mfilter_free_add(m);

	for (s=m->passed_splist; s!=NULL; s=s->next) {
		if(s->score==0) continue;
		tot += s->score;
		if(runner<750) {
			if(runner>0) strcat(reason,", ");
			strcat(reason,s->reason);
			runner+=lib_strlen(s->reason);
			sprintf(reason+lib_strlen(reason),"=%1.1f",s->score);
			runner = lib_strlen(reason);
		}
	}
	for (s=m->splist; s!=NULL; s=s->next) {
		if(s->score==0)
			continue;
		tot += s->score;
		if(runner<750)
		{
			if(runner>0) strcat(reason,", ");
			strcat(reason,s->reason);
			runner+=lib_strlen(s->reason);
			sprintf(reason+lib_strlen(reason),"=%1.1f",s->score);
			runner = lib_strlen(reason);
		}
	}

	if(tot>1.0f || m->show_always)
	{
		char stars[256];
		int intscore, i;

		intscore= (int) tot;
		if(intscore>30)
			intscore=30;
		for(i=0;i<intscore;i++)
			stars[i]='*';
		stars[i]=0;

		lprintf(bf,BFSZ-1,"X-SpamDetect: %s: %f %.750s",stars,tot,reason);
		ncpy(m->spamdetect_save,bf+14,BFSZ-1);

		if (!m->df_add_disabled) df_hdrs_add(m,bf);
	}
	
	return tot;
}
int df_spamdetect_free(Mfilter *m)
{
	Spamlist *s,*next;
	for (s=m->splist; s!=NULL; s=next) {
		next = s->next;
		free(s->reason);
		free(s);
	}
	m->splist = NULL;
	return TRUE;
}
char *cleanstr(char *s,char *s2)
{
	char *pos,c;
	int pos2=0;

	if(!s2) return s;
	pos=s;
	while(c=*pos) {
		if((c>='A'&&c<='Z')||(c>='a'&&c<='z')||(c>='0'&&c<='9')||c==' ')
			*(s2+(pos2++))=c;
		pos++;
	}
	*(s2+pos2)='\0';
	return s2;
}

int mf_fast_hash(char *reg)
{
	int h;
	h = tolower(reg[0]) + tolower(reg[1])*256;
	if (h<0) h = -h;
	return h%MFHASH;
}
int mf_fast_add(FProg *fp, double score, char *reg, char *reason, char *flag)
{
	int h;
	char hash[BFSZ];
	char *pcre_error;
	int pcre_errloc;
	Fast *f;
	Mfilter *m = fp->m;
	char *out;
	char *s;
	ncpy(hash,reg,10);
	for (out=hash,s=reg; *s!=0;s) {
		if (*s=='\\') {s++;};
		if (*s==0) break;
		*out++ = *s++;
	}
	*out++ = 0;
	h = mf_fast_hash(hash);
	f = calloc(1,sizeof(Fast));
	f->reg = strdup(reg);
	f->score = score;
	if(flag)
		f->xflag = df_flag_findadd(m,flag);
	else
		f->xflag = -1;
	f->len = lib_strlen(reg);
	f->reason = strdup(reason);
	f->next = fp->ftable[h];
	fp->ftable[h] = f;


	f->pcreg=pcre_compile(reg, PCRE_CASELESS | PCRE_EXTENDED | PCRE_MULTILINE,  (const char **)&pcre_error, &pcre_errloc, NULL);
	if(pcre_error!=NULL) {
		m->imsg("Regular expression failed at %d in (%.40s) (%.40s)\n",pcre_errloc, reg, pcre_error);
		pcre_freex(f->pcreg);
		return FALSE;
	}
	f->hints=pcre_study(f->pcreg, 0, (const char **)&pcre_error);
	if (pcre_error!=NULL) pcre_freex(f->hints);

	return TRUE;
}
int mf_fast_clear(FProg *fp)
{
	int h;
	Fast *f;
	for (h=0; h<MFHASH; h++) {
		for (f=fp->ftable[h];f!=NULL;f=f->next) {
			f->used = FALSE;
		}
	}
	return TRUE;
}
int mf_fast_free(FProg *fp)
{
	int h;
	Fast *f,*next;
	for (h=0; h<MFHASH; h++) {
		for (f=fp->ftable[h];f!=NULL;f=next) {
			next = f->next;
			if (f->pcreg!=NULL) pcre_freex(f->pcreg);
			if (f->hints!=NULL) pcre_freex(f->hints);
			free(f->reason);
			free(f->reg);
			free(f);
		}
		fp->ftable[h] = NULL;
	}
	return TRUE;
}
int mf_fast_scan(FProg *fp, char *msg, int len)
{
	int i;
	int h;
	int offsets[199];
	Fast *f;
	Mfilter *m = fp->m;
	int r;
	int rlen;
	mf_fast_clear(fp);
	if (!fp->m->is_rexp_fast) return FALSE;

	for (i=0; i<len-1; i++) {
		h = mf_fast_hash(msg+i);
		f=fp->ftable[h];
		for (;f!=NULL;f=f->next) {
			if (f->used) continue;
			rlen = 100;
			if ( (len-i)<rlen ) rlen = len-i;
			r = pcre_exec(f->pcreg, f->hints, msg+i, rlen, 0, 0, offsets, 99);
			if (r>=0) {
				df_spamdetect(fp->m,(float) f->score,f->reason);
				if(f->xflag>=0)
				{
					df_setflag(m,f->xflag,TRUE);
					if (m->noisey) m->imsg("setting expression flag %s\n", m->curprog->flags[f->xflag]);
				}
				f->used = TRUE;
			}
		}
	}
	return TRUE;
}

static char mytable[256];
static int done_table;
int mf_normalize(char *msg, char *out, int outsz, int inlen)
{
	char *s;
	unsigned char c,lastc;
	int len=0;
	int i;
	lastc=0;
	outsz -= 3;
	if (!done_table) {
		done_table = TRUE;
		for (i='a'; i<='z'; i++) mytable[i] = i;
		for (i='A'; i<='Z'; i++) mytable[i] = i - ('A' - 'a');
		mytable['0'] = 'o';
		mytable['1'] = 'l';

		mytable['3'] = 'e';
		mytable[0xeb] = 'e';
		mytable[130] = 'e';
		mytable[136] = 'e';
		mytable[137] = 'e';
		mytable[138] = 'e';

		mytable[0xe4] = 'a';
		mytable[131] = 'a';
		mytable[132] = 'a';
		mytable[133] = 'a';
		mytable[134] = 'a';
		mytable[160] = 'a';

		mytable[0xed] = 'i';
		mytable[139] = 'i';
		mytable[140] = 'i';
		mytable[141] = 'i';

		mytable['i'] = 'l';
		mytable['@'] = 'a';
		mytable['|'] = 'l';
	}
	for (s=msg;TRUE;s++) {
		c = *s;
		if (c==0) break;
		if ((s-msg)>inlen) break;
		c = mytable[c];
		if (c==0) continue;
		if (len>=outsz) break;
		if (lastc==c) continue;
		lastc = c;
		out[len++] = c;
	}
	out[len++] = 0;
	return TRUE;
}
int mf_hidden_scan(Mfilter *m, char *msg, int len)
{
	char bigmsg[4002];
	char reason[BFSZ];
	Hidden *h;
	char *s;
	if (!m->ishidden) return TRUE;
	if (m->hlist==NULL) return TRUE;
	mf_normalize(msg,bigmsg,4000,len);
	for (h=m->hlist;h!=NULL;h=h->next) {
		if (strstr(bigmsg,h->word)!=NULL) {
			if (strstrnc(msg,h->original)==NULL) {
				s = strstr(bigmsg,h->word);
				if (s==NULL) s = "none";
				lprintf(reason,BFSZ-1,"Hidden (%s) (%.8s)",h->original,s);
				df_spamdetect(m,(float) h->score,reason);
			}
		}
	}
	return TRUE;
}
int mf_hidden_free(Mfilter *m)
{
	Hidden *h,*next;
	for (h=m->hlist;h!=NULL;h=next) {
		next = h->next;
		free(h->word);
		free(h->original);
		free(h);
	}
	m->hlist = NULL;
	return TRUE;
}
/****************************************************************

	Code for executing boolean expressions on flags

****************************************************************/

enum {ET_LBRACKET, ET_RBRACKET, ET_OR, ET_AND, ET_FLAG, ET_NOT};

int check_expr(expr_token *tok)
{
	int depth, last;

	depth=0;
	if(tok->type==ET_NOT)
		last=ET_NOT;
	else if(tok->type==ET_FLAG)
		last=ET_FLAG;
	else if(tok->type==ET_LBRACKET)
	{
		depth++;
		last=ET_LBRACKET;
	}
	else
		return 0;

	tok=tok->next;
	while(tok!=NULL)
	{
		if((last==ET_OR)||(last==ET_AND)||(last==ET_LBRACKET)||(last==ET_NOT))
		{
			if((tok->type==ET_OR)||(tok->type==ET_AND)||(tok->type==ET_RBRACKET))
				return 0;
		}
		else if((last==ET_RBRACKET)||(last==ET_FLAG))
		{
			if((tok->type==ET_LBRACKET)||(tok->type==ET_FLAG)||(tok->type==ET_NOT))
				return 0;
		}

		last=tok->type;
		if(tok->type==ET_LBRACKET)
			depth++;
		if(tok->type==ET_RBRACKET)
			depth--;
		tok=tok->next;

		if(depth<0)
			return 0;
	}

	if(last==ET_FLAG)
		return 1;
	if((last==ET_RBRACKET)&&(depth==0))
		return 1;
	return 0;
}

expr_token *parse_expr(Mfilter *m, char *expr)
{
	char *upto, *s, c;
	expr_token *cur_token, *rval;

	if(expr==NULL)
		return NULL;

	rval=calloc(1, sizeof(expr_token));
	cur_token=rval;
	upto=expr;
	while(((*upto=='\t')||(*upto==' '))&&(*upto!=0))
		upto++;
	while(*upto!=0)
	{
		if(*upto=='(')
		{
			cur_token->next=calloc(1, sizeof(expr_token));
			cur_token=cur_token->next;
			cur_token->type=ET_LBRACKET;
			upto++;
		}
		else if(*upto==')')
		{
			cur_token->next=calloc(1, sizeof(expr_token));
			cur_token=cur_token->next;
			cur_token->type=ET_RBRACKET;
			upto++;
		}
		else if(*upto=='!')
		{
			cur_token->next=calloc(1, sizeof(expr_token));
			cur_token=cur_token->next;
			cur_token->type=ET_NOT;
			upto++;
		}
		else if(strncmp(upto, "||", 2)==0)
		{
			cur_token->next=calloc(1, sizeof(expr_token));
			cur_token=cur_token->next;
			cur_token->type=ET_OR;
			upto+=2;
		}
		else if(strncmp(upto, "&&", 2)==0)
		{
			cur_token->next=calloc(1, sizeof(expr_token));
			cur_token=cur_token->next;
			cur_token->type=ET_AND;
			upto+=2;
		}
		else if((*upto=='_')||((*upto>='A')&&(*upto<='Z')))
		{
			s=upto;
			while((*s=='_')||((*s>='A')&&(*s<='Z'))||((*s>='0')&&(*s<='9')))
				s++;
			c=*s;
			*s=0;

			cur_token->next=calloc(1, sizeof(expr_token));
			cur_token=cur_token->next;
			cur_token->type=ET_FLAG;
			cur_token->flag=df_flag_findadd(m,upto);

			*s=c;
			upto=s;
		}
		else
		{
			cur_token=NULL;
			break;
		}
		while(((*upto=='\t')||(*upto==' '))&&(*upto!=0))
			upto++;
	}

	if(cur_token!=NULL)
	{
		if(!check_expr(rval->next))
			cur_token=NULL;
	}

	if(cur_token!=NULL)
	{
		cur_token=rval->next;
		free(rval);
		return cur_token;
	}

	while(rval!=NULL)
	{
		cur_token=rval->next;
		free(rval);
		rval=cur_token;
	}

	return rval;
}


int df_expr_findadd(Mfilter *m,char *expr)
{
	expr_token *e;
	expr_handle *h;

	for(h=m->curprog->expr_handles;h!=NULL;h=h->next)
	{
		if(strcmp(h->orig_expr, expr)==0)
			return h->n;
	}

	e=parse_expr(m,expr);

	if(e==NULL)
	{
		m->fmsg("Bad expression: %s line=%d\n", expr,m->lineno);
		return FALSE;
	}

	h=calloc(1, sizeof(expr_handle));
	h->expr=e;
	h->orig_expr=strdup(expr);
	if(m->curprog->expr_handles==NULL)
		h->n=0;
	else
		h->n=m->curprog->expr_handles->n+1;
	h->next=m->curprog->expr_handles;
	m->curprog->expr_handles=h;

	return h->n;
}

expr_token *df_expr_evalsub(Mfilter *m, expr_handle *h, expr_token *e, int *cur)
{
	int notting=FALSE;
	int lvalue, rvalue;
	int opand;

	while(e->type==ET_NOT)
	{
		notting=!notting;
		e=e->next;
	}
	if(e->type==ET_FLAG)
	{
		lvalue=(df_flag_get(m,e->flag)!=0);
		e=e->next;
	}
	else if(e->type==ET_LBRACKET)
		e=df_expr_evalsub(m,h, e->next, &lvalue);
	else
		return NULL;

	if(notting)
		lvalue=!lvalue;

	while(e!=NULL)
	{
		if(e->type==ET_RBRACKET)
		{
			*cur=lvalue;
			return e->next;
		}
		if((e->type==ET_LBRACKET)||(e->type==ET_NOT)||(e->type==ET_FLAG))
			return NULL;
		if(e->type==ET_AND)
			opand=TRUE;
		else
			opand=FALSE;
		e=e->next;
		if(e==NULL)
			return NULL;

		notting=FALSE;
		while(e->type==ET_NOT)
		{
			notting=!notting;
			e=e->next;
		}
		if(e->type==ET_FLAG)
		{
			rvalue=(df_flag_get(m,e->flag)!=0);
			e=e->next;
		}
		else if(e->type==ET_LBRACKET)
			e=df_expr_evalsub(m,h, e->next, &rvalue);
		else
			return NULL;

		if(notting)
			rvalue=!rvalue;

		if(opand)
			lvalue=(lvalue && rvalue);
		else
			lvalue=(lvalue || rvalue);
	}

	*cur=lvalue;

	return NULL;
}

void df_free_expr_token(expr_token *t)
{
	expr_token *next;
	while(t) {
		next=t->next;
		free(t);
		t=next;
	}
}

void df_expr_free(FProg *fp)
{
	expr_handle *h,*next;
	for(h=fp->expr_handles;h!=NULL;h=next)
	{
		next=h->next;
		df_free_expr_token(h->expr);

		free(h->orig_expr);
		free(h);
	}
	fp->expr_handles = NULL;
}

int df_expr_eval(Mfilter *m,int expr)
{
	expr_handle *h;
	int rval=FALSE;

	if(expr<0)
		return FALSE;

	for(h=m->curprog->expr_handles;h!=NULL;h=h->next)
	{
		if(h->n==expr)
			break;
	}
	if(h==NULL)
		return FALSE;

	df_expr_evalsub(m,h, h->expr, &rval);

	return rval;
}


#ifdef PAIRTEST
//	FILE *pairfile;
#endif

int comp_invalidpairs(FProg *fp,Exp *exp, char *arg1, char *arg2, char *arg3[]) /* letterpairs("header","significance","max_percent_words_not_matching") */
{
	exp->header=df_header_findadd(exp->m,arg1);
	exp->p2=mystrdup(arg2);
	exp->p3=mystrdup(*arg3);
#ifdef PAIRTEST
	pairfile=fopen("mfilter.pair","w");
	if(pairfile) fclose(pairfile);
#endif
	return TRUE;
}

int mfilter_pairs(Mfilter *m)
{
	return m->g_pairs;
}
int run_invalidpairs(Exp *exp)
{
	int sig,mxpct,wdcnt,pos,bad;
	char *wd,lc;
	Mfilter *m = exp->m;
	char p[3]={0};
	sig=atoi(exp->p2);
	m->g_pairs = 0;
	if(sig<=0 || sig>=100) return 0;
	mxpct=atoi(exp->p3);
	if(mxpct<=0 || mxpct>=100) return 0;

	wd=df_hdrs_get(m,exp->header);
	if(!wd) return 0;

	for(wdcnt=1,lc=*wd,bad=0;*wd && *(wd+1);wd++) {
		if(*wd=='\t'||*wd=='\r'||*wd=='\n'||*wd==' ') {
			if(lc!=' ') wdcnt++;
		} else {
			if(mfisalpha(*wd) && mfisalpha(*(wd+1))) {
				p[0]=*wd;
				p[1]=*(wd+1);
				strlwr(p);
				p[0]-='a';
				p[1]-='a';
				pos=26*p[0];
				pos+=p[1];

				if(lp_frequency_of_combinations[pos]<(unsigned int)lp_significance[sig]) {
					bad++;
					while(*wd && !strchr("\t \r\n",*wd)) wd++;
					wd--;
				}
			} else if( (
				mfisnum(*wd)
				&& mfisalpha(*(wd+1))
				)
				|| (
				mfisalpha(*wd)
				&& mfisnum(*(wd+1))
				) ) {
					bad++;
					while(*wd && !strchr("\t \r\n",*wd)) wd++;
					wd--;
			}
		}
		if(*wd=='\t'||*wd=='\r'||*wd=='\n'||*wd==' ') lc=' ';
		else lc=*wd;
	}

#ifdef PAIRTEST
	pairfile=fopen("mfilter.pair","a");
	if(pairfile) {
		fprintf(pairfile,"%d %d\n",sig,(100*bad)/wdcnt);
		fclose(pairfile);
	}
#endif

	if((100*bad)/wdcnt>mxpct) {
		m->g_pairs = 1;
		return 1;
	}
	return 0;
}

char *df_strcatn(char *dst,char *src,int len)
{
        char *xdst = dst;
        int xlen = len;
		int r = lib_strlen(dst);

		len -= r;
		dst += r;
        for (;(*src !=0) && (len>0); ) {*dst++  = *src++; len--;}
        *dst++ = 0;
        xdst[xlen] = 0;
        return xdst;
}
char *df_decode_token(char *wd) /* RL */
{
	char *s;
	int l=lib_strlen(wd);
	char out[BFSZ];

	if(*wd=='=' && *(wd+1)=='?') { /* Looks like an encoded word */
		for(s=(wd+2);*s;s++) {
			if(*s==' '||*s=='\t') return wd;
			if(*s=='?') { s++; break; }
		}
		if (!*s) return wd;
		if(*(s+1)!='?') return wd;
		if(wd[l-1]=='=') wd[l-1]='\0';
		if(*s=='b'||*s=='B') { /* Base64 encoded token */
			decode_base64(s+2,out,BFSZ-1);
		} else if(*s=='q'||*s=='Q') { /* Quoted printable encoded token */
			decode_quoted(s+2,out,BFSZ-1);
		} else return wd; /* Not encoded or encoding not defined in RFC2047 */
		ncpy(wd,out,BFSZ-1);
	}
	return wd;
}

char *df_decode_words(Mfilter *m, char *bf) /* Decode RFC2047 encoded text, RL */
{
	int x;
	int len;
	char out[BFSZ];
	char tmp[BFSZ];

#ifdef SURGEMAIL
	len=lib_strlen(bf);
	if (len==0) return bf;
	for(x=1,*tmp='\0';x;x++) {
		lib_get_word(out,bf,x);
		if(*out) {
			if(x>1) df_strcatn(tmp," ",BFSZ-1);
			df_decode_token(out);
			df_strcatn(tmp,out,BFSZ-1);
		} else break;
	}
	ncpy(bf,tmp,len);
#endif
	return bf;
}


int df_bounce_hash(char *s)
{
	int h = tolower(s[0]) + (tolower(s[1])*255);
	h = h % BHASH;
	if (h<0) h = -h;
	return h;
}
int df_bounce_fast_add(Mfilter *m,double score, char *find, char *show)
{
	int h = df_bounce_hash(find);
	Bounce *b;
	b = calloc(1,sizeof(Bounce));
	b->find = strdup(find);
	b->show = strdup(show);
	b->score = score;
	b->next = m->bhash[h];
	m->bhash[h] = b;
	return TRUE;
}
int df_bounce_fast_find(Mfilter *m,char *body, int len, char *show)
{
	int flen;
	int h;
	Bounce *b;

	for (;len>0;body++,len--) {
		if (*body==0) break;
		h = df_bounce_hash(body);
		for (b=m->bhash[h];b!=NULL;b=b->next) {
//printf("Rule (%s) %.*s\n",b->find,flen,body);
			flen = lib_strlen(b->find);
			if (strncmpnc(body,b->find,flen)==0) {
				ncpy(show,b->show,BFSZ-1);
				return TRUE;
			}
		}
	}
	return FALSE;
}
int df_bounce_fast_clear(Mfilter *m)
{
	Bounce *b,*next;
	int h;
	for (h=0; h<BHASH; h++) {
		for (b=m->bhash[h];b!=NULL;b=next) {
			next = b->next;
			free(b->find);
			free(b->show);
			free(b);
		}
		m->bhash[h] = NULL;
	}
	return TRUE;
}

void mfilter_exit(Mfilter *m)
{
	df_spamdetect_free(m);
	df_bounce_fast_clear(m);
	mf_hidden_free(m);
	dfilter_free_statics(m);
}
