/*
MIME Message formatting code for smtp sending (or IMAP upload) - designed to be generally used 
	in conjuction with prcc.c and smail.c 
	
(used to be part of surgeweb level code, now also used by surgealert)

Example of use:
*/

#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include <stdarg.h>
#include <time.h>
#include "debug.h"
#include "tcplib.h"
#include "smail.h"
#include "lprintf.h"
#include "lib.h"
#include "decode.h"
#include "thread.h"
#ifdef USE_DFOPEN
#include "dfopen.h"
#endif
#include "sformat.h"
#include "parts.h"
#include "prcc.h"
#include "buffer.h"
#include "html.h"
#include "css.h"
#include "wash.h"

#ifndef FALSE
#define FALSE 0
#endif

#ifndef TRUE
#define TRUE (!FALSE)
#endif

extern int emsg(const char * arg_list, ...);
extern int dmsg(const char * arg_list, ...);

extern int cloudfile_datfile(char *file);
extern int cloudfile_decode(char *line, int *expire, int*size, char*filename, char *link, int bfsz);
void sf_output_bigfiles(SFormat *sf, FILE *f2, int update_top);


static int (*imsg)(const char *, ...)=NULL;
void sformat_set_imsg(int (*fn)(const char *, ...)) {
	imsg = fn;
}
static char* (*sformat_tmpfile)(void*,char *)=NULL;
void sformat_set_tmpfile(char* (*fn)(void*,char *)) {
	sformat_tmpfile = fn;
}
static char* (*sformat_decode_filename)(void*,char *)=NULL;
void sformat_set_decode_filename(char* (*fn)(void*,char*)) {
	sformat_decode_filename = fn;
}
static FILE* (*sformat_wash_fordisplay_html)(void *, FILE *, int, int, Pair *)=NULL;
void sformat_set_wash_fordisplay_html(FILE* (*fn)(void *, FILE *, int, int, Pair *)) {
	sformat_wash_fordisplay_html = fn;
}

int sformat_new(SFormat **sf)
{
	SFormat *s;
	s = calloc(sizeof(SFormat),1);
	*sf = s;
	return TRUE;
}
int sformat_init(SFormat *sf, char *type)
{
	int t;
	sf->r=rand();
	t=(int)time(NULL);
	lprintf(sf->sep_mix,BFSZ,"----=_%s_%d_%d_mixed=",type,sf->r,t);
	lprintf(sf->sep_mpa,BFSZ,"----=_%s_%d_%d_mpa=",type,sf->r,t);
	lprintf(sf->sep_mpr,BFSZ,"----=_%s_%d_%d_mpr=",type,sf->r,t);
	return TRUE;
}
int sformat_free(SFormat **sf)
{
    pair_free(&(*sf)->param);

    free(*sf);
	*sf = NULL;
	return TRUE;
}


int sformat_recipient_header(FILE *f,char *hdr, Pair* rcpts)
{
	Pair *p;
	char bf[BFSZ];
	int i=0;
	if (!rcpts) return FALSE;
	fprintf(f,"%s: ",hdr);
	for (p=rcpts;p;p=p->next,i++){
		if(i%2==0 && i>0)
			fprintf(f,"\r\n	");
		if (lib_strlen(p->val)>0 && strcmp(p->var,p->val)!=0){	// SEARCH: COMMA IN ADDRESS
			if(lib_strlen(p->val)>0 && !lib_isquoted(p->val) && ( strchr(p->val,',')!=NULL || strchr(p->val,';')!=NULL))
				fprintf(f," \"%s\"",sformat_encode_rfc1522(p->val,bf));
			else	
				fprintf(f," %s",sformat_encode_rfc1522(p->val,bf));
		}
		fprintf(f," <%s>",p->var);
		if(p->next)
			fprintf(f,",");
	}
	fprintf(f,"\r\n");
	return TRUE;
}

// UPDATE: now that we have: utf8_decode_next this would be trivial to implement :-) do at some stage!
//
// UTF encode all on one line for now... Technically this is slightly bad as headers should really be length limited say:
//Subject: =?UTF-8?B?5pyf5Ye95pWw5pyf5Ye95pWw5pyf5Ye95pWw5pyf5Ye95pWw5pyf5Ye9?=
// =?UTF-8?B?5pWw5pyf5Ye95pWw5pyf5Ye95pWw5pyf5Ye95pWw5pyf5Ye95pWw5pyf5Ye95pWw?=
// =?UTF-8?B?5pWw5pyf5Ye95pWw5pyf5Ye95pWw5pyf5Ye95pWw?=
// Old webmail has NEVER length limited its subject field so I think I am ok not doing so for now!!!
char *sformat_encode_rfc1522(char *src, char *dst)
{
	char *s,bf[BFSZ*4],tmp[BFSZ];
	int encoded=FALSE,len;
	for(s=src;*s;s++){
		if (*s < 0x20 || *s > 0x7e){
			encoded = TRUE;
			break;
		}
	}
	if(!encoded){
		ncpy(dst,src,BFSZ-1);
		return dst;
	}
	len=lib_strlen(src);
	if(len>BFSZ-100) len=BFSZ-100;
	encode_base64(src,bf,len+1);
	// All a bit of a hack but try and deal with long strings reasonably nicely within 1KB buffer constraints
	if(lib_strlen(bf)>BFSZ-100){
		ncpy(tmp,src,200);
		ncat(tmp,"...",BFSZ-1);
		encode_base64(tmp,bf,lib_strlen(tmp)+1);
	}
	lprintf(dst,BFSZ,"=?%s?B?%s?=","UTF-8",bf);
	return dst;
}
// Decode routime is:  	mime_header_decode(m->headers[i],bfp,len);
// also relevant: decode_iso

char *sformat_get_content_type(SFormat *sf, char *bf, int level)
{
	int found=FALSE;
	if (!found && sf->attach && level<=SF_ATTACH){
		lprintf(bf,BFSZ,"multipart/mixed; boundary=\"%s\"", sf->sep_mix);
		found=TRUE;
	}
	if (!found && sf->text_only && level<=SF_MPA){
		lprintf(bf,BFSZ,"text/plain; charset=%s%s", sf->n_txt==0?"us-ascii":"utf-8", (sf->flowed_txt?"; format=flowed":""));
		found=TRUE;
	}
	if (!found && level<=SF_MPA){
		lprintf(bf,BFSZ,"multipart/alternative; boundary=\"%s\"", sf->sep_mpa);
		found=TRUE;
	}
	if (!found && level<=SF_MPR){
		lprintf(bf,BFSZ,"multipart/related; boundary=\"%s\"", sf->sep_mpr);
		found=TRUE;
	}
	return bf;
}

int sformat_message_header(SFormat *sf, FILE *f2,char *mime_header, int *fstart)
{
	char bf[BFSZ],msgid[BFSZ],*s;
	
	*fstart=0;
	if(sf->rcpts_bcc){
		sformat_recipient_header(f2,"BCC",sf->rcpts_bcc);
		*fstart=ftell(f2);
	}

	fprintf(f2,"From: %s\r\n",pair_find(sf->param,"from_rfc1522"));	// from_enc is already rfc1522 encoded
	if(sf->rcpts_to)
		sformat_recipient_header(f2,"To",sf->rcpts_to);
	else
		fprintf(f2,"To: %s\r\n",pair_find(sf->param,"rpc"));
	sformat_recipient_header(f2,"CC",sf->rcpts_cc);

	s=pair_find(sf->param,"reply_to");
	if(lib_strlen(s)>0){
		fprintf(f2,"Reply-To: %s\r\n",s);
	}
	s=pair_find(sf->param,"sender");
	if(lib_strlen(s)>0){
		fprintf(f2,"Sender: %s\r\n",s);
	}

	fprintf(f2,"Subject: %s\r\n",sformat_encode_rfc1522(pair_find(sf->param,"subject"),bf));
	fprintf(f2,"Date: %s\r\n",lib_mail_time(bf,time(NULL)));
    s=pair_find(sf->param,"message-id");
    if (lib_strlen(s)) {
        lprintf(msgid,BFSZ,"%s",s);
    }else{
        lprintf(msgid,BFSZ,"%x.%x.%x.%x@%s",(int) time(NULL) , th_pid(), th_id(), (int) myrand(), pair_find(sf->param,"domain"));
    }
    fprintf(f2,"Message-ID: <%s>\r\n",msgid);

	s=pair_find(sf->param,"in_reply_to");
	if (lib_strlen(s)>0) {
		fprintf(f2,"In-Reply-To: <%s>\r\n",s);
	}
    
	s=pair_find(sf->param,"priority");
	if(lib_strlen(s)>0){
		if(strcmp(s,"high")==0){
			fprintf(f2,"Importance: High\r\n");
			fprintf(f2,"X-Priority: 1 (Highest)\r\n");
			fprintf(f2,"Priority: urgent\r\n");
		}else{
			fprintf(f2,"Importance: Low\r\n");
			fprintf(f2,"X-Priority: 5 (Lowest)\r\n");
		}
	}
	
	s=pair_find(sf->param,"confirm");
	if(lib_strlen(s)>0){
		if(strcmp(s,"read")==0 || strcmp(s,"both")==0){
			fprintf(f2,"Disposition-Notification-to: %s\r\n",pair_find(sf->param,"from_address"));
			fprintf(f2,"X-Confirm-Reading-to: %s\r\n",pair_find(sf->param,"from_address"));
		}
	}

	s=pair_find(sf->param,"encrypt");
	if(lib_strlen(s)>0){
		if(strcmp(s,"surgevault")==0){
			fprintf(f2,"X-SurgeVault-Force: yes\r\n");
		}else if(strcmp(s,"always")==0){
			fprintf(f2,"X-SurgeVault-Force: always\r\n");
		}else if(strcmp(s,"never")==0){
			fprintf(f2,"X-SurgeVault-Force: never\r\n");
		}
	}

	// set via surgeweb clientside, preferences pref.vault_lock && pref.vault_ifnew,
	// these in turn get set using calls to mini_ifnew and mini_lock_byuser_domain
	s=pair_find(sf->param,"vaultlock_ifnew");
	if(lib_strlen(s)>0){
		if(strcmp(s,"validate")==0){
			fprintf(f2,"X-SurgeVault-Token: true\r\n");
		}
	}

	if(sf->cal > 0){
		fprintf(f2,"Content-class: urn:content-classes:calendarmessage\r\n");
		fprintf(f2,"Content-type: text/calendar; method=REPLY; charset=UTF-8\r\n");
		fprintf(f2,"Content-transfer-encoding: 8BIT\r\n");
	}else{
		fprintf(f2,"MIME-Version: 1.0\r\n");
		fprintf(f2,"Content-Type: %s\r\n", mime_header);
		if(!sf->attach && sf->enc_txt>0){
			fprintf(f2,"Content-Transfer-Encoding: %s\r\n", (sf->enc_txt==2?"base64":"quoted-printable"));
		}
	}
	s=pair_find(sf->param,"origip");
	if (lib_strlen(s)>0) {
		fprintf(f2,"X-Originating-IP: %s\r\n",s);
	}

	s=pair_find(sf->param,"mailer");
	if (lib_strlen(s)>0) {
		fprintf(f2,"X-Mailer: %s\r\n",s);
	}
	s=pair_find(sf->param,"xcontact");
	if (lib_strlen(s)>0) {
		fprintf(f2,"X-Surge-Contact: %s\r\n",s);
	}
    
    if(istrue(pair_find(sf->param,"is_note"))){
        s=pair_find(sf->param,"note_id");
        if (lib_strlen(s)==0) s = msgid;
        fprintf(f2,"X-Surge-NoteID: <%s>\r\n",s);
        
        s=pair_find(sf->param,"note_opt");
        if (lib_strlen(s)>0) {
            fprintf(f2,"X-Surge-NoteOptions: %s\r\n",s);
        }
    }
	fprintf(f2,"\r\n");

	if (lib_starts_with(mime_header,"multipart")){
		fprintf(f2,"This is a multi-part message in MIME format.\r\n");
		fprintf(f2,"\r\n");
	}
	return TRUE;
}

// 3/11/21: PARTIAL file io checking added to this function, for fileio problems to not get ignored totally silently!!
int sformat_whole_message(SFormat *sf, FILE **f_out, char *result)
{
	char bf[BFSZ];
	FILE *f2;
	int fstart=0,ret=TRUE,fail_count=0, failed=FALSE;

	imsg("sformat_message");
	ncpy(result,"FILE IO FAULT",BFSZ-1);

	// Prepare the appropriate parts for display
	if (sf->text_only){		// Plaintext input
		if (sformat_prepare_message_textpart(sf,pair_find(sf->param,"msg_body"), result)!=TRUE) failed = TRUE;
	}else{					// Html text input
		sf->text_only=strcmp(pair_find(sf->param,"send_mode"),"plain_text")==0;
		imsg("selected %s message send",(sf->text_only?"text_only":"html mpa"));
		sf->flowed_txt=!istrue(pair_find(sf->param,"no_replycolor"));
		if (sformat_prepare_message_textparts_mpa(sf,pair_find(sf->param,"msg_body"), result)!=TRUE) failed = TRUE;
	}
	if(failed){
		emsg("surgeweb: sformat_whole_message prepare failed [%s]",result);
		return FALSE;	
	}

	f2=fopen(sformat_tmpfile(sf->data, bf),"w+b");
	if(!f2){
		emsg("surgeweb: sformat_message failed to open temp file [%s]",bf);
		ncpy(result,"FILE WRITE FAILED 3",BFSZ-1);
		return FALSE;
	}

	// Work out what top level header should be so we can output the message headers
	sformat_get_content_type(sf,bf,SF_ATTACH);
	sformat_message_header(sf,f2,bf,&fstart);

	if (sf->attach){ 
		if (fprintf(f2,"--%s\r\n",sf->sep_mix)<0) failed = TRUE;
		if (fprintf(f2,"Content-Type: %s\r\n", sformat_get_content_type(sf,bf,SF_MPA))<0) failed = TRUE;
		if(sf->enc_txt>0){
			if (fprintf(f2,"Content-Transfer-Encoding: %s\r\n", (sf->enc_txt==2?"base64":"quoted-printable"))<0) failed = TRUE;
		}
		if (fprintf(f2,"\r\n")<0) failed = TRUE;
		imsg("attachment ids: %s",sf->attach_ids);
	}else{
		imsg("no attachments");
	}

	if (sf->text_only){
		// output text part..
		imsg("sformat_message text");
		// Should never have to add header here
		if (sformat_message_textpart(sf,f2,sf->sep_mpa,FALSE/*sf->attach?TRUE:FALSE*/, result)!=TRUE) failed = TRUE;
	} else {
		// Output MPA
		imsg("sformat_message mpa");

		//  - output text part... generated from html
		if (sformat_message_textpart(sf,f2,sf->sep_mpa,TRUE, result)!=TRUE) failed = TRUE;

		//  - output html part trying for mpr
		if(!sf->related){
			imsg("sformat_message not mpr");
			if (sformat_message_htmlpart(sf,f2,sf->sep_mpa,result)!=TRUE) failed = TRUE;
		}else{
			imsg("sformat_message mpr");
			if (fprintf(f2,"--%s\r\n",sf->sep_mpa)<0) failed = TRUE;
			if (fprintf(f2,"Content-Type: %s\r\n\r\n", sformat_get_content_type(sf,bf,SF_MPR))<0) failed = TRUE;
			if (sformat_message_htmlpart(sf,f2,sf->sep_mpr,result)!=TRUE) failed = TRUE;

			// For each related image
			if (sformat_message_relateds(sf,f2,sf->sep_mpr,sf->attach_ids,&fail_count,result)!=TRUE) failed = TRUE;

			if (fprintf(f2,"\r\n--%s--\r\n", sf->sep_mpr)<0) failed = TRUE;
		}
		if (fprintf(f2,"\r\n--%s--\r\n", sf->sep_mpa)<0) failed = TRUE;
	}
	if(failed){
		emsg("surgeweb: sformat_whole_message output failed [%s]",result);
		return FALSE;	
	}

	if (sf->attach){
		// Output each attachment
#ifdef YESIMONLINE
        sformat_message_attachments_files(sf,f2,sf->sep_mix,sf->attach_ids,&fail_count);
#else		
        if (sformat_message_attachments(sf,f2,sf->sep_mix,sf->attach_ids,&fail_count, result) != TRUE) failed = TRUE;
#endif
		if (fprintf(f2,"\r\n--%s--\r\n", sf->sep_mix)<0) failed = TRUE;
	}

	if( fprintf(f2,"\r\n")<0) failed = TRUE;
	
	// NOTE: File contains BCC header at start of file which does not get sent via smtp, 
	// but does get saved when saving to sent folder or saving as draft.
	if(fseek(f2,fstart,SEEK_SET)<0) failed = TRUE;

	*f_out=f2;

	if(failed){
		ncpy(result,"FILE WRITE FAILED 6", BFSZ-1);
		return FALSE;
	}
	if(fail_count>0) {
		lprintf(result,BFSZ,"%d attachments failed",fail_count);
		return FALSE;
	}

	return TRUE;
}

int sformat_message_textpart(SFormat *sf, FILE *f2, char *separator, int do_headers, char *result)
{
	char bf[BFSZ];
	int failed=FALSE;

	if(do_headers){
		if (fprintf(f2,"--%s\r\n",separator)<0) failed = TRUE;
		if (fprintf(f2,"Content-Type: text/plain; charset=%s%s\r\n", (sf->n_txt==0?"us-ascii":"utf-8"), (sf->flowed_txt?"; format=flowed":""))<0) failed = TRUE;
		if (sf->enc_txt>0){
			if (fprintf(f2,"Content-Transfer-Encoding: %s\r\n", (sf->enc_txt==2?"base64":"quoted-printable"))<0) failed = TRUE;
		}
		if( fprintf(f2,"\r\n")<0) failed = TRUE;
	}
	for (;!feof(sf->f_txt);) {
		if (!fgets(bf,BFSZ-1,sf->f_txt)) break;
		if (fprintf(f2,"%s",bf)<0) failed = TRUE;
	}
	if(fclose(sf->f_txt)<0) failed = TRUE;
	sf->f_txt=NULL;
	if(fprintf(f2,"\r\n")<0) failed = TRUE;

	if(failed){
		ncpy(result,"FILE WRITE FAILED 5",BFSZ-1);
		return FALSE;
	}

	return TRUE;
}

int sformat_prepare_message_textpart(SFormat *sf, char *body, char *result)
{
	char bf[BFSZ], *s;
	FILE *f;
	Strlines lines;
	int n_total=0, n_high=0, too_wide=FALSE, disable_reformat, failed=FALSE;

	// Temp file with raw html part
	f=fopen(sformat_tmpfile(sf->data, bf),"w+b");
	if(!f){
		emsg("surgeweb: sformat_prepare_message_textpart failed to open temp file [%s]",bf);
		ncpy(result,"FILE OPEN FAILED",BFSZ-1);
		return FALSE;
	}

	str_lines_start(&lines,body,lib_strlen(body));
	while(s = str_lines_next(&lines)) { 
        int len = lib_strlen(s);
        if (fprintf(f,"%s\r\n",s)<0) failed = TRUE; 
    }
	if(fseek(f,0,SEEK_SET)<0) failed = TRUE;

	if(failed){
		fclose(f);
		emsg("surgeweb: sformat_prepare_message_textpart failed to write temp file [%s]",bf);
		ncpy(result,"FILE WRITE FAILED 1",BFSZ-1);
		return FALSE;		
	}

	// Encode suitably for email (short mime encoded strings)
	// n_txt used to determine character set - a bit blunt, but safe without having charset detection code
	// (some of the high end chars may not need UTF-8 but that's OK)
//	f=prcc_mime_encode_qp(sf->data,f,&sf->n_txt);
//    sf->enc_txt = 1;

	disable_reformat = sf->cal;
	if(disable_reformat){
		// Sometimes we never want any smarts!!
	}else{
	    sf->flowed_txt = TRUE;
    
		// check what we need to do next
		prcc_count_chars(f,&n_total,&n_high);
		sf->n_txt=n_high;
		// Now jump through some hoops to encode suitably for email transport (ie short mime encoded strings)
		// Try for just format=flowed
		//	if more is needed use quoted-printable / base64 as appropriate
		if(sf->flowed_txt){
			imsg("trying for format=flowed");
			f=prcc_reflow_real(sf->data, f, &too_wide, TRUE);	
			// Set header to format flowed
		} 
		if ( (float)n_high / n_total > 0.5){
			// Lots of UTF8 so base64 encode it instead for efficiency
			imsg("lots of high ascii / utf8 (%d/%d) using base64 encoding",n_high,n_total);
			f=prcc_base64_mime(sf->data, f);
			sf->enc_txt=2;
			// MW 01/03/11 : Possible change of behaviour as below was commented out but should always have failed as far as I can tell...
		} else if(/*n_high>0 && sw->flowed ||*/ too_wide || !sf->flowed_txt) {
			imsg("Applying quoted-printable encoding (nhigh=%d/%d : too_wide=%d : sf->flowed_txt=%d)",n_high,n_total,too_wide,sf->flowed_txt);
			f=prcc_mime_encode_qp(sf->data,f,&sf->n_txt);
			sf->enc_txt=1;
		}
	}

	// Now keep that file handle for a while...
	sf->f_txt=f;

	return TRUE;
}


int sformat_wash_snippet(char *out, char *in, int max)
{
	char bf[BFSZ], *p;
    int len;
    ncpy(bf, in, BFSZ-1);
    if (lib_starts_with(bf, "[...]")) {
        p=bf+5;
    }else{
        p=bf;
    }
	while(sisspace(*p)) p++;
    len=lib_strlen(p);
    if(len>BFSZ-1) len=BFSZ-1;
    if (lib_ends_with(p, "[...]")) {
        len-=5;
    }
    ncpy(out, p, len);
	if (lib_strlen(out)>60){
		strcpy(out+60, "...");
	}
    return TRUE;
}
void sformat_update_subject(SFormat *sf, FILE *f)
{
	char bf[BFSZ], tmp[BFSZ];
	if(sf->generate_subject==0) return;
	prcc_snippet_from_file_keepf(bf, f, 100, TRUE);
	sformat_wash_snippet(tmp, bf, 60);
	if(lib_strlen(tmp)>0){
		if(sf->generate_subject==1){
			lprintf(bf,BFSZ,"~ %s",tmp);
		}else{
			lprintf(bf,BFSZ,"%s",tmp);
		}
	}else{
		lprintf(bf,BFSZ,"");
	}
	ncpy(sf->generated_subject,bf,BFSZ-1);
	pair_add(&sf->param,"subject",sf->generated_subject);
}


void sw_update_for_newmpa(void *sw, SFormat *sf);

// Setup two temp files for with the formatted content of each of the text and html parts
int sformat_prepare_message_textparts_mpa(SFormat *sf, char *body, char*result)
{
	char bf[BFSZ], *s;
	FILE *f,*ff;
	Strlines lines;
	int n_total=0, n_high=0, too_wide=FALSE, failed=FALSE;
	Pair *options=NULL;

	imsg("sformat_prepare_message_textparts_mpa start");

	// Temp file with raw html part
	ff=fopen(sformat_tmpfile(sf->data, bf),"w+b");
	if(!ff){
		emsg("surgeweb: sformat_prepare_message_textparts_mpa failed to open temp file [%s]",bf);
		return FALSE;
	}

	str_lines_start(&lines,body,lib_strlen(body));
	sf_output_bigfiles(sf,ff,true);
	while(s = str_lines_next(&lines)) {
		if (fprintf(ff,"%s\r\n",s)<0) failed = TRUE;
	}
	sf_output_bigfiles(sf,ff,false);
	if(fseek(ff,0,SEEK_SET)<0) failed = TRUE;

	if(failed){
		fclose(ff);
		emsg("surgeweb: sformat_prepare_message_textparts_mpa failed to write temp file [%s]",bf);
		ncpy(result,"FILE WRITE FAILED 2",BFSZ-1);
		return FALSE;		
	}

	// Convert raw html temp file to text only temp file
	pair_add(&options,"flowed","true");
	f=sformat_wash_fordisplay_html(sf->data,ff,TRUE,WASH_STRIP_TO_TEXT, options);
	pair_free(&options);

	// Remove additional encoded characters which we still seem to get passed (in particular: &nbsp; )
	f=prcc_web_cleanup(sf->data,f);

	// Optionally autogenerate a subject from the processed body if we need to :-)
	sformat_update_subject(sf,f);	// (keeps f open)

	// check what we need to do next
	prcc_count_chars(f,&n_total,&n_high);
	sf->n_txt=n_high;

	// Now jump through some hoops to encode suitably for email transport (ie short mime encoded strings)
	// Try for just format=flowed
	//	if more is needed use quoted-printable / base64 as appropriate
	if(sf->flowed_txt){
		imsg("trying for format=flowed");
		f=prcc_reflow(sf->data, f, &too_wide);	
		// Set header to format flowed
	} 
	if ( (float)n_high / n_total > 0.5){
		// Lots of UTF8 so base64 encode it instead for efficiency
		imsg("lots of high ascii / utf8 (%d/%d) using base64 encoding",n_high,n_total);
		f=prcc_base64_mime(sf->data, f);
		sf->enc_txt=2;
	// MW 01/03/11 : Possible change of behaviour as below was commented out but should always have failed as far as I can tell...
	} else if(/*n_high>0 && sw->flowed ||*/ too_wide || !sf->flowed_txt) {
		imsg("Applying quoted-printable encoding (nhigh=%d/%d : too_wide=%d : sf->flowed_txt=%d)",n_high,n_total,too_wide,sf->flowed_txt);
		f=prcc_mime_encode_qp(sf->data,f,&sf->n_txt);
		sf->enc_txt=1;
	}

	imsg("sformat_prepare_message_textparts_mpa text part done");

	// Now keep that file handle for a little while...
	sf->f_txt=f;

	// If we are sending as text no need to format as html
	if(sf->text_only) {
		fclose(ff); ff=NULL;
		return TRUE;
	}

	// *** OK done with text part now on with HTML part ***
	
	// Convert raw html temp file to stripped ready for sending
	f=sformat_wash_fordisplay_html(sf->data,ff,FALSE,WASH_STRIP_FOR_SEND, NULL);		// This extracts text from html. original ff is closed on exit

	imsg("before sw_update_for_newmpa %d",sf);

	// if html was identified more MPA parts update send_format "attachment_ids" etc
	// bit of a hack, turn into explicit callback or some other nicer construct sometime 
	sw_update_for_newmpa(sf->data, sf);

	imsg("sformat_prepare_message_textparts_mpa html part");

	// Encode suitably for email (short mime encoded strings)
	// count used to determine character set - a bit blunt but safe without having charset detection code (some of the high end chars may not need UTF-8)
	f=prcc_mime_encode_qp(sf->data,f,&sf->n_html);

	// Now keep that file handle for a while...
	sf->f_html=f;

	imsg("sformat_prepare_message_textparts_mpa end");

	return TRUE;
}

int sformat_message_attachments(SFormat *sf, FILE *f2, char *part_separator, char *attach_ids, int *fail_count, char *result)
{
	Wordlist *attachments=NULL;
	char *s, *upto, bf[BFSZ], data[BFSZ], *path, *fname, fullname[BFSZ], *type;
	int n, more,len,base64,wanted=0,got=0, failed=FALSE;
	SWILD_HANDLE *h;
	WILD_DATA mydata;
	FILE *f;

	imsg("sw_format_message_attachments ids:%s",attach_ids);

	// These are the attachments we are supposed to actually send (some may have been uploaded then removed client side)
	ncpy(bf,attach_ids,BFSZ-1);
	for(s = mystrtok(bf,",",&upto); s; s = mystrtok(NULL,",",&upto)){
		if(pair_find_null(sf->cid_index,s)) continue;	// ignore mpr attach QQQS
		if(pair_find_null(sf->cloudfiles,s)) continue;	// SW_CLOUDFILE	SMTP ATTACHMENT DROP

		word_add(&attachments,s);
		wanted++;
	}

	path=pair_find(sf->param,"attach_path");
	more = swild_first(&h, path, &mydata);
	for (;more;more=swild_next(h,&mydata)) {
		lprintf(bf,BFSZ,"%d",atoi(mydata.name));
	
		imsg("trying to attach:%s",mydata.name);
		if (cloudfile_datfile(bf)) continue;	// SW_CLOUDFILE	SMTP ATTACHMENT DROP

		if (!word_find(attachments,bf)) continue;
		if (lib_strlen(mydata.name)<3) continue;
		s=strchr(mydata.name,'_');
		if(s==NULL || *(s+1)=='\0') continue;
		fname=s+1;
		imsg("ok attaching:%s",fname);

		if (fprintf(f2,"\r\n--%s\r\n", part_separator)<0) failed = TRUE;

		if(istrue(pair_find(sf->param,"calendar_invite")) && strcmp(fname,"calendar_invite.ics")==0 ){
			type="text/calendar";
			base64=FALSE;
			if (fprintf(f2,"Content-Type: %s; charset=\"UTF-8\"; method=REQUEST\r\n",type)<0) failed = TRUE;
			if (fprintf(f2,"Content-Transfer-Encoding: %s\r\n",(base64?"base64":"7bit"))<0) failed = TRUE;
		}else{
			type=lib_guess_content(fname);
			base64=(strcmp(type,"message/rfc822")!=0);
			if (fprintf(f2,"Content-Type: %s; name=\"%s\"\r\n",type,fname)<0) failed = TRUE;
			if (fprintf(f2,"Content-Transfer-Encoding: %s\r\n",(base64?"base64":"7bit"))<0) failed = TRUE;
			if (fprintf(f2,"Content-Disposition: attachment; filename=\"%s\"\r\n",fname)<0) failed = TRUE;
		}
		if( fprintf(f2,"\r\n")<0) failed = TRUE;
		lprintf(fullname,BFSZ,"%s%s",path,mydata.name);
		f=fopen(fullname,"rb");
		if(!f){
			emsg("surgeweb: Error reading attachment file [%s]",fullname);
			ncpy(data,"Error reading attached file",BFSZ-1);
			n=encode_base64(data,bf,lib_strlen(data)+1);
			fprintf(f2,"%s\r\n",bf);
			continue;
		}
		for(;!feof(f);){
			len = (int)fread(data,1,57,f);
			if (len<=0) break;
			if(base64){
				n=encode_base64(data,bf,len+1);
				if (fprintf(f2,"%s\r\n",bf)<0) failed = TRUE;
			}else{
				if( fwrite(data,1,len,f2)!=len) failed = TRUE;
			}
		}
		if (fclose(f)<0) failed = TRUE; 
		f=NULL;
		imsg("attaching competed:%s [%s]",mydata.name,fname);
		got++;
	}
	swild_end(&h);
	word_free(&attachments);
	*fail_count += wanted-got;
	if(failed){
		ncpy(result,"FILE WRITE FAILED 4", BFSZ-1);
		return FALSE;
	}

	return TRUE;
}
int sformat_message_relateds(SFormat *sf, FILE *f2, char *part_separator, char *attach_ids, int *fail_count, char *result)
{
	// Add attachments much like the normal attachments adding code
	Wordlist *attachments=NULL;
	char *s, *upto, bf[BFSZ], data[BFSZ], *path, fname[BFSZ], fullname[BFSZ], *type, attachment_id[BFSZ];
	int n, more,len,base64,wanted=0,got=0, failed=FALSE;
	SWILD_HANDLE *h;
	WILD_DATA mydata;
	FILE *f;
	char *item_cid;

	imsg("sformat_message_relateds ids:%s",attach_ids);

	// These are the MPR attachments we are supposed to actually send (some may have been uploaded then removed client side)
	ncpy(bf,attach_ids,BFSZ-1);
	for(s = mystrtok(bf,",",&upto); s; s = mystrtok(NULL,",",&upto)){
		if(!pair_find_null(sf->cid_index,s)) continue;	// ignore standard attach QQQS
		word_add(&attachments,s);
		wanted++;
	}

	path=pair_find(sf->param,"attach_path");
	more = swild_first(&h, path, &mydata);
	for (;more;more=swild_next(h,&mydata)) {
		lprintf(attachment_id,BFSZ,"%d",atoi(mydata.name));
	
		imsg("trying to attach:%s",mydata.name);
		
		if (!word_find(attachments,attachment_id)) continue;

		// id to CID mapping here!!! using sw->cid_index  QQQS
		item_cid = pair_find(sf->cid_index,attachment_id);
		if(!word_find(sf->cids_found, item_cid)){ 
			// mpr item still in attachment list but removed from the html removed so should not get attached
			wanted--;
			continue;
		}

		if (lib_strlen(mydata.name)<3) continue;
		s=strchr(mydata.name,'_');
		if(s==NULL || *(s+1)=='\0') continue;
		sformat_decode_filename(fname,s+1);

		imsg("ok attaching:%s",fname);

		if (fprintf(f2,"\r\n--%s\r\n", part_separator)<0) failed = TRUE;

		// Possibly combine this code with the sw_format_message_attachments version which is much the same

		type=lib_guess_content(fname);
		base64=(strcmp(type,"message/rfc822")!=0);
		if (fprintf(f2,"Content-Type: %s; name=\"%s\"\r\n",type,fname)<0) failed = TRUE;
		if (fprintf(f2,"Content-Transfer-Encoding: %s\r\n",(base64?"base64":"7bit"))<0) failed = TRUE;
		if (fprintf(f2,"Content-ID: <%s>\r\n\r\n",item_cid)<0) failed = TRUE;
		lprintf(fullname,BFSZ,"%s%s",path,mydata.name);
		f=fopen(fullname,"rb");
		if(!f){
			emsg("surgeweb: Error reading attachment file [%s]",fullname);
			ncpy(data,"Error reading attached file",BFSZ-1);
			n=encode_base64(data,bf,lib_strlen(data)+1);
			if(fprintf(f2,"%s\r\n",bf)<0) failed = TRUE;
			continue;
		}
		for(;!feof(f);){
			len = (int)fread(data,1,57,f);
			if (len<=0) break;
			if(base64){
				n=encode_base64(data,bf,len+1);
				if(fprintf(f2,"%s\r\n",bf)<0) failed = TRUE;
			}else{
				if(fwrite(data,1,len,f2)!=len) failed = TRUE;
			}
		}
		fclose(f); f=NULL;
		imsg("attaching competed:%s [%s]",mydata.name,fname);
		got++;
	}
	swild_end(&h);
	word_free(&attachments);
	*fail_count += wanted-got;
	if(failed){
		ncpy(result,"FILE WRITE FAILED 8", BFSZ-1);
		return FALSE;
	}

	return TRUE;
}

// Output the "html message body" and "text equivalent"
int sformat_message_htmlpart(SFormat *sf, FILE *f2, char *separator, char *result)
{
	char bf[BFSZ];
	int failed = FALSE;

	if (fprintf(f2,"--%s\r\n",separator)<0) failed = TRUE;
	if (fprintf(f2,"Content-Type: text/html; charset=%s\r\n",sf->n_html==0?"us-ascii":"utf-8")<0) failed = TRUE;
	if (fprintf(f2,"Content-Transfer-Encoding: quoted-printable\r\n\r\n")<0) failed = TRUE;

	// Send actual html part
	if (fprintf(f2,"<html>\r\n<head>\r\n<style>\r\n")<0) failed = TRUE;
	if (fprintf(f2," .sw_message P{margin:0px;padding:0px;}\r\n")<0) failed = TRUE;
	if (fprintf(f2," .sw_message {FONT-SIZE: %spt;FONT-FAMILY:Tahoma,Arial,Helvetica,sans-serif;background:white;}\r\n",pair_find(sf->param,"font_size"))<0) failed = TRUE;

	if(!istrue(pair_find(sf->param,"send_no_swformat"))){
		if (fprintf(f2," .sw_message blockquote{margin-left:5px;padding-left:5px;border-left:2px solid #144fae;color: #144fae;}\r\n")<0) failed = TRUE;
		if (fprintf(f2," .sw_message blockquote blockquote{border-left:2px solid #006312;color: #006312;}\r\n")<0) failed = TRUE;
		if (fprintf(f2," .sw_message blockquote blockquote blockquote{border-left:2px solid #8e5656;color: #8e5656;}\r\n")<0) failed = TRUE;
		if (fprintf(f2," .sw_message blockquote blockquote blockquote blockquote{border-left:2px solid #888;color: #888;}\r\n")<0) failed = TRUE;
	}
	if (fprintf(f2,"</style>\r\n</head>\r\n")<0) failed = TRUE;
	// WARNING: do not put '</style>'  and '</head>' on the same output line. If you do,
	//  old NetWin webmail will eat the rest of the message in its washing and display nothing at all

	// =3d is "=" encoded
	if (fprintf(f2,"<body class=3d\"sw_message\">\r\n")<0) failed = TRUE;
	for (;!feof(sf->f_html);) {
		if (!fgets(bf,BFSZ-1,sf->f_html)) break;
		if (fprintf(f2,"%s",bf)<0) failed = TRUE;
	}
	if (fclose(sf->f_html)<0) failed = TRUE;
	sf->f_html=NULL;
	if (fprintf(f2,"</body></html>\r\n")<0) failed = TRUE;

	if(failed){
		ncpy(result,"FILE WRITE FAILED 7", BFSZ-1);
		return FALSE;
	}

	return TRUE;
}


int sformat_message_attachments_files(SFormat *sf, FILE *f2, char *part_separator, char *attach_files, int *fail_count)
{
	char *upto, *s, names[BFSZ*10], bf[BFSZ], data[BFSZ], path[BFSZ], fname[BFSZ], fullname[BFSZ], *type;
	int n, len,base64,wanted=0,got=0;
	FILE *f;
    
	imsg("sw_format_message_attachments :%s",attach_files);
    
	// These are the attachments we are supposed to actually send (some may have been uploaded then removed client side)
	ncpy(names,attach_files,BFSZ*10-1);
	for(s = mystrtok(names,",",&upto); s; s = mystrtok(NULL,",",&upto)){

        lib_path_split(s, path, fname);
    
		imsg("trying to attach:%s",fname);
		if (lib_strlen(fname)<3) continue;
        
		fprintf(f2,"\r\n--%s\r\n", part_separator);
        
		type=lib_guess_content(fname);
		base64=(strcmp(type,"message/rfc822")!=0);
		fprintf(f2,"Content-Type: %s; name=\"%s\"\r\n",type,fname);
		fprintf(f2,"Content-Transfer-Encoding: %s\r\n",(base64?"base64":"7bit"));
		fprintf(f2,"Content-Disposition: attachment; filename=\"%s\"\r\n\r\n",fname);
        
        ncpy(fullname, s, BFSZ-1);
		f=fopen(fullname,"rb");
		if(!f){
			emsg("surgeweb: Error reading attachment file [%s]",fullname);
			ncpy(data,"Error reading attached file",BFSZ-1);
			n=encode_base64(data,bf,lib_strlen(data)+1);
			fprintf(f2,"%s\r\n",bf);
			continue;
		}
		for(;!feof(f);){
			len = (int)fread(data,1,57,f);
			if (len<=0) break;
			if(base64){
				n=encode_base64(data,bf,len+1);
				fprintf(f2,"%s\r\n",bf);
			}else{
				fwrite(data,1,len,f2);
			}
		}
		fclose(f); f=NULL;
		imsg("attaching competed:[%s]",fname);
		got++;
	}
	*fail_count += wanted-got;
	return TRUE;
}

// SW_CLOUDFILE	BODY MODIFICATION
void sf_output_bigfiles(SFormat *sf, FILE *f, int update_top)
{
	Pair *cloudfile=NULL;
	Wordlist *attachments=NULL;
	char *s=NULL, *upto=NULL, bf[BFSZ];

	if (update_top && !sf->cloudfile_top) return;
	if (!update_top && sf->cloudfile_top) return;


	// These are the attachments we are supposed to actually send - some may have been uploaded then removed client side, 
	// and some may not be cloud attachments
	ncpy(bf,sf->attach_ids,BFSZ-1);
	for(s = mystrtok(bf,",",&upto); s; s = mystrtok(NULL,",",&upto)){
		word_add(&attachments,s);
	}

	for(cloudfile=sf->cloudfiles; cloudfile; cloudfile=cloudfile->next){
		int expire=0, size=0;
		char filename[BFSZ], link[BFSZ], tmp[BFSZ];
		cloudfile_decode(cloudfile->val, &expire, &size, filename, link, BFSZ-1);
		
		if (!word_find(attachments, cloudfile->var)) 
			continue;	// Already removed from email clientside

		fprintf(f,"<br>");
		fprintf(f,"<b>Cloud Attachment:</b> <br>");
		fprintf(f," File: %s<br>", filename);
		fprintf(f," Size: %s<br>", lib_bytes_string_basic(size,tmp,BFSZ-1,0));
		fprintf(f," Link: %s<br>", link);
		fprintf(f," Expires: %d days<br>", expire);
		fprintf(f,"<br>");
	}

	word_free(&attachments);
}