
#ifdef __XCODE__
#include "xcode_quiet.h"
#endif

#include <errno.h>
#include <string.h>
#include <stdio.h>

#include "lib.h"
#include "parts.h"
#include "imaplib.h"
#include "lprintf.h"
#include "decode.h"
#include "miconv.h"
#define DMID 14
#include "debug.h"
#include "thread.h"
#include "rstring.h"
#include "mime.h"

#include "vsnprintf.h"


#ifndef FALSE
#define FALSE 0
#endif

#ifndef TRUE
#define TRUE 1
#endif

extern int emsg(const char * arg_list, ...);
extern int imsg(const char * arg_list, ...);
extern int dmsg(const char * arg_list, ...);

void im_errno_error(ImapCon *i);
void im_tcp_error(ImapCon *i);
void im_reset_cb(ImapCon *i);

/************/

ImapCon *im_new()
{
	ImapCon *i = (ImapCon *)calloc(1,sizeof(ImapCon));
	i->result = i->smallbf;
    i->exiting = FALSE;
	i->outbf = dstr_plain();
	return i;
}
void im_free(ImapCon **i)
{
	if (!i || !*i) return;
	im_logout(*i);
	dstr_free(&(*i)->outbf);
	ffree((*i)->user);
	ffree((*i)->bigbf);
	free(*i);
	*i = NULL;
}
// Make sure our formatting buffer is large enough for any number of uid's 
// and any additional parameters like large folder names
void im_outbf_checksize(ImapCon *i, char *uidset)
{
	int desired_size;
	if(i->outbf==NULL) return;
	desired_size = (int)strlen(uidset) + 512;
	dstr_big_enough(i->outbf, desired_size);
}

void im_exit(ImapCon *i)
{
    if(!i) return;
    i->exiting = TRUE;
    i->stop_idle = TRUE;
}

void im_set_cb(ImapCon *i, void (*cb)(void*,ImapCon*), void *data)
{
	i->data = data;
	i->cb = cb;
}

int im_connect(ImapCon *i, char *host, int port)
{
	if (!i) return FALSE;
	if (i->c) im_disconnect(i);
	tcp_new(&i->c);
	if (!tcp_open(i->c,host,port,NULL)) {
		im_tcp_error(i);
		return FALSE;
	}
	return TRUE;
}
void im_disconnect(ImapCon *i)
{
	if (!i) return ;
	if (!i->c) return ;
	tcp_close_nice(i->c);
	tcp_free(&i->c);
}

int im_login(ImapCon *i, char *user, char *pass)
{
	char tmp[BFSZ];
	if (!i) return FALSE;
	if (i->user) {
		if (strcmpnc(i->user,user)==0) return TRUE;
		free(i->user);
	}
	if (!im_command(i,TRUE,"LOGIN %s \"%s\"",user,lib_slash_encode(pass,tmp))) return FALSE;
//	if (!im_command(i,TRUE,"xlogin_local %s %s",user,pass)) return FALSE;
	i->loggedin = TRUE;
	i->user = (char *)strdup(user);
	return TRUE;
}
int im_login_sm(ImapCon *i, char *user, char *pass)
{
	if (!i) return FALSE;
	if (i->user) {
		if (strcmpnc(i->user,user)==0) return TRUE;
		free(i->user);
	}
	if (!im_command(i,TRUE,"xlogin_local %s \"%s\"",user,pass)) return FALSE;
	i->loggedin = TRUE;
	i->user = (char *)strdup(user);
	return TRUE;
}
int im_logout(ImapCon *i)
{
	if (!i) return FALSE;
	if (!i->c) return TRUE;
	im_command(i,TRUE,"LOGOUT");
	im_disconnect(i);	
	if(i->log) fclose(i->log);
	return TRUE;
}

int im_command(ImapCon *i, int read_response, char *format, ...)
{
	char *bf,tstr[BFSZ];
	va_list args;
	int x, bfsz;

	if (!i) return FALSE;
	if (!i->c) return FALSE;

	bf=dstr_ss(i->outbf);
	bfsz = dstr_bfsz(i->outbf);

	va_start(args,format);
	x = vsnprintf(bf,bfsz,format,args);
	if (x == -1 || x == bfsz) bf[(x = bfsz-1)] = 0;

	if(i->log){
		lib_emsg_time(tstr);
		fprintf(i->log,"%s:CLIENT:%d(%d){",tstr,th_id(),x);
		fwrite(bf,1,x,i->log);	// Allow for full debug logging
		fprintf(i->log,"}\n");
	}
	if(i->imsg) {
		i->imsg("IMAP:CLIENT:(%d){%s}",x,bf);		
	}
	if (!im_send_command(i,bf,bfsz,x)) return FALSE;
	if (read_response) {
		if (!im_read_response(i)) return FALSE;
		if (!im_ok(i)) return FALSE;
	}

	return TRUE;
}

int im_capability(ImapCon *i)
{
	return im_command(i,TRUE,"CAPABILITY");
}

int im_select(ImapCon *i, char *mbx)
{
	return im_command(i,TRUE,"SELECT \"%s\"",mbx);
}
int im_examine(ImapCon *i, char *mbx)
{
	return im_command(i,TRUE,"EXAMINE \"%s\"",mbx);
}
int im_create(ImapCon *i, char *mbx)
{
	return im_command(i,TRUE,"CREATE \"%s\"",mbx);
}
int im_delete(ImapCon *i, char *mbx)
{
	return im_command(i,TRUE,"DELETE \"%s\"",mbx);
}
int im_rename(ImapCon *i, char *ombx, char *nmbx)
{
	return im_command(i,TRUE,"RENAME \"%s\" \"%s\"",ombx,nmbx);
}
int im_subscribe(ImapCon *i, char *mbx)
{
	return im_command(i,TRUE,"SUBSCRIBE \"%s\"",mbx);
}
int im_unsubscribe(ImapCon *i, char *mbx)
{
	return im_command(i,TRUE,"UNSUBSCRIBE \"%s\"",mbx);
}

int im_list(ImapCon *i, char *ref, char *mbx)
{
	return im_command(i,TRUE,"LIST \"%s\" \"%s\"",ref,mbx);
}
int im_lsub(ImapCon *i, char *ref, char *mbx)
{
	return im_command(i,TRUE,"LSUB \"%s\" \"%s\"",ref,mbx);
}

int im_status(ImapCon *i, char *mbx, char *itms)
{
	return im_command(i,TRUE,"STATUS \"%s\" %s",mbx,itms);
}

int im_append(ImapCon *i, char *mbx, char *flags, char *date, int length, int(*cb)(void*,ImapCon*), void *data)
{
	if (!im_command(i,FALSE,"APPEND \"%s\" (%s)%s%s {%d}",
		mbx,
		flags?flags:"",
		date?" ":"",
		date?date:"",
		length)) return FALSE;

	if (!im_read_response(i)) return FALSE; //+ Ready for argument
	while(cb(data,i)) {}
	tcp_printf(i->c,"\r\n"); //write extra blank line
	if (!im_read_response(i)) return FALSE;
	if (!im_ok(i)) return FALSE;
	return TRUE;
}

int im_check(ImapCon *i)
{
	return im_command(i,TRUE,"CHECK");
}
int im_close(ImapCon *i)
{
	return im_command(i,TRUE,"CLOSE");
}
int im_expunge(ImapCon *i)
{
	return im_command(i,TRUE,"EXPUNGE");
}

int im_search(ImapCon *i, char *charset, char *criteria)
{
	return im_command(i,TRUE,"SEARCH %s%s%s",
		charset?charset:"",
		charset?" ":"",
		criteria);
}
int im_uid_search(ImapCon *i, char *charset, char *criteria)
{
	return im_command(i,TRUE,"UID SEARCH %s%s%s%s",
		charset?"CHARSET ":"",
		charset?charset:"",
		charset?" ":"",
		criteria);
}

int im_fetch(ImapCon *i, char *msgset, char *items)
{
	im_outbf_checksize(i,msgset);
	return im_command(i,TRUE,"FETCH %s %s",msgset,items);
}
int im_store(ImapCon *i, char *msgset, char *item, char *value)
{
	im_outbf_checksize(i,msgset);
	return im_command(i,TRUE,"STORE %s %s %s",msgset,item,value);
}
int im_copy(ImapCon *i, char *msgset, char *mbx)
{
	im_outbf_checksize(i,msgset);
	return im_command(i,TRUE,"COPY %s \"%s\"",msgset,mbx);
}

int im_uid_fetch(ImapCon *i, char *uidset, char *items)
{
	im_outbf_checksize(i,uidset);
	return im_command(i,TRUE,"UID FETCH %s %s",uidset,items);
}
int im_uid_store(ImapCon *i, char *uidset, char *item, char *value)
{
	im_outbf_checksize(i,uidset);
	return im_command(i,TRUE,"UID STORE %s %s (%s)",uidset,item,value);
}
int im_uid_copy(ImapCon *i, char *uidset, char *mbx)
{
	im_outbf_checksize(i,uidset);
	return im_command(i,TRUE,"UID COPY %s \"%s\"",uidset,mbx);
}
// Synchronous implementation, also see non synchornous implementation further down file
int im_idle(ImapCon *i) 
{	
	int ret;
	i->idling=TRUE;
	ret=im_command(i,TRUE,"IDLE");
	i->idling=FALSE;
	return ret;
}

int im_rawcmd(ImapCon *i, char *cmd)
{
	return im_command(i,TRUE,"%s",cmd);
}
// cf im_append
int im_rawcmd_large(ImapCon *i, char *cmd, int(*cb)(void*,ImapCon*), void *data)
{
	if (!im_command(i,FALSE,"%s",cmd)) return FALSE;
	if (!im_read_response(i)) return FALSE; //+ Ready for argument
	if (cb && data){
        tcp_buffer_set(i->c,TRUE);
		while(cb(data,i)) {}
        tcp_buffer_set(i->c,FALSE);
        tcp_printf(i->c,"\r\n"); //write extra blank line
		if (!im_read_response(i)) return FALSE;
	}
	if (!im_ok(i)) return FALSE;
	return TRUE;
}


/************/

char *im_tag(ImapCon *i)
{	
	if (!i) return NULL;
	lprintf(i->tag,TAGSZ,"%c%03d",'A'+(i->itag/1000)%26,i->itag%1000);	
	i->itag++;
	return i->tag;
}

int im_send_command(ImapCon *i, char *bf, int bfsz, int x)
{	
	int ret;
    if (!i) return FALSE;

	/* Ensure exactly one \r\n on command */
	for(; x > 0 && iseol(bf[x-1]); x--) bf[x-1] = '\0';
	if (x > bfsz-3) x = bfsz-3;
	strcpy(&bf[x],"\r\n");
	x += 2;

	/* Send tag, space, command */
	tcp_buffer_set(i->c,TRUE);
	ret = (tcp_write(i->c,im_tag(i),TAGSZ-1) == TAGSZ-1 &&
           tcp_write(i->c," ",1) &&
           tcp_write(i->c,bf,x) == x);

    tcp_buffer_set(i->c,FALSE);
    if(!ret){
        im_tcp_error(i);
        return FALSE;	
    }
	
    return TRUE;	
}

int im_log_server(ImapCon *i)
{
	char tstr[BFSZ];
	if(i->log){
		lib_emsg_time(tstr);
		fprintf(i->log,"%s:SERVER:%d(%d){",tstr,th_id(),(int) lib_strlen(i->result));
		fwrite(i->result,1,lib_strlen(i->result),i->log);	// Allow for full debug logging
		fprintf(i->log,"}\n");
	}
	if(i->imsg) {
		i->imsg("IMAP:SERVER:(%d){%s}",lib_strlen(i->result),i->result);		
	}
	
	return TRUE;
}

void im_logimsg_init(ImapCon *i, int (*fn)(const char *, ...))
{
	i->imsg = fn;
}

int im_expect_long(ImapCon *i, int expect_long)
{
	if(i->expect_long==expect_long) return TRUE;
	
	if(expect_long){
		i->expect_long=TRUE;
	}else {
		i->expect_long=FALSE;
		i->result = i->smallbf;
		ffree(i->bigbf);
		i->ptr=NULL;
	}

	return TRUE;
}
/*
There are  three possible server completion responses: OK (indicating success),
NO (indicating failure), or BAD (indicating protocol error such as unrecognized 
command or command syntax error).
*/
int im_read_response(ImapCon *i)
{
	int res = FALSE, r;

	if (!i) return FALSE;
	if(i->idling) return im_idle_response(i);
    
//    imsg("awaiting imap response B4");
	
	for (;;/*i->ptr = NULL; tcp_readline(i->c,i->result,BFSZ); i->ptr = NULL*/) {
		i->ptr = NULL;
		if (i->expect_long) {
			ffree(i->bigbf);
			r=tcp_readline_big(i->c, &i->bigbf, 100000,0);
			i->result=i->bigbf;
		}else {
			r=tcp_readline(i->c,i->result,BFSZ);
		}
		if(!r) break;
		
		im_log_server(i);
		if (im_bad(i)) { res = TRUE; break; }
		if (im_end(i) || im_plus(i)) { res = TRUE; break; }
		if (i->cb) i->cb(i->data,i);
	}
    //imsg("awaiting imap response Af");
	im_reset_cb(i);
	if (!res) im_tcp_error(i);
	return res;
}
int im_end(ImapCon *i)
{
	if (!i) return FALSE;
	return strncmp(i->result,i->tag,TAGSZ-1) == 0;
}
int im_ok(ImapCon *i)
{
	if (!i) return FALSE;
	return strncmp(&i->result[TAGSZ],"OK ",3) == 0;
}
int im_no(ImapCon *i)
{
	if (!i) return FALSE;
	return strncmp(&i->result[TAGSZ],"NO ",3) == 0;
}
int im_bad(ImapCon *i)
{
	if (!i) return FALSE;
	return strncmp(&i->result[TAGSZ],"BAD ",4) == 0;
}
int im_err(ImapCon *i)
{
	if (!i) return TRUE;
	return strncmp(&i->result[TAGSZ],"ERR ",4) == 0;
}
int im_plus(ImapCon *i)
{
	if (!i) return FALSE;
	return strncmp(i->result,"+ ",2) == 0;
}
int im_star(ImapCon *i)
{
	if (!i) return FALSE;
	return strncmp(i->result,"* ",2) == 0;
}
int im_idle_response(ImapCon *i)
{
	int res = FALSE, len, count=0;
	int ignore_done=FALSE;
    int sleep_detect=0, net_detect=0, detect_dropout=0;
	time_t end_idle=0;
	char *done="DONE\r\n";

	if(!i->idling) return TRUE;

	// + idling
	if(!tcp_readline(i->c,i->result,BFSZ)) goto done;
	im_log_server(i);
	if(!im_plus(i)) { res = TRUE; goto done; }

	sleep_detect = int_time(NULL);
	net_detect = int_time(NULL);
	// loop, trying to read idle line
	for(;;){
        if(count++>50){
            imsg("imap idle loop on thread=%d", th_id());
            count=0;
        }
		lib_msleep(200);
        
        // Make our channel detect various forms of failures...
        // - Wake from sleep / hibernate - assume channel is no longer valid, dropout immediately and reconnect 
        if((time(NULL)-sleep_detect)>20){
            i->dropout_type = IMAP_DROPOUT_SLEEP;
            tcp_close_nice(i->c);
            break;
        }
        sleep_detect=int_time(NULL);
        //  - heartbeat of momentarily dropping out of idle loop
        if((time(NULL)-net_detect)>60*2){
            i->stop_idle = TRUE;
            i->dropout_type = IMAP_DROPOUT_HEARTBEAT;
            if(end_idle==0) end_idle = time(NULL);
        }
        if (end_idle && time(NULL)-end_idle>10){
            // idle "done" dropout will not responding, something must be wrong on our channel...
            tcp_close_nice(i->c);
            break;
        }        

		// Drop out of idle loop if required for whatever reason
		if(i->stop_idle && !i->stop_idle_pending){
			len=lib_strlen(done);
			if (tcp_write(i->c,done,len) != len) break;
			i->stop_idle_pending=TRUE;
		}
        
		if(!tcp_wait_read_buf(i->c,0)) continue;

		if(!tcp_readline(i->c,i->result,BFSZ)) break;
		im_log_server(i);

		if (im_bad(i)) { res = TRUE; break; }
		if (im_end(i) || im_plus(i)) { res = TRUE; break; }
		if (!im_star(i)) { res = TRUE; break; }

		if (i->cb) i->cb(i->data,i);
	}
done:
    i->stop_idle_pending=FALSE;
    i->stop_idle=FALSE;
    
	if (!res) im_tcp_error(i);
	return res;
}

// Asynchronous idle implementation
int im_idle_setup(ImapCon *i)
{
//    last_idle = time(NULL);
	if(!im_command(i,FALSE,"IDLE")) return FALSE;
	if(!tcp_readline(i->c,i->result,BFSZ)) return FALSE;
	im_log_server(i);
	if(!im_plus(i)) return FALSE;
    i->idling2=TRUE;
    imsg("IMAP: Idle setup completed");
    i->last_idle_tick = i->last_idle_response = (int)time(NULL);
    i->stop_idle_pending = FALSE;

    return TRUE;
}

int im_idle_tick(ImapCon *i, int *data_ready)
{
    if(tcp_wait_read_buf(i->c,0)){
        *data_ready = TRUE;// something to read
    } else{
        *data_ready = FALSE;
        if(!im_idle_fix(i)) 
            return FALSE;
    }
        
    return TRUE;  
}
int im_idle_fix(ImapCon *i)
{
    long now = int_time(NULL);
    if (!i->idling2) return TRUE;
    if (now - i->last_idle_tick > 15) {
        imsg("imap idle sleep detect hit, reconnecting");
        i->exiting = TRUE;
        i->dropout_type = IMAP_DROPOUT_SLEEP;
        goto error;
    }
    i->last_idle_tick = now;
    if (now - i->last_idle_response > 30) {
        imsg("imap idle heartbeat timeout, done idle");
        
        if(!im_idle_exit(i)) goto error;
        if(!tcp_readline(i->c,i->result,BFSZ)) goto error;

        imsg("imap idle heartbeat timeout, start idle again");
        
        if(!im_idle_setup(i)) goto error; // go back to idle mode
    }
    return TRUE;
    
error:
    return FALSE;
}
int im_idle_exit(ImapCon *i)
{
    int len;
	char *done="DONE\r\n";
    
    if(!i) return TRUE;
    if(i->stop_idle_pending) return TRUE;
    if(!i->idling2) return FALSE;
    
    len=lib_strlen(done);
    if (tcp_write(i->c,done,len) != len) return FALSE;
    i->stop_idle_pending = TRUE;
    i->stop_idle = TRUE;

//    string result;
//    if (run_state != r.R_IDLE)
//    {
//        stat.imsg("IMAP: state was wrong, it was {0}", run_state);
//        return false; // wrong state to exit... 
//    }
//    //netprintf("done\r\n"); 
//    send("done\r\n"); // we don't use netprintf as the id is not sent/updated.
//    if (!response(out result,false,30000))
//    {
//        stat.imsg("IMAP: response wrong [{0}]", result);
//        netclose(); // close the socket if we get a bumb response from imap...
//        return false;
//    }
//    stat.imsg("Idle_exit complted {0}", result);
//    return true;
	return TRUE;
}






static char mutf7_base64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,";
int im_decode_folder_old(char *folder, char *res, int bfsz)
{
	int rval = FALSE;
	Miconv *i = NULL;
	char *s = folder;
	char *r = res;
	char tmp[BFSZ];
	int t;
	char *p;
	int val;
	int k;

	for(; *s && r-res < bfsz-1; s++) {
		if (*s != '&') { // copy across everything that doesn't start with & 
			*r++ = *s;
			continue;
		}
		
		s++;
		
		//Escaped &, skip -
		if (*s == '-') {			
			*r++ = '&';
			continue;
		}

		if (!i) i = miconv_new("ASCII");
		if (!miconv_set(i,"UCS-2BE")) {
			emsg("miconv_set: %s",strerror(errno));
			goto err;
		}

		t = 0;
		while(*s != '-') {  // Until we get to the ending '-'
			if (!*s) goto err; // Or end of string.

			for(val = k = 0; k <= 3 && *s != '-'; k++) {
				if (!*s) goto err;

				p = strchr(mutf7_base64,*s++);
				if (!p) goto err;
				val += (int) (p-mutf7_base64) << (3-k)*6;
			}

			for(k = 2; k >= 0; k--) {
				// chrisp, probably don't need BFSZ check...
				if (t>=BFSZ-1) break;
				if (t>=bfsz-1) break;
				tmp[t++] = val & 0xff;
				val >>= 8;
			}
		}

		if (miconv_direct(i,tmp,t-1,&r,BFSZ-(r-res)-1) == 0) {
			emsg("miconv_direct: %s",strerror(errno));
			goto err;
		}
	}
	rval = TRUE;
err:
	if (i) miconv_free(&i);	
	if (!rval) ncpy(res,folder,bfsz-1);
	else *r = 0;
	return rval;
}

#define need_encode_bad(x) (x && (x == '&' || !sisprint(x)))
// Marijn: This function is broken in several ways but leave alone...
int im_encode_folder_old(unsigned char *folder, char *res, int bfsz)
{
	char tmp[BFSZ];	
	int i,x;

	for(i = 0; i < bfsz-1 && *folder; ) {
		if (!need_encode_bad(*folder)) res[i++] = *folder++;
		else {
			//convert from ASCII to UCS-2BE (simple, iconv not required)
			for(x = 0; x < BFSZ-2 && need_encode_bad(*folder); ) {
				tmp[x++] = 0;
				tmp[x++] = *folder++;
			}
			tmp[x] = 0;
			res[i++] = '&';
			if (i+(x*2)+1 >= bfsz) break;
			i += encode_base64(tmp,&res[i],x+1);
			//replace = with -
			while(res[i-1] == '=') { res[i-1] = '-'; i--; }
			if (res[i-1] != '-') res[i++] = '-';
			res[i] = 0;
		}
	}
	res[i] = 0;

	return TRUE;
}

// utf8 <-> modified utf7 conversion : utf8 for browser; modified utf7 for imap folder names
#define need_encode(x) (x && !sisprint(x))
int im_encode_folder(unsigned char *s, char *res, int bfsz)		// utf8_to_mutf7
{
	char tmp[BFSZ],bf[BFSZ],*pbf=bf,*p;	
	int i,x,rval=FALSE;
	Miconv *ic = NULL;

	for(i = 0; i < bfsz-5 && *s; ) {		// Any single characters should be no larger that 4 bytes so 5 is safe
		if (!need_encode(*s)){
			res[i++] = *s;
			if(*s=='&') 
				res[i++]='-';
			s++;
		}else {
			// Get just the chunk of string that needs encoding
			for(x = 0; x < BFSZ-1 && need_encode(*s); ) {
				tmp[x++] = *s++;
			}
			tmp[x] = 0;
			
			// UTF8 to 16bit unicode conversion
			if (!ic){
				ic = miconv_new("UCS-2BE");							// TO ...
				if (!miconv_set(ic,"UTF-8")) {	// ... FROM
					emsg("miconv_set: %s",strerror(errno));
					goto err;
				}
			}
			pbf = bf;	// <<< This line Added 26/7/2016 by marijn to correctly imap encode utf8 words separated by non utf8 characters (eg spaces)
			x=miconv_direct(ic,tmp,lib_strlen(tmp),&pbf,BFSZ-1);
			if (x==0) {	
				emsg("miconv_direct: failed on %s Error: %s",s, strerror(errno));
				goto err;
			}
			// NOW we have the string as UNICODE in bf...

			res[i++] = '&';
			if (i+(x*2)+1 >= bfsz) break;
			p=&res[i];
			i += encode_base64(bf,&res[i],x+1);
			//trim any '=' and close with a '-'
			while(res[i-1] == '=') { res[i-1] = '-'; i--; }
			if (res[i-1] != '-') res[i++] = '-';
			res[i] = 0;
			
			for(;*p;p++) // Modified UTF7 uses ',' instead of '/' tweak here so we can use generic encode_base64
				if(*p=='/') 
					*p=',';
		}
	}
	res[i] = 0;
	rval=TRUE;
err:
	if (ic) miconv_free(&ic);
	return rval;
}

// Convert imap modified UTF7 to UTF8
int im_decode_folder(char *folder, char *res, int bfsz){	// mutf7_to_utf8
	int rval = FALSE;
	Miconv *ic = NULL;
	char *s = folder;
	char *r = res;
	char tmp[BFSZ];
	int t;
	char *p;
	int val;
	int b,i,k;

	for(; *s && r-res < bfsz-1; s++) {
		
		if (*s != '&') {		// copy all unencoded characters ie. everything that doesn't start with '&'
			*r++ = *s;
			continue;
		}
		s++;
		
		if (*s == '-') {		//Escaped '&' so skip '-'
			*r++ = '&';
			continue;
		}

		if (!ic){
			ic = miconv_new("UTF-8");			// Each encoded chunk to UTF8,
			if (!miconv_set(ic,"UCS-2BE")) {	//  from 16 bit unicode
				emsg("miconv_set: %s",strerror(errno));
				goto err;
			}
		}
		t = 0;
		while(*s != '-') {						// Decode until we get to the ending '-'
			if (!*s) goto err;					//   or end of string.

			for(val = k = 0; k <= 3 && *s != '-'; k++) {	// process up to 4 characters at a time
				if (!*s) goto err;

				p = strchr(mutf7_base64,*s++);
				if (!p) goto err;
				b = (int)(p - mutf7_base64);
				val += (b) << (3-k)*6;
			}
	
			for(i = 2; i >= 0; i--) {						// output those - ie upto 3 decoded characters
				if (t>=bfsz-1) goto err;
				tmp[t+i] = val & 0xff;
				val >>= 8;
			}

			// Marijn: Following logic replaced 19/8/2020 as has been broken for 11 years
			if(k<=4 && k>=2){
				t += k-1;	// Source character (6bit) count to destination character (8bit) count
			}else if(k==1){
				// Should not be possible I think??
				emsg("miconv_direct: decode failed on %s",folder);
				goto err;
			}
		}
		if (miconv_direct(ic,tmp,t,&r,BFSZ-1) == 0) {	// Now convert 16bit unicode to UTF8
			emsg("miconv_direct: failed on %s Error: %s",folder, strerror(errno));
			goto err;
		}
	}
	if (*s) goto err;
	rval = TRUE;
err:
	*r = 0;
	if (ic) miconv_free(&ic);
	return rval;
}


int im_parse(ImapCon *i, char *res, int bfsz)
{
	char *s = i->ptr;

	if (!s) s = i->ptr = i->result;
	*res = 0;

	while(sisspace(*s)) s++;
	if (*s == '\0') return FALSE;

	if (*s == '(' || *s == ')') { res[0] = *s++; res[1] = '\0'; }
	else if (*s == '\"') {
		s++;
		lib_copy_upto(&s,'\"',res,bfsz);
		if (*s != '\"') return FALSE;
		s++;
	}
	else if(lib_starts_with(s,"name=\"")){
		// support brackets in quoted strings eg name="default (2).abk"
		ncpy(res,s,6);
		s+=6;		
		lib_copy_upto(&s,'\"',res+6,bfsz-6);
		if(*s == '\"') {
			ncat(res,"\"",bfsz-1);
			s++;
		}
	}else{
		lib_copy_upto_any(&s," )",res,bfsz);
	}

	i->ptr = s;
	return TRUE;
}

char *im_rest(ImapCon *i)
{
	char *s = i->ptr;
	if (!s) s = i->result;
	i->ptr = NULL;
	return s;
}

/*
* 1 FETCH (UID 1 FLAGS (\Seen) INTERNALDATE "16-Feb-2006 09:58:46 +1300" RFC822.SIZE 213 BODY[HEADER] {207}
From: test@ally
Subject: Bulletin message from domain administrator
X-DPOP-Domains: ally
X-Bulletin: Bulletin message
Date: Tue, 14 Feb 2006 10:59:48 +1300
Content-Type: text/plain; charset="koi8-r"

)
*/
int im_parse_fetch(ImapCon *i, char *item, char *value, int bfsz)
{
	if (!i->ptr) while(im_parse(i,item,bfsz) && strcmp(item,"(")!=0) {}
	if (!im_parse(i,item,bfsz)) return FALSE;
	if (!im_parse(i,value,bfsz)) return FALSE;
	if (strcmp(value,"(")==0) {
		char tmp[BFSZ];

		*value = 0;
		while(im_parse(i,tmp,BFSZ)) {
			if (strcmp(tmp,")")==0) return TRUE;
			if (value[0]) ncat(value," ",bfsz-1);
			ncat(value,tmp,bfsz-1);
		}

		return FALSE;
	}
	return TRUE;
}
int im_parse_append_uid(ImapCon *i, char *dummy, char *uid, int bfsz)
{
	char *s;
	if (!i->ptr) while(im_parse(i,dummy,bfsz) && strcmp(dummy,"[APPENDUID")!=0) {}
	if (!im_parse(i,dummy,bfsz)) return FALSE;
	if (!im_parse(i,uid,bfsz)) return FALSE;
	s=strchr(uid,']');
	if (!s) return FALSE;
	*s='\0';
	return TRUE;
}
int im_parse_copy_uid(ImapCon *i, char *new_uids, int bfsz)
{
	char *s,bf[BFSZ];
	if (!new_uids) return TRUE;
	if (!i->ptr) while(im_parse(i,bf,bfsz) && strcmp(bf,"[COPYUID")!=0) {}
	if (!im_parse(i,bf,bfsz)) return FALSE;
	if (!im_parse(i,bf,bfsz)) return FALSE;
	if (!im_parse(i,new_uids,bfsz)) return FALSE;
	s=strchr(new_uids,']');
	if (!s) return FALSE;
	*s='\0';
	return TRUE;
}


int im_parse_flags(char *string)
{
	char extra[BFSZ];
	return im_parse_flags_real(string,extra);
}
int im_parse_flags_real(char *string, char *extra)
{		
	int flags = 0;
	char *s, *p=extra, *last_p, bf[BFSZ];
	
	s = string;
	if (*s == '(') s++;

	while(*s) {		
		while(sisspace(*s)) s++;  // find first flag
		if (*s == ')') break;
		if (strncmp(s,"\\Answered",9)==0) {flags |= FLAGS_ANSWERED; s += 9;}
		else if (strncmp(s,"\\Deleted",8)==0) {flags |= FLAGS_DELETED; s += 8;}
		else if (strncmp(s,"\\Flagged",8)==0) {flags |= FLAGS_FLAGGED; s += 8;}
		else if (strncmp(s,"\\Seen",5)==0) {flags |= FLAGS_SEEN; s += 5;}
		else if (strncmp(s,"\\Draft",6)==0) {flags |= FLAGS_DRAFT; s += 6;}
		else if (strncmp(s,"\\Recent",7)==0) {flags |= FLAGS_RECENT; s += 7;}
		else if (strncmp(s,"\\NoInferiors",12)==0) {flags |= FLAGS_NOINFERIORS; s += 12;}		
		else if (strncmp(s,"$Forwarded",10)==0) {flags |= FLAGS_FORWARDED; s += 10;}		
		else {
			last_p=p;
			while(*s && !sisspace(*s) && *s != ')') { // read till end of flag adding to 'extras...
				if(p-extra<BFSZ-1) *p++=*s;
				s++;
			}
			if(p!=last_p) *p++=' '; // if we got something add a space. 
			if (p-last_p-1>1) {
				ncpy(bf,last_p,(int)(p-last_p-1)); // if we got nothing blow up ? 
				//imsg("im_parse_flags: custom flag (%s)",bf); // log the thing we got... 
			}
		}
	}
	if(p!=extra && *(p-1)==' ') p--;
	*p='\0';

	return flags;
}

char *im_make_flags(int flags, char *bf, int bfsz)
{
	*bf = 0;
	if (flags & FLAGS_ANSWERED) {if (bf[0]) ncat(bf," ",bfsz-1); ncat(bf,"\\Answered",bfsz-1); }
	if (flags & FLAGS_DELETED) {if (bf[0]) ncat(bf," ",bfsz-1); ncat(bf,"\\Deleted",bfsz-1); }
	if (flags & FLAGS_FLAGGED) {if (bf[0]) ncat(bf," ",bfsz-1); ncat(bf,"\\Flagged",bfsz-1); }
	if (flags & FLAGS_SEEN) {if (bf[0]) ncat(bf," ",bfsz-1); ncat(bf,"\\Seen",bfsz-1); }
	if (flags & FLAGS_DRAFT) {if (bf[0]) ncat(bf," ",bfsz-1); ncat(bf,"\\Draft",bfsz-1); }
	if (flags & FLAGS_RECENT) {if (bf[0]) ncat(bf," ",bfsz-1); ncat(bf,"\\Recent",bfsz-1); }
	if (flags & FLAGS_NOINFERIORS) {if (bf[0]) ncat(bf," ",bfsz-1); ncat(bf,"\\NoInferiors",bfsz-1); }
	if (flags & FLAGS_FORWARDED) {if (bf[0]) ncat(bf," ",bfsz-1); ncat(bf,"$Forwarded",bfsz-1); }
	return bf;
}

int im_read_data_direct(ImapCon *i, char *mem, int bytes)
{
	int done,x;

	for(done = 0; done < bytes; done += x) {
		x = tcp_read_timed(i->c,&mem[done],bytes-done,60000);
		if (x <= 0) { im_tcp_error(i); return FALSE; }
	}
	mem[done] = 0;

	return TRUE;
}

#define RDBUF 32000
int im_read_data(ImapCon *i, int bytes, int(*xwrite)(void*,int,int,void*), void *xp)
{
	char buffer[RDBUF];
	int block,done,x;

	for(done = 0; done < bytes; done += x) {
		block = bytes-done;
		if (block > RDBUF) block = RDBUF;
		x = tcp_read_timed(i->c,buffer,block,60000);
		if (x <= 0) { 
			im_tcp_error(i);
			return FALSE;
		}
		if (xwrite && xwrite(buffer,1,x,xp) != x) {
			return FALSE;
		}
	}

	return TRUE;
}

int im_skip_data(ImapCon *i, int bytes)
{
	return im_read_data(i,bytes,NULL,NULL);
}



//************
// Smarter IMAP message retrieval
//************
int im_parse_envelope(ImapCon *i, int foo)		// cf imap_read_envelope
{
	char bf[BFSZ];
	im_parse(i,bf,BFSZ);
	return TRUE;
}

int im_parse_pairs(ImapCon *i, char *token, int bfsz, Pair **pair)
{		
	char val[BFSZ];
	if(!im_parse(i,token,bfsz)) return FALSE;	
	if(strcmp(token,"(")!=0) return TRUE;	// OK, was not a pair list of pairs after all

	if(!im_parse(i,token,bfsz)) return FALSE;
	for(;strcmp(token,")")!=0;){
		if(!im_parse(i,val,bfsz)) return FALSE;
		if(pair)pair_add(pair,token,val);
		if(!im_parse(i,token,bfsz)) return FALSE;
	}
	return TRUE;
}
int im_parse_skiprest(ImapCon *i, char *token, int bfsz, Pair **pair, SwPart *p)
{
	int level=1;
	ncpy(token,"",BFSZ-1);
	for(;!(level==0 && strcmp(token,")")==0);){
		if(!im_parse(i,token,bfsz)) return FALSE;
		if(strcmp(token,"(")==0) level++;
		if(strcmp(token,")")==0) level--;
        if(strstrnc(token,"attachment")!=0) p->is_attachment = TRUE;
	}
	if(strcmp(token,")")!=0) return FALSE;
	return TRUE;
}

// some examples to to parse
//
// BODY (("TEXT" "PLAIN" ("format" "flowed" "charset" "us-ascii") NIL NIL "7BIT" 124 9)										1
//		 ("TEXT" "HTML" ("charset" "utf-8") NIL NIL "QUOTED-PRINTABLE" 552 14) "ALTERNATIVE"))								2
//
//BODY ((	  ("TEXT" "PLAIN" ("format" "flowed" "charset" "us-ascii") NIL NIL "7BIT" 13 1)
//		  ("TEXT" "HTML" ("charset" "us-ascii") NIL NIL "QUOTED-PRINTABLE" 247 10)
//		"ALTERNATIVE")
//		("IMAGE" "PNG" ("name" "sw_img.png") NIL NIL "BASE64" 4882)
//	  "MIXED")
//
//BODYSTRUCTURE ((
//			("TEXT" "PLAIN" ("format" "flowed" "charset" "us-ascii") NIL NIL "7BIT" 13 1 NIL NIL NIL NIL)					1.1
//			("TEXT" "HTML" ("charset" "us-ascii") NIL NIL "QUOTED-PRINTABLE" 247 10 NIL NIL NIL NIL)						1.1
//		"ALTERNATIVE" ("boundary" "----=_SW_147891247_1284436523_mpa=") NIL NIL)											1
//		("IMAGE" "PNG" ("name" "sw_img.png") NIL NIL "BASE64" 4882 NIL ("ATTACHMENT" ("filename" "sw_img.png")) NIL NIL)	2
//	"MIXED" ("boundary" "----=_SW_147891247_1284436523_mixed=") NIL NIL)
//)

int im_parse_part(ImapCon *i, PartAllocator *pa, char *token, int depth, int sibl, char *parentname, Pair **pair)
{
	int n=1;
	char bf[BFSZ];
	SwPart *p=parts_new_part(pa, depth, NULL);
	if(depth<=1){
		lprintf(bf,BFSZ,"%d",sibl);
	}else{
		lprintf(bf,BFSZ,"%s.%d",parentname,sibl);
	}
	p->name=strdup(bf);

	if(!im_parse(i,token,BFSZ)) return FALSE;

	if (strcmp(token,"(")==0){
		// recursively read other body parts
		for(;strcmp(token,"(")==0;){
			im_parse_part(i, pa, token, depth+1, n++, p->name, pair);
			if(!im_parse(i,token,BFSZ)) return FALSE;	
		}

		// read multipart info
		p->type=strdup("multipart");
		p->subtype=strdup(token);

		if(!im_parse_pairs(i,token,BFSZ, pair)) return FALSE;

		// read any extension tokens till ")"
		if(!im_parse_skiprest(i,token,BFSZ,pair,p)) return FALSE;	// content 

		if (strcmp(token,")")!=0) return FALSE;
	}else{
		// reading plain BODY a body part type
		p->type=strdup(token);
		if(!im_parse(i,token,BFSZ)) return FALSE;
		p->subtype=strdup(token);

		//if ( read as many data pairs until )
		if(!im_parse_pairs(i,token,BFSZ,&p->hdrs)) return FALSE;	// QQQParts
		if(strcmp(pair_findnc(p->hdrs,"format"),"flowed")==0)
			pair_add(&p->hdrs,"flowed","true");				// add format surgeweb expects

		// read std tokens,
		if(!im_parse(i,token,BFSZ)) return FALSE;	// content id
		if(!im_parse(i,token,BFSZ)) return FALSE;	// content description
		if(!im_parse(i,token,BFSZ)) return FALSE;	// content encoding
		p->encoding=strdup(token);
		if(!im_parse(i,token,BFSZ)) return FALSE;	// content size
		p->size=atoi(token);

		// read any extension tokens till ")"
		if(!im_parse_skiprest(i,token,BFSZ,pair,p)) return FALSE;	// content 

		if(strcmp(token,")")!=0) return FALSE;
	}
	return TRUE;
}
int im_parse_bodystructure(ImapCon *i, PartAllocator *pa)		// imap_read_structure
{
	char token[BFSZ];
	Pair *pair=NULL;
	ncpy(token,"",BFSZ-1);
	if(!im_parse(i, token, BFSZ)) return FALSE;	// *
	if(!im_parse(i, token, BFSZ)) return FALSE;	// nn
	if(!im_parse(i, token, BFSZ)) return FALSE;	// FETCH
	if(!im_parse(i, token, BFSZ)) return FALSE;	// (
	if(!im_parse(i, token, BFSZ)) return FALSE;	// UID
	if(!im_parse(i, token, BFSZ)) return FALSE;	// nn
	if(!im_parse(i, token, BFSZ)) return FALSE;	// BODYSTRUCTURE
	if(!im_parse(i, token, BFSZ)) return FALSE;	// (
	if(strcmp(token,"(")!=0)  return FALSE;

	if(!im_parse_part(i, pa, token, 0, 1, "", &pair)) return FALSE; //(( big long nested string)(...))

	pair_free(&pair);
	return TRUE;
}
// ALSO version two that returns whole list of parts

// part retrieval uses standard command with : im_read_data or im_read_data_direct


/************/

void im_errno_error(ImapCon *i)
{
	if (!i) return ;
	lprintf(i->result,BFSZ,"%s ERR %s",i->tag,strerror(errno));
}

void im_tcp_error(ImapCon *i)
{
	char tmp[BFSZ];
	char tmp2[BFSZ];
	if (!i || i->result) return ;
	im_expect_long(i,FALSE); // disable before using the buffer for something else...
	lprintf(tmp2,BFSZ,"%s ERR ",i->tag);
	sprintf(i->c->erradd,"");
	tcp_error(i->c,tmp);
	lpcat(tmp2,BFSZ-1,tmp);
	ncpy(i->result,tmp2,BFSZ-1);
}

void im_reset_cb(ImapCon *i)
{
	if (!i) return ;
	i->cb = NULL;
	i->data = NULL;
}


/************/

