/* utf8.c */

/*
SOURCED:  Copyright (c) 2005 JSON.org	(http://www.json.org/JSON_checker/)
MODIFIED: Marijn to be usable in a threaded environment
PURPOSE:  Allow easy UTF8 <--> html code point conversion (without iconv etc)
*/

#ifdef __XCODE__
#include "xcode_quiet.h"
#endif

#include <stdlib.h>
#include <string.h>
#include <ctype.h>

#include "utf8.h"
#include "lib.h"
#include "lprintf.h"

#ifndef FALSE
#define FALSE 0
#endif

#ifndef TRUE
#define TRUE 1
#endif


#include "utf8_lib.h"

int imsg(const char *arg_list, ...);

/*
    Very Strict UTF-8 Decoder

    UTF-8 is a multibyte character encoding of Unicode. A character can be
    represented by 1-4 bytes. The bit pattern of the first byte indicates the
    number of continuation bytes.

    Most UTF-8 decoders tend to be lenient, attempting to recover as much
    information as possible, even from badly encoded input. This UTF-8
    decoder is not lenient. It will reject input which does not include
    proper continuation bytes. It will reject aliases (or suboptimal
    codings). It will reject surrogates. (Surrogate encoding should only be
    used with UTF-16.)

    Code     Contination Minimum Maximum
    0xxxxxxx           0       0     127
    10xxxxxx       error
    110xxxxx           1     128    2047
    1110xxxx           2    2048   65535 excluding 55296 - 57343
    11110xxx           3   65536 1114111
    11111xxx       error
*/

/*
    Get the next byte. It returns UTF8_END if there are no more bytes.
*/
static int get(Utf8 *u)
{
    int c;
    if (u->the_index >= u->the_length) {
        return UTF8_END;
    }
    c = u->the_input[u->the_index] & 0xFF;
    u->the_index += 1;
    return c;
}


/*
    Get the 6-bit payload of the next continuation byte.
    Return UTF8_ERROR if it is not a contination byte.
*/
static int cont(Utf8 *u)
{
    int c = get(u);
    return ((c & 0xC0) == 0x80) ? (c & 0x3F) : UTF8_ERROR;
}


/*
    Initialize the UTF-8 decoder. The decoder is not reentrant,
*/
void utf8_decode_init(Utf8 *u, char p[], int length)
{
    u->the_index = 0;
    u->the_input = p;
    u->the_length = length;
    u->the_char = 0;
    u->the_byte = 0;
}


/*
    Get the current byte offset. This is generally used in error reporting.
*/
int utf8_decode_at_byte(Utf8 *u)
{
    return u->the_byte;
}


/*
    Get the current character offset. This is generally used in error reporting.
    The character offset matches the byte offset if the text is strictly ASCII.
*/
int utf8_decode_at_character(Utf8 *u)
{
    return u->the_char > 0 ? u->the_char - 1 : 0;
}


/*
    Extract the next character.
    Returns: the character (between 0 and 1114111)
         or  UTF8_END   (the end)
         or  UTF8_ERROR (error)
*/
int utf8_decode_next(Utf8 *u)
{
    int c;  /* the first byte of the character */
    int r;  /* the result */

    if (u->the_index >= u->the_length) {
        return u->the_index == u->the_length ? UTF8_END : UTF8_ERROR;
    }
    u->the_byte = u->the_index;
    u->the_char += 1;
    c = get(u);
/*
    Zero continuation (0 to 127)
*/
    if ((c & 0x80) == 0) {
        return c;
    }
/*
    One contination (128 to 2047)
*/
    if ((c & 0xE0) == 0xC0) {
        int c1 = cont(u);
        if (c1 < 0) {
            return UTF8_ERROR;
        }
        r = ((c & 0x1F) << 6) | c1;
        return r >= 128 ? r : UTF8_ERROR;
    }
/*
    Two continuation (2048 to 55295 and 57344 to 65535)
*/
    if ((c & 0xF0) == 0xE0) {
        int c1 = cont(u);
        int c2 = cont(u);
        if (c1 < 0 || c2 < 0) {
            return UTF8_ERROR;
        }
        r = ((c & 0x0F) << 12) | (c1 << 6) | c2;
        return r >= 2048 && (r < 55296 || r > 57343) ? r : UTF8_ERROR;
    }
/*
    Three continuation (65536 to 1114111)
*/
    if ((c & 0xF8) == 0xF0) {
        int c1 = cont(u);
        int c2 = cont(u);
        int c3 = cont(u);
        if (c1 < 0 || c2 < 0 || c3 < 0) {
            return UTF8_ERROR;
        }
        r = ((c & 0x0F) << 18) | (c1 << 12) | (c2 << 6) | c3;
        return r >= 65536 && r <= 1114111 ? r : UTF8_ERROR;
    }
    return UTF8_ERROR;
}

// marijn: I _think_ this have got everything right...
int utf8_output_char(int c, char *out)
{
	char c1,c2,c3;
	int work=c;

	if(c<128){
		*out=c;
		return 1;
	}
	c1=(0x80 | (work & 0x3F));	work = work>>6;
	if(c<2048){
		*out++	= (0xC0 | (work & 0x3F));
        *out++	= c1;
		return 2;
	}
	c2=(0x80 | (work & 0x3F));	work = work>>6;
	if(c<65536){
		*out++	= (0xE0 | (work & 0x1F));
		*out++	= c2; *out++ = c1;
		return 3;
	}
	c3=(0x80 | (work & 0x3F));	work = work>>6;
	if(c<1114112){
		*out++	= (0xF0 | (work & 0x0F));
		*out++	= c3; *out++ = c2; *out++ = c1;
		return 4;
	}
	return UTF8_ERROR;
}

char *utf8_to_codepoint(char *in, char *out, int bfsz)
{
	Utf8 u;
	int c,left;
	char *pin=in,*pout=out,*s;

	left=bfsz-10;
	*pout='\0';
	utf8_decode_init(&u,in,lib_strlen(in));
	for(;left>0;left--){		// should never get hit
		// Marijn: Warning: this function seems to douple decrement only allowing 500K buffers
		// leaving alone for now as I don have time to test for side effects
		// SHOULD be fixed at some stage
		c=utf8_decode_next(&u);
		if(c==UTF8_END || c==UTF8_ERROR) 
			break;
		if(c=='&')
			lprintf(pout,left,"&amp;",c);		
		else if(c<0x80 && c>=0x20)
			lprintf(pout,left,"%c",c);
		else
			lprintf(pout,left,"&#%d;",c);
		s=strchr(pout,'\0');
		if(!s) break;
		left-= (int)(s-pout);
		pout=s;
	}
	return out;
}
char *codepoint_to_utf8(char *in, char *out, int bfsz)
{
	char *pin=in,*pout=out;
	
	int i,c,n;
	for(i = 0; pout-out < bfsz && *pin;) {
		if (*pin != '&'){
			*pout++ = *pin++;;
			continue;
		}
		if (strncmp(pin,"&amp;",5)==0) {
			*pout++ = '&';
			pin += 5;
		}else if (strncmp(pin,"&#",2)==0) {
			pin+=2;
			c=atoi(pin);
			if(c==0) goto error;
			n = utf8_output_char(c,pout);
			if (n<0) goto error;
			pout += n;
			while(sisdigit(*pin)) pin++;
			if(*pin!=';') goto error;
			pin++;
		}else{
			*pout++ = *pin++;
		}
	}
error:
	*pout = '\0';
	return out;
}

char *utf8_validate(char *in, char *out, int bfsz)
{
	Utf8 u;
	int c,len,i,index,unknown=0, strip=TRUE;
	char *pin=in,*pout=out;

	len=lib_strlen(in);
	utf8_decode_init(&u,in,len);
	for(;len>0;len--){		// should never get hit and will still have a positive value if any UTF8 characters exist
		index=u.the_index;
		if(pout-out>bfsz-6) 
			break;
		c=utf8_decode_next(&u);
		if(c==UTF8_END)  break;
		if(c!=UTF8_ERROR){
			if(strip){
				// Always remove for now, maybe handle more cleverly later if required...
				if(c==0x2028) continue;	// Strip unicode line separators as it breaks javascript based message display in FF & safari & chrome
			}
			for(i=index;i<u.the_index;i++)
				*pout++=in[i];
			continue;
		}
		pout+=utf8_output_char(0xfffd,pout);
		unknown++;
	}
	*pout = '\0';
//	if(unknown>0) 
//		imsg("%d characters corrected in UTF8 validation ",unknown);
	return out;
}

int utf8_valid2(char *in)
{
	Utf8 u;
	int c,len,index,unknown=0, strip=TRUE;
	char *pin=in;

	len=lib_strlen(in);
	utf8_decode_init(&u,in,len);
	for(;len>0;len--){
		index=u.the_index;
		c=utf8_decode_next(&u);
		if(c==UTF8_END)  break;
		if(c==UTF8_ERROR) return FALSE;
	}

	return TRUE;
}

// Only actually use utf8 searches if actually enabled
int utf8_search = FALSE;
int utf8_info = TRUE;

void utf8_search_init(int use_utf8)
{
	utf8_search = use_utf8;
}

int utf8_strcmpnc(char *src1, char *src2)
{
	if(!utf8_search) return strcmpnc(src1, src2);

	return utf8casecmp((const char *)src1, (const char *)src2); 
}

int utf8_strncmpnc(char *src1, char *src2, int n)
{
	if(!utf8_search) return strncmpnc(src1, src2, n);

	// WARNING: Important that search terms is first parameter when calling utf8ncasecmp and length N to matches parameter 1
	return utf8ncasecmp((const char *)src2, (const char *)src1, (size_t)n); 
}

char *utf8_strstrnc(char *s1, char *s2)	// marijn: fundamental logic based on strstrnc
{
	utf8_int32_t s1_cp, s2_cp, b1_cp, b2_cp;
	char *a,*a1,*a2, bf[BFSZ], tmp[BFSZ];
	int n,i,j,n2;

	if(!utf8_search) return strstrnc(s1, s2);
	if(strlen(s2)==utf8len(s2)) {
		//if(utf8_info) imsg("SEARCH standard search, no utf8 characters"); 
		// chrisp removed this which was trashing the log files, possibly utf8_info should be 'false', but I"m not sure of it's meaning...
		return strstrnc(s1, s2);
	}

	if(utf8_info){
		char bf[BFSZ], tmp[BFSZ], *block=s1, *srch=s2;
		int i;
		//imsg("SEARCH ---");
		//imsg("SEARCH len=%d [%s] [%s]", strlen(block), block,srch); 
		lprintf(bf, BFSZ, "SEARCH RAW [");
		for(i=0;i<40;i++){
			if(block[i]==0) break;
			lprintf(tmp,BFSZ,"%s%02x",(i==0?"":"|"),(unsigned char)block[i]);
			ncat(bf,tmp,BFSZ-1);
		}
		ncat(bf,"] [",BFSZ-1);
		for(i=0;i<40;i++){
			if(srch[i]==0) break;
			lprintf(tmp,BFSZ,"%s%02x",(i==0?"":"|"),(unsigned char)srch[i]);
			ncat(bf,tmp,BFSZ-1);
		}
		ncat(bf,"]",BFSZ-1);
		//imsg("%s", bf); 
	}

	utf8codepoint(s2, &s2_cp);
	//n = utf8len(s2);
	n = strlen(s2);

	b1_cp = utf8lwrcodepoint(s2_cp);
	b2_cp = utf8uprcodepoint(s2_cp);

	//if(utf8_info) imsg("SEARCH utfCP cp=%08x lwr=%08x upr=%08x", s2_cp, b1_cp, b2_cp);
	// UTF8 code tables https://www.utf8-chartable.de/unicode-utf8-table.pl

	if (n==0) return NULL;
	for (a=s1,i=0;a!=NULL;i++) {

		a1 = (char*)utf8chr(a,b1_cp);
		a2 = (char*)utf8chr(a,b2_cp);

		//if(utf8_info) imsg("SEARCH CP find i=%d lwr_index=%d upr_index=%d", i, a1?a1-a:0, a2?a2-a:0);

		if (a2) {
			if (a1) {
				if (a2<a1) a1 = a2;
			} else {
				a1 = a2;
			}
		}
		if (a1==NULL) break;

		if(utf8_info){
			char *ch = utf8codepoint(a1, &s1_cp);
			n2 = ch-a1;
			lprintf(bf,BFSZ,"SEARCH possible match cp and bytes CP=%08x n=%d [", s1_cp, n2);
			for(j=0;j<n2;j++){
				lprintf(tmp,BFSZ,"%s%02x",(j==0?"":"|"),(unsigned char)a1[j]);
				ncat(bf,tmp,BFSZ-1);
			}
			ncat(bf,"]",BFSZ-1);
			//imsg("%s", bf);
		}
		if (utf8_strncmpnc(a1,s2,n)==0) {
			//if(utf8_info) imsg("SEARCH match at raw_index=%d raw_char=%02x", a1-a, (unsigned char)*a1);
			return a1;
		}
		a = utf8codepoint(a1, &s1_cp);
	}
	//if(utf8_info) imsg("SEARCH nomatch");
	return NULL;
}
