diff options
Diffstat (limited to 'libc/string/wstring.c')
-rw-r--r-- | libc/string/wstring.c | 684 |
1 files changed, 647 insertions, 37 deletions
diff --git a/libc/string/wstring.c b/libc/string/wstring.c index 531b1c9fd..c1ead6e00 100644 --- a/libc/string/wstring.c +++ b/libc/string/wstring.c @@ -26,6 +26,13 @@ * * ATTENTION! ATTENTION! ATTENTION! ATTENTION! ATTENTION! */ +/* Dec 20, 2002 + * + * Initial test implementation of strcoll, strxfrm, wcscoll, and wcsxfrm. + * The code needs to be cleaned up a good bit, but I'd like to see people + * test it out. + */ + #define _STDIO_UTILITY #define _GNU_SOURCE #include <string.h> @@ -36,11 +43,12 @@ #include <stdlib.h> #include <errno.h> #include <signal.h> +#include <assert.h> +#include <locale.h> #ifdef WANT_WIDE #include <wchar.h> #include <wctype.h> -#include <locale.h> #define Wvoid wchar_t #define Wchar wchar_t @@ -627,17 +635,14 @@ int Wmemcmp(const Wvoid *s1, const Wvoid *s2, size_t n) #ifdef L_strcmp +#ifdef __LOCALE_C_ONLY +#warning c only #ifdef L_wcscmp -#ifdef __UCLIBC_MJN3_ONLY__ -#warning implement wcscoll and remove weak alias (or enable for C locale only) -#endif weak_alias(wcscmp,wcscoll); #else /* L_wcscmp */ -#ifdef __UCLIBC_MJN3_ONLY__ -#warning implement strcoll and remove weak alias (or enable for C locale only) -#endif weak_alias(strcmp,strcoll); #endif /* L_wcscmp */ +#endif /* __LOCALE_C_ONLY */ int Wstrcmp(register const Wchar *s1, register const Wchar *s2) { @@ -661,23 +666,6 @@ int Wstrcmp(register const Wchar *s1, register const Wchar *s2) } #endif /**********************************************************************/ -#ifdef L_strcoll -#error implement strcoll and remove weak_alias!! - -#if 0 -extern unsigned char *_ctype_collate; -int strcoll(register const char *s1, const char *s2) -{ - int r; - - while (!(r = (_ctype_collate[(int)(*s1++)]-_ctype_collate[(int)(*s2++)]))); - - return r; -} -#endif - -#endif -/**********************************************************************/ #ifdef L_wcsncmp #define L_strncmp #define Wstrncmp wcsncmp @@ -713,11 +701,6 @@ int Wstrncmp(register const Wchar *s1, register const Wchar *s2, size_t n) #endif /**********************************************************************/ -#ifdef L_strxfrm -#error implement strxfrm -/* size_t strxfrm(char *dst, const char *src, size_t len); */ -#endif -/**********************************************************************/ #ifdef L_wmemchr #define L_memchr #define Wmemchr wmemchr @@ -1923,28 +1906,37 @@ size_t strlcat(register char *__restrict dst, #endif /**********************************************************************/ -#ifdef L_wcsxfrm +#ifdef WANT_WIDE +extern size_t __wcslcpy(wchar_t *__restrict dst, + const wchar_t *__restrict src, + size_t n); +#endif + + +#ifdef L___wcslcpy #define L_strlcpy -#define Wstrlcpy wcsxfrm +#define Wstrlcpy __wcslcpy +#ifdef __LOCALE_C_ONLY +weak_alias(__wcslcpy,wcsxfrm); +#endif #endif #ifdef L_strlcpy -#ifndef L_wcsxfrm +#ifndef L___wcslcpy #define Wstrlcpy strlcpy -#ifdef __UCLIBC_MJN3_ONLY__ -#warning implement wcscoll and remove weak alias (or enable for C locale only) -#endif +#ifdef __LOCALE_C_ONLY weak_alias(strlcpy,strxfrm); #endif +#endif /* OpenBSD function: * Copy at most n-1 chars from src to dst and nul-terminate dst. * Returns strlen(src), so truncation occurred if the return value is >= n. */ size_t Wstrlcpy(register Wchar *__restrict dst, - register const Wchar *__restrict src, - size_t n) + register const Wchar *__restrict src, + size_t n) { const Wchar *src0 = src; Wchar dummy[1]; @@ -2145,3 +2137,621 @@ void psignal(int signum, register const char *message) #endif /**********************************************************************/ +#ifndef __LOCALE_C_ONLY + +#ifdef L_strxfrm +#ifndef WANT_WIDE +#error WANT_WIDE should be defined for L_strxfrm +#endif +#ifdef L_wcsxfrm +#error L_wcsxfrm already defined for L_strxfrm +#endif + +#define wcscoll strcoll +#define L_wcsxfrm +#undef WANT_WIDE + +#undef Wvoid +#undef Wchar +#undef Wuchar +#undef Wint + +#define Wchar char + +#endif /* L_strxfrm */ + + + +#ifdef L_wcsxfrm + +#define CUR_COLLATE (&__global_locale.collate) + +#define MAX_PENDING 8 + +typedef struct { + const Wchar *s; + const Wchar *eob; /* end of backward */ + + __uwchar_t weight; + __uwchar_t ui_weight; /* undefined or invalid */ + int colitem; + int weightidx; + int rule; + size_t position; + /* should be wchar_t. if wchar < 0 do EILSEQ? */ + __uwchar_t *cip; + __uwchar_t ci_pending[MAX_PENDING]; /* nul-terminated */ + + char *back_buf; + char *bbe; /* end of back_buf (actual last... not 1 past end) */ + char *bp; /* ptr into backbuf, NULL if not in backward mode */ + char ibb[128]; + size_t bb_size; + + int ru_pushed; +} col_state_t; + + +#define WEIGHT_MASK 0x3fffU +#define RULE_MASK 0xc000U + +#define RULE_FORWARD (1 << 14) +#define RULE_POSITION (1 << 15) + +#define UI_IDX (WEIGHT_MASK-6) +#define POSIT_IDX (WEIGHT_MASK-5) +#define RANGE_IDX (WEIGHT_MASK-4) +#define UNDEF_IDX (WEIGHT_MASK-3) +#define INVAL_IDX (WEIGHT_MASK-2) +#define DITTO_IDX (WEIGHT_MASK-1) + + +#undef TRACE +#if 0 +#define TRACE(X) printf##X +#else +#define TRACE(X) ((void)0) +#endif + +static int lookup(wchar_t wc) +{ + unsigned int sc, n, i0, i1; + + if (((__uwchar_t) wc) > 0xffffU) { + return 0; + } + + sc = wc & CUR_COLLATE->ti_mask; + wc >>= CUR_COLLATE->ti_shift; + n = wc & CUR_COLLATE->ii_mask; + wc >>= CUR_COLLATE->ii_shift; + + i0 = CUR_COLLATE->wcs2colidt_tbl[wc]; + i0 <<= CUR_COLLATE->ii_shift; + i1 = CUR_COLLATE->wcs2colidt_tbl[CUR_COLLATE->ii_len + i0 + n]; + i1 <<= CUR_COLLATE->ti_shift; + return CUR_COLLATE->wcs2colidt_tbl[CUR_COLLATE->ii_len + CUR_COLLATE->ti_len + i1 + sc]; + +} + +static void init_col_state(col_state_t *cs, const Wchar *wcs) +{ + memset(cs, 0, sizeof(col_state_t)); + cs->s = wcs; + cs->bp = cs->back_buf = cs->ibb; + cs->bb_size = 128; + cs->bbe = cs->back_buf + (cs->bb_size -1); +} + +static void next_weight(col_state_t *cs, int pass) +{ + int r, w, ru, ri, popping_backup_stack; + ssize_t n; + const uint16_t *p; +#ifdef WANT_WIDE +#define WC (*cs->s) +#define N (1) +#else /* WANT_WIDE */ + mbstate_t mbstate; + wchar_t WC; + size_t n0, nx; +#define N n0 + + mbstate.mask = 0; +#endif /* WANT_WIDE */ + + do { + + if (cs->ru_pushed) { + ru = cs->ru_pushed; + TRACE(("ru_pushed = %d\n", ru)); + cs->ru_pushed = 0; + goto POSITION_SKIP; + } + +#ifdef __UCLIBC_MJN3_ONLY__ +#warning should we walk pendings backwards? +#endif + if (cs->cip) { /* possible pending weight */ + if ((r = *(cs->cip++)) == 0) { + cs->cip = NULL; + continue; + } + cs->weightidx = r & WEIGHT_MASK; + assert(cs->weightidx); +/* assert(cs->weightidx != WEIGHT_MASK); */ + } else { /* get the next collation item from the string */ + TRACE(("clearing popping flag\n")); + popping_backup_stack = 0; + + IGNORE_LOOP: + /* keep first pos as 0 for a sentinal */ + if (*cs->bp) { /* pending backward chars */ + POP_BACKUP: + popping_backup_stack = 1; + TRACE(("setting popping flag\n")); + n = 0; + if (*cs->bp > 0) { /* singles pending */ + cs->s -= 1; + if ((*cs->bp -= 1) == 0) { + cs->bp -= 1; + } + } else { /* last was a multi */ + cs->s += *cs->bp; + cs->bp -= 1; + } + } else if (!*cs->s) { /* not in backward mode and end of string */ + cs->weight = 0; + return; + } else { + cs->position += 1; + } + + BACK_LOOP: +#ifdef WANT_WIDE + n = 1; + cs->colitem = r = lookup(*cs->s); +#else /* WANT_WIDE */ + n = n0 = mbrtowc(&WC, cs->s, SIZE_MAX, &mbstate); + if (n < 0) { + __set_errno(EILSEQ); + cs->weight = 0; + return; + } + cs->colitem = r = lookup(WC); +#endif /* WANT_WIDE */ + + TRACE((" r=%d WC=%#lx\n", r, (unsigned long)(WC))); + + if (r > CUR_COLLATE->max_col_index) { /* starting char for one or more sequences */ + p = CUR_COLLATE->multistart_tbl; + p += p[r-CUR_COLLATE->max_col_index -1]; + do { + n = N; + r = *p++; + do { + if (!*p) { /* found it */ + cs->colitem = r; + TRACE((" found multi %d\n", n)); + goto FOUND; + } +#ifdef WANT_WIDE + /* the lookup check here is safe since we're assured that *p is a valid colidx */ + if (!cs->s[n] || (lookup(cs->s[n]) != *p)) { + do {} while (*p++); + break; + } + ++p; + ++n; +#else /* WANT_WIDE */ + if (cs->s[n]) { + nx = mbrtowc(&WC, cs->s + n, SIZE_MAX, &mbstate); + if (nx < 0) { + __set_errno(EILSEQ); + cs->weight = 0; + return; + } + } + if (!cs->s[n] || (lookup(WC) != *p)) { + do {} while (*p++); + break; + } + ++p; + n += nx; +#endif /* WANT_WIDE */ + } while (1); + } while (1); + } else if (r == 0) { /* illegal, undefined, or part of a range */ + if ((CUR_COLLATE->range_count) +#ifdef __UCLIBC_MJN3_ONLY__ +#warning .. need to introduce range as a collating item? +#endif + && (((__uwchar_t)(WC - CUR_COLLATE->range_low)) <= CUR_COLLATE->range_count) + ) { /* part of a range */ + /* Note: cs->colitem = 0 already. */ + TRACE((" found range\n")); + ru = CUR_COLLATE->ruletable[CUR_COLLATE->range_rule_offset*CUR_COLLATE->MAX_WEIGHTS + pass]; + assert((ru & WEIGHT_MASK) != DITTO_IDX); + if ((ru & WEIGHT_MASK) == WEIGHT_MASK) { + ru = (ru & RULE_MASK) | RANGE_IDX; + cs->weight = CUR_COLLATE->range_base_weight + (WC - CUR_COLLATE->range_low); + } + goto RANGE_SKIP_TO; + } else if (((__uwchar_t)(WC)) <= 0x7fffffffUL) { /* legal but undefined */ + UNDEFINED: + /* Note: cs->colitem = 0 already. */ + ri = CUR_COLLATE->undefined_idx; + assert(ri != 0); /* implicit undefined isn't supported */ + + TRACE((" found explicit UNDEFINED\n")); +#ifdef __UCLIBC_MJN3_ONLY__ +#warning right now single weight locales do not support .. +#endif + if (CUR_COLLATE->num_weights == 1) { + TRACE((" single weight UNDEFINED\n")); + cs->weightidx = RANGE_IDX; + cs->weight = ri; + cs->s += n; + goto PROCESS_WEIGHT; + } + + ri = CUR_COLLATE->index2ruleidx[ri - 1]; + ru = CUR_COLLATE->ruletable[ri * CUR_COLLATE->MAX_WEIGHTS + pass]; + assert((ru & WEIGHT_MASK) != WEIGHT_MASK); /* TODO: handle ".." */ + if ((ru & WEIGHT_MASK) == DITTO_IDX) { + cs->colitem = CUR_COLLATE->undefined_idx; + } + goto RANGE_SKIP_TO; + } else { /* illegal */ + TRACE((" found illegal\n")); + __set_errno(EINVAL); + /* We put all illegals in the same equiv class with maximal weight, + * and ignore them after the first pass. */ + if (pass > 0) { + cs->s += n; + goto IGNORE_LOOP; + } + ru = (RULE_FORWARD | RANGE_IDX); + cs->weight = 0xffffU; + goto RANGE_SKIP_TO; + } + } else if (CUR_COLLATE->num_weights == 1) { + TRACE((" single weight\n")); + cs->weightidx = RANGE_IDX; + cs->weight = cs->colitem; + cs->s += n; + goto PROCESS_WEIGHT; + } else { + TRACE((" normal\n")); + } + + /* if we get here, it is a normal char either singlely weighted, undefined, or in a range */ + FOUND: + ri = CUR_COLLATE->index2ruleidx[cs->colitem - 1]; + TRACE((" ri=%d ", ri)); +#ifdef __UCLIBC_MJN3_ONLY__ +#warning make sure this is correct +#endif + if (!ri) { + TRACE(("NOT IN THIS LOCALE\n")); + goto UNDEFINED; + } + ru = CUR_COLLATE->ruletable[ri * CUR_COLLATE->MAX_WEIGHTS + pass]; + + RANGE_SKIP_TO: + +#ifdef __UCLIBC_MJN3_ONLY__ +#warning ignoreables probably should not interrupt backwards processing, but this is wrong +#endif +/* if (!(ru & WEIGHT_MASK)) { */ +/* TRACE(("IGNORE\n")); */ +/* cs->s += n; */ +/* continue; */ +/* } */ + + + TRACE((" rule = %#x weight = %#x popping = %d s = %p eob = %p\n", + ru & RULE_MASK, ru & WEIGHT_MASK, popping_backup_stack, + cs->s, cs->eob)); + /* now we need to check if we're going backwards... */ + + if (!popping_backup_stack) { + if (!(ru & RULE_MASK)) { /* backward */ + TRACE(("backwards\n")); + assert(cs->bp <= cs->bbe); + if (cs->bp == cs->bbe) { + if (cs->back_buf == cs->ibb) { /* was using internal buffer */ + cs->bp = malloc(cs->bb_size + 128); + if (!cs->bp) { + __set_errno(ENOMEM); +#ifdef __UCLIBC_MJN3_ONLY__ +#warning what to do here? +#endif + cs->weight = 0; + return; + } + memcpy(cs->bp, cs->back_buf, cs->bb_size); + + } else { + cs->bp = realloc(cs->back_buf, cs->bb_size + 128); + if (!cs->bp) { + __set_errno(ENOMEM); +#ifdef __UCLIBC_MJN3_ONLY__ +#warning what to do here? +#endif + cs->weight = 0; + return; + } + } + cs->bb_size += 128; + cs->bbe = cs->bp + (cs->bbe - cs->back_buf); + cs->back_buf = cs->bp; + cs->bp = cs->bbe; + + } + if (n==1) { /* single char */ + if (*cs->bp && (((unsigned char)(*cs->bp)) < CHAR_MAX)) { + *cs->bp += 1; /* increment last single's count */ + } else { /* last was a multi, or just starting */ + if (!cs->bp) { + cs->bp = cs->back_buf; + } else { + assert(cs->bp < cs->bbe); + ++cs->bp; + } + *cs->bp = 1; + } + } else { /* multichar */ + assert(n>1); + assert(cs->bp < cs->bbe); + *++cs->bp = -n; + } + cs->s += n; + if (*cs->s) { + goto BACK_LOOP; + } + /* end-of-string so start popping */ + cs->eob = cs->s; + TRACE(("popping\n")); + goto POP_BACKUP; + } else if (*cs->bp) { /* was going backward but this element isn't */ + /* discard current and use previous backward element */ + assert(!cs->cip); + cs->eob = cs->s; + TRACE(("popping\n")); + goto POP_BACKUP; + } else { /* was and still going forward */ + TRACE(("forwards\n")); + if ((ru & (RULE_POSITION|WEIGHT_MASK)) > RULE_POSITION) { + assert(ru & WEIGHT_MASK); + cs->ru_pushed = ru; + cs->weight = cs->position; +#ifdef __UCLIBC_MJN3_ONLY__ +#warning devel code +#endif + cs->position = 0; /* reset to reduce size for strcoll? */ + cs->s += n; + cs->weightidx = RANGE_IDX; + goto PROCESS_WEIGHT; + } + } + } else { /* popping backwards stack */ + TRACE(("popping (continued)\n")); + if (!*cs->bp) { + cs->s = cs->eob; + } + cs->s -= n; + } + + cs->s += n; + POSITION_SKIP: + cs->weightidx = ru & WEIGHT_MASK; + cs->rule = ru & RULE_MASK; + } + +#ifdef __UCLIBC_MJN3_ONLY__ +#warning for pending we only want the weight... _not_ the rule +#endif + if (!cs->weightidx) { /* ignore */ + continue; + } + + PROCESS_WEIGHT: + assert(cs->weightidx); + + + if (((unsigned int)(cs->weightidx - UI_IDX)) <= (INVAL_IDX-UI_IDX)) { + if (cs->weightidx == UI_IDX) { + cs->weight = cs->ui_weight; + } + return; + } + + assert(cs->weightidx != WEIGHT_MASK); + if (cs->weightidx == DITTO_IDX) { /* want the weight of the current collating item */ + TRACE(("doing ditto\n")); + w = CUR_COLLATE->index2weight[cs->colitem -1]; + } else if (cs->weightidx <= CUR_COLLATE->max_col_index) { /* normal */ + TRACE(("doing normal\n")); + w = CUR_COLLATE->index2weight[cs->weightidx -1]; + } else { /* a string */ + TRACE(("doing string\n")); + assert(!(cs->weightidx & RULE_MASK)); + /* note: iso14561 allows null string here */ + p = CUR_COLLATE->weightstr + (cs->weightidx - (CUR_COLLATE->max_col_index + 2)); + if (*p & WEIGHT_MASK) { + r = 0; + do { + assert(r < MAX_PENDING); + cs->ci_pending[r++] = *p++; + } while (*p & WEIGHT_MASK); + cs->cip = cs->ci_pending; + } + continue; + } + + cs->weight = w; + return; + } while (1); +} + +int wcscoll (const Wchar *s0, const Wchar *s1) +{ + col_state_t ws[2]; + int pass; + + if (!CUR_COLLATE->num_weights) { /* C locale */ +#ifdef WANT_WIDE + return wcscmp(s0, s1); +#else /* WANT_WIDE */ + return strcmp(s0, s1); +#endif /* WANT_WIDE */ + } + + pass = 0; + do { /* loop through the weights levels */ + init_col_state(ws, s0); + init_col_state(ws+1, s1); + do { /* loop through the strings */ + /* for each string, get the next weight */ + next_weight(ws, pass); + next_weight(ws+1, pass); + TRACE(("w0=%lu w1=%lu\n", + (unsigned long) ws[0].weight, + (unsigned long) ws[1].weight)); + + if (ws[0].weight != ws[1].weight) { + return ws[0].weight - ws[1].weight; + } + } while (ws[0].weight); + } while (++pass < CUR_COLLATE->num_weights); + + return 0; +} + +#ifdef WANT_WIDE + +size_t wcsxfrm(wchar_t *__restrict ws1, const wchar_t *__restrict ws2, size_t n) +{ + col_state_t cs; + size_t count; + int pass; + + if (!CUR_COLLATE->num_weights) { /* C locale */ + return __wcslcpy(ws1, ws2, n); + } + +#ifdef __UCLIBC_MJN3_ONLY__ +#warning handle empty string as a special case +#endif + + count = pass = 0; + do { /* loop through the weights levels */ + init_col_state(&cs, ws2); + do { /* loop through the string */ + next_weight(&cs, pass); + TRACE(("weight=%lu (%#lx)\n", (unsigned long) cs.weight, (unsigned long) cs.weight)); + if (count < n) { + ws1[count] = cs.weight +1; + } + ++count; + TRACE(("--------------------------------------------\n")); + } while (cs.weight); + if (count <= n) { /* overwrite the trailing 0 end-of-pass marker */ + ws1[count-1] = 1; + } + TRACE(("-------------------- pass %d --------------------\n", pass)); + } while (++pass < CUR_COLLATE->num_weights); + if (count <= n) { /* oops... change it back */ + ws1[count-1] = 0; + } + return count-1; +} + +#else /* WANT_WIDE */ + +static const unsigned long bound[] = { + 1UL << 7, + 1UL << 11, + 1UL << 16, + 1UL << 21, + 1UL << 26, +}; + +static unsigned char first[] = { + 0x0, 0xc0, 0xe0, 0xf0, 0xf8, 0xfc +}; + +/* Use an extension of UTF-8 to store a 32 bit val in max 6 bytes. */ + +static size_t store(unsigned char *s, size_t count, size_t n, __uwchar_t weight) +{ + int i, r; + + i = 0; + do { + if (weight < bound[i]) { + break; + } + } while (++i < sizeof(bound)/sizeof(bound[0])); + + r = i+1; + if (i + count < n) { + s += count; + s[0] = first[i]; + while (i) { + s[i] = 0x80 | (weight & 0x3f); + weight >>= 6; + --i; + } + s[0] |= weight; + } + + return r; +} + +size_t strxfrm(char *__restrict ws1, const char *__restrict ws2, size_t n) +{ + col_state_t cs; + size_t count, inc; + int pass; + + if (!CUR_COLLATE->num_weights) { /* C locale */ + return strlcpy(ws1, ws2, n); + } + +#ifdef __UCLIBC_MJN3_ONLY__ +#warning handle empty string as a special case +#endif + + inc = count = pass = 0; + do { /* loop through the weights levels */ + init_col_state(&cs, ws2); + do { /* loop through the string */ + next_weight(&cs, pass); + TRACE(("weight=%lu (%#lx)\n", (unsigned long) cs.weight, (unsigned long) cs.weight)); + inc = store((unsigned char *)ws1, count, n, cs.weight + 1); + count += inc; + TRACE(("--------------------------------------------\n")); + } while (cs.weight); + /* overwrite the trailing 0 end-of-pass marker */ + assert(inc == 1); + if (count <= n) { + ws1[count-1] = 1; + } + TRACE(("-------------------- pass %d --------------------\n", pass)); + } while (++pass < CUR_COLLATE->num_weights); + if (count <= n) { /* oops... change it back */ + ws1[count-1] = 0; + } + return count-1; +} + + +#endif /* WANT_WIDE */ + +#endif /* wcscoll */ + +#endif /* __LOCALE_C_ONLY */ +/**********************************************************************/ + |