The big thing is locale dependent collation support.

Also added outdigit support and (legacy) YESSTR/NOSTR support.

1 files changed, 3920 insertions, 0 deletions

diff --git a/extra/locale/gen_collate.c b/extra/locale/gen_collate.c
new file mode 100644
index 000000000..9121345bc
--- /dev/null
+++ b/extra/locale/gen_collate.c
@@ -0,0 +1,3920 @@
+/* TODO:
+ *
+ * add UNDEFINED at end if not specified
+ * convert POSITION -> FORWARD,POSITION
+ *
+ *
+ * deal with lowercase in <Uhhhh>
+ *
+ * what about reorders that keep the same rule?
+ *
+ * remove "unused" collation elements? (probably doesn't save much)
+ *
+ * add_rule function ... returns index into rule table after possibly adding custom-indexed rule
+ * but don't forget about multichar weights... replace with strings of indexes
+ *
+ */
+
+
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdarg.h>
+#include <limits.h>
+#include <ctype.h>
+#include <assert.h>
+#include <search.h>
+
+
+typedef struct {
+	char *name;					/*  */
+
+	int num_weights;			/*  */
+
+	int ii_shift;				/*  */
+	int ti_shift;				/*  */
+	int ii_len;					/*  */
+	int ti_len;					/*  */
+	int max_weight;				/*  */
+	int num_col_base;			/*  */
+	int max_col_index;			/*  */
+	int undefined_idx;			/*  */
+	int range_low;				/*  */
+	int range_count;			/* high - low */
+	int range_base_weight;		/*  */
+	int num_starters;			/*  */
+
+	int range_rule_offset;		/*  */
+	int wcs2colidt_offset;		/*  */
+	int index2weight_offset;	/*  */
+	int index2ruleidx_offset;	/*  */
+	int multistart_offset;		/*  */
+
+} base_locale_t;
+
+#define BASE_LOCALE_LEN 20
+static base_locale_t base_locale_array[BASE_LOCALE_LEN];
+static size_t base_locale_len;
+
+typedef struct {
+	char *name;					/*  */
+
+	int base_idx;				/*  */
+
+	int undefined_idx;			/*  */
+	
+	int overrides_offset;		/*  */
+	int multistart_offset;		/*  */
+} der_locale_t;
+
+#define DER_LOCALE_LEN 300
+static der_locale_t der_locale_array[DER_LOCALE_LEN];
+static size_t der_locale_len;
+
+
+#define OVERRIDE_LEN  50000
+static uint16_t override_buffer[OVERRIDE_LEN];
+static size_t override_len;
+
+#define MULTISTART_LEN 10000
+static uint16_t multistart_buffer[MULTISTART_LEN];
+static size_t multistart_len;
+
+#define WCS2COLIDT_LEN 200000
+static uint16_t wcs2colidt_buffer[WCS2COLIDT_LEN];
+static size_t wcs2colidt_len;
+
+#define INDEX2WEIGHT_LEN 200000
+static uint16_t index2weight_buffer[INDEX2WEIGHT_LEN];
+static size_t index2weight_len;
+
+static uint16_t index2ruleidx_buffer[INDEX2WEIGHT_LEN];
+static size_t index2ruleidx_len;
+
+#define WEIGHTSTR_LEN 10000
+static uint16_t weightstr_buffer[WEIGHTSTR_LEN];
+static size_t weightstr_len;
+
+#define RULETABLE_LEN (1L<<16)
+static uint16_t ruletable_buffer[RULETABLE_LEN];
+static size_t ruletable_len;
+
+
+#define RANGE (0x10000UL)
+
+typedef uint16_t tbl_item;
+
+static uint16_t u16_buf[10000];
+static int u16_buf_len;
+static int u16_starter;
+
+typedef struct {
+	uint16_t ii_len;
+	uint16_t ti_len;
+	uint16_t ut_len;
+
+	unsigned char ii_shift;
+	unsigned char ti_shift;
+
+	tbl_item *ii;
+	tbl_item *ti;
+	tbl_item *ut;
+} table_data;
+
+
+static size_t newopt(tbl_item *ut, size_t usize, int shift, table_data *tbl);
+
+
+#define MAX_COLLATION_WEIGHTS 4
+
+#define MAX_FNO 1
+#define MAX_FILES  (MAX_FNO + 1)
+
+static FILE *fstack[MAX_FILES];
+static char *fname[MAX_FILES];
+static int lineno[MAX_FILES];
+static int fno = -1;
+
+static tbl_item wcs2index[RANGE];
+
+static char linebuf[1024];
+static char *pos;
+static char *pos_e = NULL;
+static char end_of_token = 0;		/* slot to save */
+
+#define IN_ORDER			0x01
+#define IN_REORDER			0x02
+#define IN_REORDER_SECTIONS	0x04
+static int order_state;
+static int cur_num_weights;		/* number of weights in current use */
+static char cur_rule[MAX_COLLATION_WEIGHTS];
+
+static int anonsection = 0;
+
+typedef struct ll_item_struct ll_item_t;
+
+struct ll_item_struct {
+	ll_item_t *next;
+	ll_item_t *prev;
+	void *data;
+	int data_type;
+	int idx;
+};
+
+static ll_item_t *reorder_section_ptr = NULL;
+static int superset;
+static int superset_order_start_cnt; /* only support one order for now */
+static int superset_in_sync;
+static ll_item_t *comm_cur_ptr;
+static ll_item_t *comm_prev_ptr;
+
+enum {
+	R_FORWARD =		0x01,
+	R_POSITION =	0x02,
+	R_BACKWARD =	0x04		/* must be largest in value */
+};
+
+typedef struct {
+	size_t num_weights;
+	char rule[MAX_COLLATION_WEIGHTS];
+	const char *colitem[MAX_COLLATION_WEIGHTS];
+} weight_t;
+
+static void *root_weight = NULL;
+size_t unique_weights = 0;
+
+typedef struct {
+	const char *symbol;
+	weight_t *weight;
+} weighted_item_t;
+
+typedef struct {
+	const char *symbol1;
+	const char *symbol2;
+	int length;
+	weight_t *weight;
+} range_item_t;
+
+typedef struct {
+	const char *name;
+	ll_item_t *itm_list;		/* weighted_item_t list .. circular!!! */
+	size_t num_items;
+	size_t num_rules;
+	char rules[MAX_COLLATION_WEIGHTS];
+} section_t;
+
+static section_t *cur_section = NULL;
+
+typedef struct {
+	const char *symbol;
+	ll_item_t *node;
+} wi_index_t;
+
+typedef struct col_locale_struct col_locale_t;
+
+struct  col_locale_struct {
+	char *name;
+	void *root_colitem;			/* all base and derived, or just derived */
+	void *root_element;
+	void *root_scripts;
+	void *root_wi_index;
+	void *root_wi_index_reordered;
+	ll_item_t *section_list;
+	col_locale_t *base_locale;	/* null if this is a base */
+	void *root_derived_wi;
+	ll_item_t *derived_list;
+	void *root_starter_char;
+	void *root_starter_all;
+	ll_item_t *undefined_idx;
+};
+
+typedef struct {
+	const char *symbol;
+	int idx;
+} col_index_t;
+
+static void *root_col_locale = NULL;
+
+typedef struct {
+    const char *keyword;
+    void (*handler)(void);
+} keyword_table_t;
+
+typedef struct {
+    const char *string;
+    const char *element;	/* NULL if collating symbol */
+} colitem_t;
+
+static col_locale_t *cur_base = NULL;
+static col_locale_t *cur_derived = NULL;
+static col_locale_t *cur_col = NULL;
+
+static void *root_sym = NULL;
+static size_t num_sym = 0;
+static size_t mem_sym = 0;
+
+static void error_msg(const char *fmt, ...) __attribute__ ((noreturn, format (printf, 1, 2)));
+static void *xmalloc(size_t n);
+static char *xsymdup(const char *s); /* only allocate once... store in a tree */
+static void pushfile(char *filename);
+static void popfile(void);
+static void processfile(void);
+static int iscommentchar(int);
+static void eatwhitespace(void);
+static int next_line(void);
+static char *next_token(void);
+static void do_unrecognized(void);
+static col_locale_t *new_col_locale(char *name);
+static ll_item_t *new_ll_item(int data_type, void *data);
+static weight_t *register_weight(weight_t *w);
+static size_t ll_len(ll_item_t *l);
+static size_t ll_count(ll_item_t *l, int mask);
+static void add_wi_index(ll_item_t *l);
+static size_t tnumnodes(const void *root);
+static ll_item_t *find_wi_index(const char *sym, col_locale_t *cl);
+static void mark_reordered(const char *sym);
+static ll_item_t *find_wi_index_reordered(const char *sym);
+static ll_item_t *next_comm_ptr(void);
+static ll_item_t *init_comm_ptr(void);
+static ll_item_t *find_ll_last(ll_item_t *p);
+static void dump_weights(const char *name);
+static void finalize_base(void);
+static int is_ucode(const char *s);
+static int sym_cmp(const void *n1, const void *n2);
+static void do_starter_lists(col_locale_t *cl);
+static void dump_base_locale(int n);
+static void dump_der_locale(int n);
+static void dump_collate(FILE *fp);
+
+enum {
+	DT_SECTION = 0x01,
+	DT_WEIGHTED = 0x02,
+	DT_REORDER = 0x04,		  /* a section to support reorder_after */
+	DT_COL_LOCALE = 0x08,
+	DT_RANGE = 0x10,
+};
+
+static section_t *new_section(const char *name)
+{
+	section_t *p;
+	char buf[128];
+
+	p = xmalloc(sizeof(section_t));
+	if (!name) {				/* anonymous section */
+		name = buf;
+		snprintf(buf, sizeof(buf), "anon%05d", anonsection);
+		++anonsection;
+	} else if (*name != '<') {	/* reorder */
+		name = buf;
+		snprintf(buf, sizeof(buf), "%s %05d", cur_col->name, anonsection);
+		++anonsection;
+	}
+#warning devel code
+/* 	fprintf(stderr, "section %s\n", name); */
+	p->name = xsymdup(name);
+	p->itm_list = NULL;
+	p->num_items = 0;
+	p->num_rules = 0;
+	memset(p->rules, 0, MAX_COLLATION_WEIGHTS);
+/* 	cur_num_weights = p->num_rules = 0; */
+/* 	memset(p->rules, 0, MAX_COLLATION_WEIGHTS); */
+/* 	memset(cur_rule, R_FORWARD, 4); */
+
+#warning devel code
+	if (*p->name == 'a') {
+		cur_num_weights = p->num_rules = 4;
+		memset(p->rules, R_FORWARD, 4);
+		memset(cur_rule, R_FORWARD, 4);
+		p->rules[3] |= R_POSITION;
+		cur_rule[3] |= R_POSITION;
+	}
+/* 	fprintf(stderr, "new section %s -- cur_num_weights = %d\n", p->name, cur_num_weights); */
+
+	return p;
+}
+
+
+
+static void do_order_start(void);
+static void do_order_end(void);
+static void do_reorder_after(void);
+static void do_reorder_end(void);
+static void do_reorder_sections_after(void);
+static void do_reorder_sections_end(void);
+static void do_copy(void);
+static void do_colsym(void);
+static void do_colele(void);
+static void do_script(void);
+static void do_range(void);
+
+static col_locale_t *new_col_locale(char *name);
+static int colitem_cmp(const void *n1, const void *n2);
+static int colelement_cmp(const void *n1, const void *n2);
+static void del_colitem(colitem_t *p);
+static colitem_t *new_colitem(char *item, char *def);
+static void add_colitem(char *item, char *def);
+static void add_script(const char *s);
+static unsigned int add_rule(weighted_item_t *wi);
+static unsigned int add_range_rule(range_item_t *ri);
+
+static const keyword_table_t keyword_table[] = {
+    { "collating-symbol", do_colsym },
+    { "collating-element", do_colele },
+	{ "script", do_script },
+    { "copy", do_copy },
+    { "order_start", do_order_start },
+    { "order_end", do_order_end },
+    { "order-end", do_order_end },
+    { "reorder-after", do_reorder_after },
+    { "reorder-end", do_reorder_end },
+    { "reorder-sections-after", do_reorder_sections_after },
+    { "reorder-sections-end", do_reorder_sections_end },
+	{ "UCLIBC_RANGE", do_range },
+    { NULL, do_unrecognized }
+};
+
+
+static void do_unrecognized(void)
+{
+#if 1
+    error_msg("warning: unrecognized: %s", pos);
+#else
+/*     fprintf(stderr, "warning: unrecognized initial keyword \"%s\"\n", pos); */
+	fprintf(stderr, "warning: unrecognized: %s", pos);
+	if (end_of_token) {
+		fprintf(stderr, "%c%s", end_of_token, pos_e+1);
+	}
+	fprintf(stderr, "\n");
+#endif
+}
+
+/* typedef struct { */
+/* 	const char *symbol1; */
+/* 	const char *symbol2; */
+/* 	int length; */
+/* 	weight_t *weight; */
+/* } range_item_t; */
+
+static void do_range(void)
+{
+	range_item_t *ri;
+	weight_t w;
+	int i;
+	char *s;
+	char *s1;
+	char *s2;
+	const char **ci;
+	ll_item_t *lli;
+
+	assert(!superset);
+	assert(order_state == IN_ORDER);
+
+	s1 = next_token();
+	if (!s1) {
+		error_msg("missing start of range");
+	}
+	if (!is_ucode(s1)) {
+		error_msg("start of range is not a ucode: %s", s1);
+	}
+	s1 = xsymdup(s1);
+
+	s2 = next_token();
+	if (!s2) {
+		error_msg("missing end of range");
+	}
+	if (!is_ucode(s2)) {
+		error_msg("end of range is not a ucode: %s", s2);
+	}
+	s2 = xsymdup(s2);
+
+	ri = (range_item_t *) xmalloc(sizeof(range_item_t));
+	ri->symbol1 = s1;
+	ri->symbol2 = s2;
+	ri->length = strtoul(s2+2, NULL, 16) - strtoul(s1+2, NULL, 16);
+	if (ri->length <= 0) {
+		error_msg("illegal range length %d", ri->length);
+	}
+
+	s = next_token();
+	w.num_weights = cur_num_weights;
+
+	for (i=0 ; i < cur_num_weights ; i++) {
+		w.rule[i] = cur_rule[i];
+	}
+	ci = w.colitem + (i-1);
+	/* now i == cur_num_weights */
+
+#define STR_DITTO "."
+
+	while (s && *s && i) {
+		--i;
+		if (*s == ';') {
+			ci[-i] = xsymdup(STR_DITTO);
+			if (*++s) {
+				continue;
+			}
+		}
+		if (*s) {
+			ci[-i] = xsymdup(s);
+		}
+		s = next_token();
+		if (s) {
+			if (*s == ';') {
+				++s;
+			} else if (i) {
+				error_msg("missing seperator");
+			}
+		}
+	}
+	if (s) {
+		error_msg("too many weights: %d %d |%s| %d", cur_num_weights, i, s, (int)*s);
+	}
+
+	while (i) {					/* missing weights are not an error */
+		--i;
+		ci[-i] = xsymdup(STR_DITTO);
+	}
+
+	ri->weight = register_weight(&w);
+
+/* 	if ((i = is_ucode(t)) != 0) { */
+/* 		assert(!t[i]); */
+/* 		add_colitem(t, NULL); */
+/* 	} */
+
+	lli = new_ll_item(DT_RANGE, ri);
+	if (!cur_section->itm_list) {
+/* 		printf("creating new item list: %s\n", wi->symbol); */
+		cur_section->itm_list = lli;
+		lli->prev = lli->next = lli;
+		++cur_section->num_items;
+	} else {
+		insque(lli, cur_section->itm_list->prev);
+/* 		printf("adding item to list: %d - %s\n", ll_len(cur_section->itm_list), wi->symbol); */
+		++cur_section->num_items;
+	}
+/* 	add_wi_index(lli); */
+
+
+}
+
+static weighted_item_t *add_weight(char *t)
+{
+	weighted_item_t *wi;
+	weight_t w;
+	int i;
+	char *s;
+	const char **ci;
+
+	t = xsymdup(t);
+
+	s = next_token();
+	w.num_weights = cur_num_weights;
+
+	for (i=0 ; i < cur_num_weights ; i++) {
+		w.rule[i] = cur_rule[i];
+	}
+	ci = w.colitem + (i-1);
+	/* now i == cur_num_weights */
+
+	while (s && *s && i) {
+		--i;
+		if (*s == ';') {
+			ci[-i] = xsymdup(STR_DITTO);
+			if (*++s) {
+				continue;
+			}
+		}
+		if (*s) {
+			if (!strcmp(s,t)) {
+				s = STR_DITTO;
+			}
+			ci[-i] = xsymdup(s);
+		}
+		s = next_token();
+		if (s) {
+			if (*s == ';') {
+				++s;
+			} else if (i) {
+				error_msg("missing seperator");
+			}
+		}
+	}
+	if (s) {
+		error_msg("too many weights: %d %d |%s| %d", cur_num_weights, i, s, (int)*s);
+	}
+
+	while (i) {					/* missing weights are not an error */
+		--i;
+		ci[-i] = xsymdup(STR_DITTO);
+	}
+
+	wi = xmalloc(sizeof(weighted_item_t));
+	wi->symbol = t;
+	wi->weight = register_weight(&w);
+
+	if ((i = is_ucode(t)) != 0) {
+		assert(!t[i]);
+		add_colitem(t, NULL);
+	}
+
+	return wi;
+}
+
+static void add_superset_weight(char *t)
+{
+	ll_item_t *lli;
+	weighted_item_t *wi;
+
+	if (!comm_cur_ptr
+		|| (strcmp(t, ((weighted_item_t *)(comm_cur_ptr->data))->symbol) != 0)
+		) {						/* now out of sync */
+		if (superset_in_sync) {	/* need a new section */
+			superset_in_sync = 0;
+
+			cur_section = new_section("R");
+			cur_num_weights = cur_section->num_rules
+				= ((section_t *)(cur_base->section_list->data))->num_rules;
+			memcpy(cur_rule,
+				   ((section_t *)(cur_base->section_list->data))->rules,
+				   MAX_COLLATION_WEIGHTS);
+			memcpy(cur_section->rules,
+				   ((section_t *)(cur_base->section_list->data))->rules,
+				   MAX_COLLATION_WEIGHTS);
+
+			insque(new_ll_item(DT_REORDER, cur_section), find_ll_last(cur_col->section_list));
+			assert(comm_prev_ptr);
+			lli = new_ll_item(DT_REORDER, cur_section);
+			lli->prev = lli->next = lli;
+			insque(lli, comm_prev_ptr);
+/* 			fprintf(stderr, "  subsection -----------------------\n"); */
+		}
+
+/* 		fprintf(stderr, "     %s   %s\n", t, ((weighted_item_t *)(comm_cur_ptr->data))->symbol); */
+		wi = add_weight(t);
+		lli = new_ll_item(DT_WEIGHTED, wi);
+		mark_reordered(wi->symbol);
+		/* 			printf("reorder: %s\n", t); */
+		if (!cur_section->itm_list) {
+			cur_section->itm_list = lli;
+			lli->prev = lli->next = lli;
+			++cur_section->num_items;
+		} else {
+			insque(lli, cur_section->itm_list->prev);
+			++cur_section->num_items;
+		}
+		add_wi_index(lli);
+
+	} else {					/* in sync */
+		superset_in_sync = 1;
+		next_comm_ptr();
+	}
+}
+
+static void do_weight(char *t)
+{
+	weighted_item_t *wi;
+	ll_item_t *lli;
+
+	if (superset) {
+		add_superset_weight(t);
+		return;
+	}
+
+	switch(order_state) {
+		case 0:
+/* 			printf("no-order weight: %s\n", t); */
+/* 			break; */
+		case IN_ORDER:
+			/* in a section */
+/* 			printf("weight: %s\n", t); */
+			wi = add_weight(t);
+			lli = new_ll_item(DT_WEIGHTED, wi);
+			if (!cur_section->itm_list) {
+/* 				printf("creating new item list: %s\n", wi->symbol); */
+				cur_section->itm_list = lli;
+				lli->prev = lli->next = lli;
+				++cur_section->num_items;
+			} else {
+				insque(lli, cur_section->itm_list->prev);
+/* 				printf("adding item to list: %d - %s\n", ll_len(cur_section->itm_list), wi->symbol); */
+				++cur_section->num_items;
+			}
+			add_wi_index(lli);
+			break;
+		case IN_REORDER:
+			/* std rule - but in a block with an insert-after pt */
+			wi = add_weight(t);
+			lli = new_ll_item(DT_WEIGHTED, wi);
+			mark_reordered(wi->symbol);
+/* 			printf("reorder: %s\n", t); */
+			if (!cur_section->itm_list) {
+				cur_section->itm_list = lli;
+				lli->prev = lli->next = lli;
+				++cur_section->num_items;
+			} else {
+				insque(lli, cur_section->itm_list->prev);
+				++cur_section->num_items;
+			}
+			add_wi_index(lli);
+			break;
+		case IN_REORDER_SECTIONS:
+			t = xsymdup(t);
+			if (next_token() != NULL) {
+				error_msg("trailing text in reorder section item: %s", pos);
+			}
+			lli = cur_col->section_list;
+			do {
+				if (lli->data_type & DT_SECTION) {
+					if (!strcmp(((section_t *)(lli->data))->name, t)) {
+						lli->data_type = DT_REORDER;
+						lli = new_ll_item(DT_REORDER, (section_t *)(lli->data));
+						insque(lli, reorder_section_ptr);
+						reorder_section_ptr = lli;
+						return;
+					}
+				}
+				lli = lli->next;
+			} while (lli);
+			error_msg("reorder_sections_after for non-base item currently not supported: %s", t);
+/* 			fprintf(stderr, "reorder_secitons: %s\n", t); */
+			break;
+		default:
+			error_msg("invalid order_state %d", order_state);
+	}
+}
+
+static int col_locale_cmp(const void *n1, const void *n2)
+{
+    return strcmp(((const col_locale_t *) n1)->name, ((const col_locale_t *) n2)->name);
+}
+
+static void processfile(void)
+{
+	char *t;
+	const keyword_table_t *k;
+
+	order_state = 0;
+#warning devel code
+/* 	cur_num_weights = 0; */
+/* 	cur_num_weights = 4; */
+/* 	memset(cur_rule, R_FORWARD, 4); */
+
+	if (cur_col != cur_base) {
+		cur_col->base_locale = cur_base;
+		cur_col->undefined_idx = cur_base->undefined_idx;
+		if (!cur_base->derived_list) {
+			cur_base->derived_list = new_ll_item(DT_COL_LOCALE, cur_col);
+		} else {
+			insque(new_ll_item(DT_COL_LOCALE, cur_col), find_ll_last(cur_base->derived_list));
+		}
+	}
+
+	if (tfind(cur_col, &root_col_locale, col_locale_cmp)) {
+		error_msg("attempt to readd locale: %s", cur_col->name);
+	}
+	if (!tsearch(cur_col, &root_col_locale, col_locale_cmp)) {
+		error_msg("OUT OF MEMORY!");
+	}
+
+	if (superset) {
+		superset_order_start_cnt = 0;
+		superset_in_sync = 0;
+		init_comm_ptr();
+	}
+
+	while (next_line()) {
+/* 		printf("%5d:", lineno[fno]); */
+/* 		while ((t = next_token()) != NULL) { */
+/* 			printf(" |%s|", t); */
+/* 		printf("\n"); */
+/* 		} */
+		t = next_token();
+		assert(t);
+		assert(t == pos);
+		if ((*t == '<') || (!strcmp(t, "UNDEFINED"))) {
+			do_weight(t);
+		} else {
+			for (k = keyword_table ; k->keyword ; k++) {
+				if (!strcmp(k->keyword, t)) {
+					break;
+				}
+			}
+			k->handler();
+		}
+	}
+
+	if (cur_base == cur_col) {
+		fprintf(stderr, "Base: %15s", cur_col->name);
+	} else {
+#if 1
+		if (!cur_col->undefined_idx) {
+#if 0
+			if (superset) {
+				if (superset_order_start_cnt == 1) {
+					--superset_order_start_cnt;	/* ugh.. hack this */
+				}
+			}
+#endif
+			/* This is an awful hack to get around the problem of unspecified UNDEFINED
+			 * definitions in the supported locales derived from iso14651_t1. */
+			if (!strcmp(cur_base->name, "iso14651_t1")) {
+				fprintf(stderr, "Warning: adding UNDEFINED entry for %s\n", cur_col->name);
+				strcpy(linebuf, "order_start forward;backward;forward;forward,position\n");
+				pos_e = NULL;
+				pos = linebuf;
+				t = next_token();
+				assert(t);
+				assert(t == pos);
+				do_order_start();
+				strcpy(linebuf, "UNDEFINED IGNORE;IGNORE;IGNORE\n");
+				pos_e = NULL;
+				pos = linebuf;
+				t = next_token();
+				assert(t);
+				assert(t == pos);
+				do_weight(t);
+			} else {
+				error_msg("no definition of UNDEFINED for %s", cur_col->name);
+			}
+		}
+#endif
+
+		fprintf(stderr, " Der: %15s", cur_col->name);
+	}
+	{
+		ll_item_t *p = cur_col->section_list;
+
+		fprintf(stderr, "%6u weights", tnumnodes(cur_col->root_wi_index));
+		if (cur_base) {
+			fprintf(stderr, "  %6u der %6u reor %6u starter - %u new stubs",
+					tnumnodes(cur_base->root_derived_wi),
+					tnumnodes(cur_base->root_wi_index_reordered),
+					tnumnodes(cur_base->root_starter_char),
+					ll_count(cur_col->section_list, DT_REORDER));
+		}
+		fprintf(stderr, "\n");
+
+#if 0
+		while (p) {
+			assert(((section_t *)(p->data))->num_items ==
+				   ll_len(((section_t *)(p->data))->itm_list));
+
+
+			if (!p->next &&
+				((*((section_t *)(p->data))->name == 'a')
+				 && (((section_t *)(p->data))->num_items == 0))
+				) {
+				break;
+			}
+
+			if (!(p->data_type & DT_REORDER)) {
+				if ((*((section_t *)(p->data))->name != 'a')
+					|| (((section_t *)(p->data))->num_items > 0)
+					) {
+					fprintf(stderr,
+/* 							"\t%-15s %zu\n", */
+							"\t%-15s %6u\n",
+							((section_t *)(p->data))->name,
+							((section_t *)(p->data))->num_items);
+				}
+			}
+			p = p->next;
+		}
+#endif
+	}
+
+
+}
+
+static void print_colnode(const void *ptr, VISIT order, int level)
+{
+    const colitem_t *p = *(const colitem_t **) ptr;
+
+    if (order == postorder || order == leaf)  {
+        printf("collating item = \"%s\"", p->string);
+		if (p->element) {
+			printf(" is %s", p->element);
+		}
+        printf("\n");
+    }
+}
+
+static void print_weight_node(const void *ptr, VISIT order, int level)
+{
+    const weight_t *p = *(const weight_t **) ptr;
+	int i;
+
+    if (order == postorder || order == leaf)  {
+        printf("weight: (%d)  ", p->num_weights);
+		for (i = 0 ; i < p->num_weights ; i++) {
+			if (p->rule[i] & R_FORWARD) {
+				printf("F");
+			}
+			if (p->rule[i] & R_BACKWARD) {
+				printf("B");
+			}
+			if (p->rule[i] & R_POSITION) {
+				printf("P");
+			}
+			printf(",");
+		}
+		for (i = 0 ; i < p->num_weights ; i++) {
+			printf("   %s", p->colitem[i]);
+		}
+        printf("\n");
+    }
+}
+
+
+typedef struct {
+	const char *der_name;
+	int base_locale;
+} deps_t;
+
+enum {
+	BASE_iso14651_t1,
+	BASE_comm,
+	BASE_cs_CZ,
+	BASE_ar_SA,
+	BASE_th_TH,
+	BASE_ja_JP,
+	BASE_ko_KR,
+	BASE_MAX
+};
+
+static const char *base_name[] = {
+	"iso14651_t1",
+	"comm",
+	"cs_CZ",
+	"ar_SA",
+	"th_TH",
+	"ja_JP",
+	"ko_KR"
+};
+
+
+
+static ll_item_t *locale_list[BASE_MAX];
+
+static void init_locale_list(void)
+{
+	int i;
+
+	for (i=0 ; i < BASE_MAX ; i++) {
+		locale_list[i] = (ll_item_t *) xmalloc(sizeof(ll_item_t));
+		locale_list[i]->prev = locale_list[i]->next = locale_list[i];
+		locale_list[i]->data = (void *) base_name[i];
+	}
+}
+
+
+deps_t deps[] = {
+	{ "af_ZA", BASE_iso14651_t1 },
+	{ "am_ET", BASE_iso14651_t1 },
+	{ "ar_AE", BASE_iso14651_t1 },
+	{ "ar_BH", BASE_iso14651_t1 },
+	{ "ar_DZ", BASE_iso14651_t1 },
+	{ "ar_EG", BASE_iso14651_t1 },
+	{ "ar_IN", BASE_iso14651_t1 },
+	{ "ar_IQ", BASE_iso14651_t1 },
+	{ "ar_JO", BASE_iso14651_t1 },
+	{ "ar_KW", BASE_iso14651_t1 },
+	{ "ar_LB", BASE_iso14651_t1 },
+	{ "ar_LY", BASE_iso14651_t1 },
+	{ "ar_MA", BASE_iso14651_t1 },
+	{ "ar_OM", BASE_iso14651_t1 },
+	{ "ar_QA", BASE_iso14651_t1 },
+	{ "ar_SA", BASE_ar_SA },
+	{ "ar_SD", BASE_iso14651_t1 },
+	{ "ar_SY", BASE_iso14651_t1 },
+	{ "ar_TN", BASE_iso14651_t1 },
+	{ "ar_YE", BASE_iso14651_t1 },
+	{ "az_AZ", BASE_iso14651_t1 },
+	{ "be_BY", BASE_iso14651_t1 },
+	{ "bg_BG", BASE_iso14651_t1 },
+	{ "bn_BD", BASE_iso14651_t1 },
+	{ "bn_IN", BASE_iso14651_t1 },
+	{ "br_FR", BASE_iso14651_t1 },
+	{ "bs_BA", BASE_iso14651_t1 },
+	{ "ca_ES", BASE_comm },
+	{ "cs_CZ", BASE_cs_CZ },
+	{ "cy_GB", BASE_iso14651_t1 },
+	{ "da_DK", BASE_comm },
+	{ "de_AT", BASE_iso14651_t1 },
+	{ "de_BE", BASE_iso14651_t1 },
+	{ "de_CH", BASE_iso14651_t1 },
+	{ "de_DE", BASE_iso14651_t1 },
+	{ "de_LU", BASE_iso14651_t1 },
+	{ "el_GR", BASE_iso14651_t1 },
+	{ "en_AU", BASE_iso14651_t1 },
+	{ "en_BW", BASE_iso14651_t1 },
+	{ "en_CA", BASE_comm },
+	{ "en_DK", BASE_iso14651_t1 },
+	{ "en_GB", BASE_iso14651_t1 },
+	{ "en_HK", BASE_iso14651_t1 },
+	{ "en_IE", BASE_iso14651_t1 },
+	{ "en_IN", BASE_iso14651_t1 },
+	{ "en_NZ", BASE_iso14651_t1 },
+	{ "en_PH", BASE_iso14651_t1 },
+	{ "en_SG", BASE_iso14651_t1 },
+	{ "en_US", BASE_iso14651_t1 },
+	{ "en_ZA", BASE_iso14651_t1 },
+	{ "en_ZW", BASE_iso14651_t1 },
+	{ "eo_EO", BASE_iso14651_t1 },
+	{ "es_AR", BASE_comm },
+	{ "es_BO", BASE_comm },
+	{ "es_CL", BASE_comm },
+	{ "es_CO", BASE_comm },
+	{ "es_CR", BASE_comm },
+	{ "es_DO", BASE_comm },
+	{ "es_EC", BASE_comm },
+	{ "es_ES", BASE_comm },
+	{ "es_GT", BASE_comm },
+	{ "es_HN", BASE_comm },
+	{ "es_MX", BASE_comm },
+	{ "es_NI", BASE_comm },
+	{ "es_PA", BASE_comm },
+	{ "es_PE", BASE_comm },
+	{ "es_PR", BASE_comm },
+	{ "es_PY", BASE_comm },
+	{ "es_SV", BASE_comm },
+	{ "es_US", BASE_comm },
+	{ "es_UY", BASE_comm },
+	{ "es_VE", BASE_comm },
+	{ "et_EE", BASE_comm },
+	{ "eu_ES", BASE_iso14651_t1 },
+	{ "fa_IR", BASE_iso14651_t1 },
+	{ "fi_FI", BASE_comm },
+	{ "fo_FO", BASE_comm },
+	{ "fr_BE", BASE_iso14651_t1 },
+	{ "fr_CA", BASE_comm },
+	{ "fr_CH", BASE_iso14651_t1 },
+	{ "fr_FR", BASE_iso14651_t1 },
+	{ "fr_LU", BASE_iso14651_t1 },
+	{ "ga_IE", BASE_iso14651_t1 },
+	{ "gd_GB", BASE_iso14651_t1 },
+	{ "gl_ES", BASE_comm },
+	{ "gv_GB", BASE_iso14651_t1 },
+	{ "he_IL", BASE_iso14651_t1 },
+	{ "hi_IN", BASE_iso14651_t1 },
+	{ "hr_HR", BASE_comm },
+	{ "hu_HU", BASE_iso14651_t1 },
+	{ "hy_AM", BASE_iso14651_t1 },
+	{ "id_ID", BASE_iso14651_t1 },
+	{ "is_IS", BASE_comm },
+	{ "it_CH", BASE_iso14651_t1 },
+	{ "it_IT", BASE_iso14651_t1 },
+	{ "iw_IL", BASE_iso14651_t1 },
+	{ "ja_JP", BASE_ja_JP },
+	{ "ka_GE", BASE_iso14651_t1 },
+	{ "kl_GL", BASE_comm },
+	{ "ko_KR", BASE_ko_KR },
+	{ "kw_GB", BASE_iso14651_t1 },
+	{ "lt_LT", BASE_comm },
+	{ "lv_LV", BASE_comm },
+	{ "mi_NZ", BASE_iso14651_t1 },
+	{ "mk_MK", BASE_iso14651_t1 },
+	{ "mr_IN", BASE_iso14651_t1 },
+	{ "ms_MY", BASE_iso14651_t1 },
+	{ "mt_MT", BASE_iso14651_t1 },
+	{ "nl_BE", BASE_iso14651_t1 },
+	{ "nl_NL", BASE_iso14651_t1 },
+	{ "nn_NO", BASE_iso14651_t1 },
+	{ "no_NO", BASE_comm },
+	{ "oc_FR", BASE_iso14651_t1 },
+	{ "pl_PL", BASE_comm },
+	{ "pt_BR", BASE_iso14651_t1 },
+	{ "pt_PT", BASE_iso14651_t1 },
+	{ "ro_RO", BASE_iso14651_t1 },
+	{ "ru_RU", BASE_iso14651_t1 },
+	{ "ru_UA", BASE_iso14651_t1 },
+	{ "se_NO", BASE_iso14651_t1 },
+	{ "sk_SK", BASE_cs_CZ },
+	{ "sl_SI", BASE_comm },
+	{ "sq_AL", BASE_iso14651_t1 },
+	{ "sr_YU", BASE_iso14651_t1 },
+	{ "sv_FI", BASE_comm },
+	{ "sv_SE", BASE_iso14651_t1 },
+	{ "ta_IN", BASE_iso14651_t1 },
+	{ "te_IN", BASE_iso14651_t1 },
+	{ "tg_TJ", BASE_iso14651_t1 },
+	{ "th_TH", BASE_th_TH },
+	{ "ti_ER", BASE_iso14651_t1 },
+	{ "ti_ET", BASE_iso14651_t1 },
+	{ "tl_PH", BASE_iso14651_t1 },
+	{ "tr_TR", BASE_comm },
+	{ "tt_RU", BASE_iso14651_t1 },
+	{ "uk_UA", BASE_iso14651_t1 },
+	{ "ur_PK", BASE_iso14651_t1 },
+	{ "uz_UZ", BASE_iso14651_t1 },
+	{ "vi_VN", BASE_iso14651_t1 },
+	{ "wa_BE", BASE_iso14651_t1 },
+	{ "yi_US", BASE_iso14651_t1 },
+	{ "zh_CN", BASE_iso14651_t1 },
+	{ "zh_HK", BASE_iso14651_t1 },
+	{ "zh_SG", BASE_iso14651_t1 },
+	{ "zh_TW", BASE_iso14651_t1 },
+};
+
+
+static int der_count[BASE_MAX];
+static const char *new_args[500];
+static int new_arg_count;
+
+static int dep_cmp(const void *s1, const void *s2)
+{
+	return strcmp( (const char *) s1, ((const deps_t *) s2)->der_name);
+}
+
+static int old_main(int argc, char **argv);
+
+int main(int argc, char **argv)
+{
+	const deps_t *p;
+	ll_item_t *lli;
+	int i;
+	int total;
+
+	if (argc < 2) {
+		return EXIT_FAILURE;
+	}
+
+	init_locale_list();	
+
+	while (--argc) {
+		p = (const deps_t *) bsearch(*++argv, deps, sizeof(deps)/sizeof(deps[0]), sizeof(deps[0]), dep_cmp);
+		if (!p) {
+			if (!strcmp("C", *argv)) {
+				printf("ignoring C locale\n");
+				continue;
+			} else {
+				printf("%s not found\n", *argv);
+				return EXIT_FAILURE;
+			}
+		}
+	
+		i = p->base_locale;
+		++der_count[i];
+
+		if (!strcmp(base_name[i], *argv)) {
+			/* same name as base, so skip after count incremented */
+			continue;
+		}
+		
+		/* add it to the list.  the main body will catch duplicates */
+		lli = (ll_item_t *) xmalloc(sizeof(ll_item_t));
+		lli->prev = lli->next = NULL;
+		lli->data = (void *) *argv;
+		insque(lli, locale_list[i]);
+	}
+
+	total = 0;
+	for (i=0 ; i < BASE_MAX ; i++) {
+/* 		printf("der_count[%2d] = %3d\n", i, der_count[i]); */
+		total += der_count[i];
+	}
+/* 	printf("total = %d\n", total); */
+
+	new_args[new_arg_count++] = "dummyprogramname";
+	for (i=0 ; i < BASE_MAX ; i++) {
+		if (!der_count[i]) {
+			continue;
+		}
+		new_args[new_arg_count++] = (i == BASE_comm) ? "-c" : "-b";
+		lli = locale_list[i];
+		do {
+			new_args[new_arg_count++] = (const char *) (lli->data);
+			lli = lli->next;
+		} while (lli != locale_list[i]);
+		new_args[new_arg_count++] = "-f";
+	}
+
+/* 	for (i=0 ; i < new_arg_count ; i++) { */
+/* 		printf("%3d: %s\n", i, new_args[i]); */
+/* 	} */
+
+	return old_main(new_arg_count, (char **) new_args);
+}
+
+
+/* usage...  prog -b basefile derived {derived} -s single {single} */
+
+static int old_main(int argc, char **argv)
+{
+	int next_is_base = 0;
+	int next_is_subset = 0;
+
+	superset = 0;
+
+	while (--argc) {
+		++argv;
+		if (**argv == '-') {
+			if ((*argv)[1] == 'd') {
+				dump_weights((*argv) + 2);
+			} else if ((*argv)[1] == 'f') {	/* dump all weight rules */
+				finalize_base();
+			} else if ((*argv)[1] == 'R') {	/* dump all weight rules */
+				twalk(root_weight, print_weight_node);
+			} else if (((*argv)[1] == 'c') && !(*argv)[2]) { /* new common subset */
+				cur_base = cur_derived = NULL;
+				next_is_subset = 1;
+				next_is_base = 1;
+				superset = 0;
+			} else if (((*argv)[1] == 'b') && !(*argv)[2]) { /* new base locale */
+				cur_base = cur_derived = NULL;
+				next_is_subset = 0;
+				next_is_base = 1;
+				superset = 0;
+			} else if (((*argv)[1] == 's') && !(*argv)[2]) { /* single locales follow */
+				cur_base = cur_derived = NULL;
+				next_is_subset = 0;
+				next_is_base = 2;
+				superset = 0;
+			} else {
+				error_msg("unrecognized option %s", *argv);
+			}
+			continue;
+		}
+		/* new file */
+		new_col_locale(*argv);	/* automaticly sets cur_col */
+		if (next_is_base) {
+			cur_base = cur_col;
+		} else {
+			cur_derived = cur_col;
+		}
+		pushfile(*argv);
+/* 		fprintf(stderr, "processing file %s\n", *argv); */
+		processfile();			/* this does a popfile */
+
+/* 		twalk(cur_col->root_colitem, print_colnode); */
+		
+		if (next_is_base == 1) {
+			next_is_base = 0;
+		}
+		if (next_is_subset) {
+			next_is_subset = 0;
+			superset = 1;
+		}
+	}
+
+	fprintf(stderr, "success!\n");
+	fprintf(stderr,
+/* 			"num_sym=%zu mem_sym=%zu  unique_weights=%zu\n", */
+			"num_sym=%u mem_sym=%u  unique_weights=%u\n",
+			num_sym, mem_sym, unique_weights);
+/* 	twalk(root_weight, print_weight_node); */
+
+	fprintf(stderr, "num base locales = %d    num derived locales = %d\n",
+			base_locale_len, der_locale_len);
+
+	fprintf(stderr,
+			"override_len = %d      multistart_len = %d    weightstr_len = %d\n"
+			"wcs2colidt_len = %d    index2weight_len = %d  index2ruleidx_len = %d\n"
+			"ruletable_len = %d\n"
+			"total size is %d bytes or %d kB\n",
+			override_len, multistart_len, weightstr_len,
+			wcs2colidt_len, index2weight_len, index2ruleidx_len,
+			ruletable_len,
+#warning mult by 2 for rule indecies
+			(override_len + multistart_len + weightstr_len
+			 + wcs2colidt_len + index2weight_len + index2ruleidx_len + ruletable_len) * 2,
+			(override_len + multistart_len + weightstr_len
+			 + wcs2colidt_len + index2weight_len + index2ruleidx_len + ruletable_len + 511) / 512);
+
+#if 0
+	{
+		int i;
+
+		for (i=0 ; i < base_locale_len ; i++) {
+			dump_base_locale(i);
+		}
+		for (i=0 ; i < der_locale_len ; i++) {
+			dump_der_locale(i);
+		}
+	}
+#endif
+
+	{
+		FILE *fp = fopen("locale_collate.h", "w");
+
+		if (!fp) {
+			error_msg("couldn't open output file!");
+		}
+		dump_collate(fp);
+		if (ferror(fp) || fclose(fp)) {
+			error_msg("write error or close error for output file!\n");
+		}
+	}
+
+    return EXIT_SUCCESS;
+}
+
+static void error_msg(const char *fmt, ...) 
+{
+	va_list arg;
+
+	fprintf(stderr, "Error: ");
+	if (fno >= 0) {
+	    fprintf(stderr, "file %s (%d): ", fname[fno], lineno[fno]);
+	}
+	va_start(arg, fmt);
+	vfprintf(stderr, fmt, arg);
+	va_end(arg);
+	fprintf(stderr, "\n");
+
+	exit(EXIT_FAILURE);
+}
+
+static void pushfile(char *filename)
+{
+	static fbuf[PATH_MAX];
+
+	snprintf(fbuf, PATH_MAX, "collation/%s", filename);
+
+	if (fno >= MAX_FNO) {
+		error_msg("file stack size exceeded");
+	}
+					  
+	if (!(fstack[++fno] = fopen(fbuf, "r"))) {
+		--fno;					/* oops */
+		error_msg("cannot open file %s", fbuf);
+	}
+
+	fname[fno] = xsymdup(filename);
+	lineno[fno] = 0;
+}
+
+static void popfile(void)
+{
+	if (fno < 0) {
+		error_msg("pop on empty file stack");
+	}
+
+/* 	free(fname[fno]); */
+	fclose(fstack[fno]);
+	--fno;
+}
+
+static void eatwhitespace(void)
+{
+	while (isspace(*pos)) {
+		++pos;
+	}
+}
+
+static int iscommentchar(int c)
+{
+	return ((c == '#') || (c == '%'));
+}
+
+static int next_line(void)
+{
+	size_t n;
+	char *s = linebuf;
+
+	assert(fno >= 0);
+
+	pos_e = NULL;
+	do {
+		if (fgets(s, sizeof(linebuf), fstack[fno]) != NULL) {
+			++lineno[fno];
+			n = strlen(linebuf);
+			if ((n == sizeof(linebuf) - 1) && (linebuf[n-1] != '\n')) {
+				/* Either line is too long or last line is very long with
+				 * no trailing newline.  But we'll always treat it as an
+				 * errro. */
+				error_msg("line too long?");
+			}
+
+			--n;
+			/* Be careful... last line doesn't need a newline. */
+			if (linebuf[n] == '\n') {
+				linebuf[n--] = 0;	/* trim trailing newline */
+			}
+
+			pos = linebuf;
+			eatwhitespace();
+			if (*pos && !iscommentchar(*pos)) { /* not empty or comment line */
+				return 1;		/* got a line */
+			}
+		} else {				/* eof */
+			popfile();
+		}
+	} while (fno >= 0);
+
+	return 0;
+}
+
+static char *next_token(void)
+{
+	char *p;
+
+#if 0
+	if (pos_e == NULL) {
+		return NULL
+		pos = pos_e;
+		*pos = end_of_token;
+		end_of_token = 0;
+	}
+#else
+	if (pos_e != NULL) {
+		pos = pos_e;
+		*pos = end_of_token;
+		end_of_token = 0;
+	}
+#endif
+	eatwhitespace();
+	p = pos;
+
+	if (!*p || iscommentchar(*p)) {	/* end of line or start of comment */
+		pos = pos_e = NULL;
+		*p = 0;					/* treat comment as end of line */
+/* 		fprintf(stdout, "returning NUL token |%s|\n", pos); */
+		return NULL;
+#if 1
+	} else if (*p == '<') {	 /* collating symbol, element, or value */
+		while (*++p) {
+			if ((*p == '/') && p[1]) {
+				++p;
+				continue;
+			}
+			if (*p == '>') {
+				pos_e = ++p;
+				end_of_token = *p;
+				*p = 0;
+/* 				fprintf(stdout, "returning col token |%s|\n", pos); */
+				return pos;
+			}
+		}
+	} else if (*p == '"') {		/* collating element value? */
+		while (*++p) {
+			if (*p == '"') {	/* found the end of the quoted string */
+				pos_e = ++p;
+				end_of_token = *p;
+				*p = 0;
+/* 				fprintf(stdout, "returning quote token |%s|\n", pos); */
+				return pos;
+			}
+		}
+#endif
+	} else {					/* some kind of keyword */
+		while (*++p) {
+			if (isspace(*p) || (*p == ';')) {
+				break;
+			}
+		}
+		pos_e = p;
+		end_of_token = *p;
+		*p = 0;
+/* 		fprintf(stdout, "returning key token |%s|\n", pos); */
+		return pos;
+	}
+
+	error_msg("illegal token |%s|", pos);
+}
+
+static void *xmalloc(size_t n)
+{
+	void *p;
+
+	if (!(p = malloc(n))) {
+		error_msg("OUT OF MEMORY");
+	}
+	return p;
+}
+
+static void do_copy(void)
+{
+	char *s;
+	char *e;
+
+	if ((s = next_token()) != NULL) {
+		e = strchr(s + 1, '"');
+		if ((*s == '"') && e && (*e == '"') && !e[1]) {
+			if (next_token() != NULL) {
+				error_msg("illegal trailing text: %s", pos);
+			}
+			*e = 0;
+			++s;
+			if (cur_base && !strcmp(cur_base->name,s)) {
+/* 				fprintf(stderr, "skipping copy of base file %s\n", s); */
+#warning need to update last in order and position or check
+				return;
+			}
+/* 			fprintf(stderr, "full copy of %s\n", s); */
+			pushfile(s);
+			return;
+		}
+	}
+	error_msg("illegal or missing arg for copy: %s", s);
+}
+
+static void do_colsym(void)
+{
+	char *s;
+	char *e;
+
+	if ((s = next_token()) != NULL) {
+		e = strrchr(s,'>');
+		if ((*s == '<') && e && (*e == '>') && !e[1]) {
+			if (next_token() != NULL) {
+				error_msg("illegal trailing text: %s", pos);
+			}
+			e[1] = 0; /* cleanup in case next_token stored something */
+			add_colitem(s,NULL);
+			return;
+		}
+	}
+	error_msg("illegal or missing arg for collating-symbol: %s", s);
+}
+
+static void do_colele(void)
+{
+	char *s;
+	char *e;
+	char *s1;
+	char *e1;
+	int n;
+
+	if ((s = next_token()) != NULL) {
+		e = strrchr(s,'>');
+		if ((*s == '<') && e && (*e == '>') && !e[1]) {
+			if (((s1 = next_token()) == NULL)
+				|| (strcmp(s1,"from") != 0)
+				|| ((s1 = next_token()) == NULL)
+				|| (*s1 != '\"')
+				) {
+				error_msg("illegal format for collating-element spec");
+			}
+			e1 = strchr(s1 + 1, '"');
+			if ((*s1 != '"') || !e1 || (*e1 != '"') || (e1[1] != 0)) {
+				error_msg("illegal definition for collating-element: %s", s1);
+			}
+			if (next_token() != NULL) {
+				error_msg("illegal trailing text: %s", pos);
+			}
+			e[1] = 0; /* cleanup in case next_token stored something */
+			e1[1] = 0;
+			add_colitem(s,s1);
+			++s1;
+			if (!(n = is_ucode(s1))) {
+				error_msg("starting char must be a <U####> code: %s", s1);
+			}
+			assert(s1[n] == '<');
+			s1[n] = 0;
+			s = xsymdup(s1);
+			if (!(tsearch(s, &cur_base->root_starter_char, sym_cmp))) {
+				error_msg("OUT OF MEMORY");
+			}
+
+			return;
+		}
+	}
+	error_msg("illegal or missing arg for collating-element: %s", s);
+}
+
+static ll_item_t *find_section_list_item(const char *name, col_locale_t *loc)
+{
+	ll_item_t *p;
+
+	if (!loc) {
+		return NULL;
+	}
+
+	p = loc->section_list;
+
+	while (p) {
+#warning devel code
+/* 		if (!((p->data_type == DT_SECTION) || (p->data_type == DT_REORDER))) { */
+/* 			fprintf(stderr, "fsli = %d\n", p->data_type); */
+/* 		} */
+		assert((p->data_type == DT_SECTION) || (p->data_type == DT_REORDER));
+		if (!strcmp(name, ((section_t *)(p->data))->name)) {
+			break;
+		}
+		p = p->next;
+	}
+	return p;
+}
+
+static ll_item_t *find_ll_last(ll_item_t *p)
+{
+	assert(p);
+
+	while (p->next) {
+		p = p->next;
+	}
+	return p;
+}
+
+static void do_script(void)
+{
+	char *s;
+	char *e;
+
+	if ((s = next_token()) != NULL) {
+		e = strrchr(s,'>');
+		if ((*s == '<') && e && (*e == '>') && !e[1]) {
+			if (next_token() != NULL) {
+				error_msg("illegal trailing text: %s", pos);
+			}
+			e[1] = 0; /* cleanup in case next_token stored something */
+			add_script(s);
+			return;
+		}
+	}
+	error_msg("illegal or missing arg for script: %s", s);
+}
+
+static col_locale_t *new_col_locale(char *name)
+{
+	ll_item_t *lli;
+	ll_item_t *lli2;
+
+	cur_col = (col_locale_t *) xmalloc(sizeof(col_locale_t));
+	cur_col->name = name;
+	cur_col->root_colitem = NULL;
+	cur_col->root_element = NULL;
+	cur_col->root_scripts = NULL;
+	cur_col->base_locale = NULL;
+	if (!superset) {
+		/* start with an anonymous section */
+		cur_section = new_section(NULL);
+		cur_col->section_list = new_ll_item(DT_SECTION, cur_section);
+	} else {
+		/* start with a reorder section */
+		cur_section = new_section("R");
+		cur_num_weights = cur_section->num_rules
+			= ((section_t *)(cur_base->section_list->data))->num_rules;
+		memcpy(cur_rule,
+			   ((section_t *)(cur_base->section_list->data))->rules,
+			   MAX_COLLATION_WEIGHTS);
+		memcpy(cur_section->rules,
+			   ((section_t *)(cur_base->section_list->data))->rules,
+			   MAX_COLLATION_WEIGHTS);
+		cur_col->section_list = new_ll_item(DT_REORDER, cur_section);
+		assert(cur_base->section_list->next == NULL); /* currently only one section allowed */
+		lli = ((section_t *)(cur_base->section_list->data))->itm_list;
+		assert(lli);
+		lli2 = new_ll_item(DT_REORDER, cur_section);
+		lli2->prev = lli2->next = lli2;
+		insque(lli2, lli->prev);
+		((section_t *)(cur_base->section_list->data))->itm_list = lli2;
+	}
+/* 	cur_col->section_list = NULL; */
+/* 	add_script(((section_t *)(cur_col->section_list->data))->name); */
+	cur_col->root_wi_index = NULL;
+	cur_col->root_wi_index_reordered = NULL;
+	cur_col->root_derived_wi = NULL;
+	cur_col->derived_list = NULL;
+	cur_col->root_starter_char = NULL;
+	cur_col->root_starter_all = NULL;
+	cur_col->undefined_idx = NULL;
+	return cur_col;
+}
+
+static int colitem_cmp(const void *n1, const void *n2)
+{
+    return strcmp(((colitem_t *)n1)->string, ((colitem_t *)n2)->string);
+}
+
+static int colelement_cmp(const void *n1, const void *n2)
+{
+    int r;
+
+    r = strcmp(((colitem_t *)n1)->string, ((colitem_t *)n2)->string);
+    if (!r) {
+		if (((colitem_t *)n1)->element && ((colitem_t *)n2)->element) {
+			r = strcmp(((colitem_t *)n1)->element, ((colitem_t *)n2)->element);
+		} else if (((colitem_t *)n1)->element == ((colitem_t *)n2)->element) {
+			r = 0;				/* both null */
+		} else {
+			r = (((colitem_t *)n1)->element == NULL) ? -1 : 1;
+		} 
+    }
+    return r;
+}
+
+static void del_colitem(colitem_t *p)
+{
+/*     free((void *) p->element); */
+/*     free((void *) p->string); */
+    free(p);
+}
+
+static colitem_t *new_colitem(char *item, char *def)
+{
+	colitem_t *p;
+
+	p = xmalloc(sizeof(colitem_t));
+	p->string = xsymdup(item);
+	p->element = (!def) ? def : xsymdup(def);
+
+	return p;
+}
+
+static void add_colitem(char *item, char *def)
+{
+	colitem_t *p;
+
+#if 0
+	printf("adding collation item %s", item);
+	if (def) {
+		printf(" with definition %s", def);
+	}
+	printf("\n");
+#endif
+
+	p = new_colitem(item, def);
+
+#warning devel code
+	if (superset) {
+		if (tfind(p, &cur_base->root_colitem, colitem_cmp)) {
+/* 			fprintf(stderr, "skipping superset duplicate collating item \"%s\"\n", p->string); */
+			del_colitem(p);
+			return;
+/* 		} else { */
+/* 			fprintf(stderr, "superset: new collating item \"%s\" = %s\n", p->string, p->element); */
+		}
+	}
+
+	if (cur_col == cur_derived) {
+		if (!tfind(p, &cur_base->root_colitem, colitem_cmp)) {
+			/* not in current but could be in base */
+			if (!tsearch(p, &cur_base->root_colitem, colitem_cmp)) {
+				error_msg("OUT OF MEMORY!");
+			}
+		} else if (!tfind(p,  &cur_base->root_colitem, colelement_cmp)) {
+			error_msg("collating element/symbol mismatch: item=%s def=%s", item, def);
+		}
+	}
+
+
+	if (!tfind(p, &cur_col->root_colitem, colitem_cmp)) {
+		/* not in current but could be in base */
+		if (!tsearch(p, &cur_col->root_colitem, colitem_cmp)) {
+			error_msg("OUT OF MEMORY!");
+		}
+	} else if (!tfind(p,  &cur_col->root_colitem, colelement_cmp)) {
+		error_msg("collating element/symbol mismatch");
+	} else {					/* already there */
+		fprintf(stderr, "duplicate collating item \"%s\"\n", p->string);
+		del_colitem(p);
+	}
+}
+
+/* add a script (section) to the current locale */
+static void add_script(const char *s)
+{
+	ll_item_t *l;
+
+	/* make sure it isn't in base if working with derived */
+	if (cur_base != cur_col) {
+		if (find_section_list_item(s, cur_base)) {
+			error_msg("attempt to add script %s for derived when already in base", s);
+		}
+	}
+
+	if (find_section_list_item(s, cur_col)) {
+		error_msg("attempt to readd script %s", s);
+	}
+	
+	l = find_ll_last(cur_col->section_list);
+	insque(new_ll_item(DT_SECTION, new_section(s)), l);
+}
+
+static const char str_forward[] =  "forward";
+static const char str_backward[] = "backward";
+static const char str_position[] = "position";
+
+static void do_order_start(void)
+{
+	const char *s;
+	char *e;
+	ll_item_t *l;
+	section_t *sect;
+	int rule;
+
+	if (order_state & ~IN_ORDER) {
+		error_msg("order_start following reorder{_sections}_after");
+	}
+	order_state |= IN_ORDER;
+
+	if (superset) {
+		if (++superset_order_start_cnt > 1) {
+			error_msg("currently only a common order_start is supported in superset");
+		}
+		return;
+	}
+
+	if (!(s = next_token())) {
+		s = str_forward;		/* if no args */
+	}
+
+	if (*s == '<') {		/* section (script) */
+		e = strrchr(s,'>');
+		if ((*s == '<') && e && (*e == '>') && !e[1]) {
+			e[1] = 0; /* cleanup in case next_token stored something */
+
+			if (!(l = find_section_list_item(s, cur_col))) {
+				error_msg("ref of undefined sections: %s", s);
+			}
+			sect = (section_t *)(l->data);
+			if (sect->num_rules) {
+				error_msg("sections already defined: %s", s);
+			}
+		} else {
+			error_msg("illegal section ref: %s", s);
+		}
+
+		if (!(s = next_token())) {
+			s = str_forward;		/* if no args */
+		} else if (*s != ';') {
+			error_msg("missing seperator!");
+		}
+	} else {					/* need an anonymous section */
+		if ((*cur_section->name != '<') && (cur_section->num_items == 0)) { /* already in an empty anonymous section */
+			sect = cur_section;
+		} else {
+			sect = new_section(NULL);
+			l = find_ll_last(cur_col->section_list);
+			insque(new_ll_item(DT_SECTION, new_section(s)), l);
+		}
+		sect->num_rules = 0;	/* setting this below so nix default */
+	}
+	cur_section = sect;
+
+#warning need to add section to weight list?
+
+	/* now do rules */
+	do {
+		rule = 0;
+		if (*s == ';') {
+			++s;
+		}
+		while (*s) {
+			if (!strncmp(str_forward, s, 7)) {
+				rule |= R_FORWARD;
+				s += 7;
+			} else if (!strncmp(str_backward, s, 8)) {
+				rule |= R_BACKWARD;
+				s += 8;
+			} else if (!strncmp(str_position, s, 8)) {
+				rule |= R_POSITION;
+				s += 8;
+			}
+
+			if (*s == ',') {
+				++s;
+				continue;
+			}
+
+			if (!*s || (*s == ';')) {
+				if (sect->num_rules >= MAX_COLLATION_WEIGHTS) {
+					error_msg("more than %d weight rules!", MAX_COLLATION_WEIGHTS);
+				}
+				if (!rule) {
+					error_msg("missing weight rule!");
+				}
+				if ((rule & (R_FORWARD|R_BACKWARD|R_POSITION)) > R_BACKWARD) {
+					error_msg("backward paired with  forward and/or position!");
+				}
+
+				sect->rules[sect->num_rules++] = rule;
+				rule = 0;
+				continue;
+			}
+
+			error_msg("illegal weight rule: %s", s);
+		}
+	} while ((s = next_token()) != NULL);
+
+	cur_section = sect;
+
+/* 	fprintf(stderr, "setting cur_num_weights to %d for %s\n", sect->num_rules, sect->name); */
+	cur_num_weights = sect->num_rules;
+	memcpy(cur_rule, sect->rules, MAX_COLLATION_WEIGHTS);
+}
+
+static void do_order_end(void)
+{
+	if (!(order_state & IN_ORDER)) {
+		error_msg("order_end with no matching order_start");
+	}
+	order_state &= ~IN_ORDER;
+
+	cur_section = new_section(NULL);
+}
+
+static void do_reorder_after(void)
+{
+	char *t;
+	ll_item_t *lli;
+	const weight_t *w;
+	int save_cur_num_weights;
+	char save_cur_rule[MAX_COLLATION_WEIGHTS];
+
+
+	if (order_state & ~IN_REORDER) {
+		error_msg("reorder_after following order_start or reorder_sections_after");
+	}
+	order_state |= IN_REORDER;
+
+	if (superset) {
+		error_msg("currently reorder_after is not supported in supersets");
+	}
+
+#warning have to use rule for current section!!!
+
+	if (!(t = next_token())) {
+		error_msg("missing arg for reorder_after");
+	}
+
+	t = xsymdup(t);
+
+	if (next_token() != NULL) {
+		error_msg("trailing text reorder_after: %s", pos);
+	}
+
+	if (cur_col == cur_base) {
+		error_msg("sorry.. reorder_after in base locale is not currently supported");
+	}
+
+	if (!(lli = find_wi_index(t, cur_base))) {
+		error_msg("reorder_after for non-base item currently not supported: %s", t);
+	}
+
+	w = ((weighted_item_t *)(lli->data))->weight;
+
+
+	save_cur_num_weights = cur_num_weights;
+	memcpy(save_cur_rule, cur_rule, MAX_COLLATION_WEIGHTS);
+
+	cur_section = new_section("R");
+	insque(new_ll_item(DT_REORDER, cur_section), lli);
+
+#if 0
+
+	{
+		ll_item_t *l1;
+		ll_item_t *l2;
+		ll_item_t *l3;
+		l1 = new_ll_item(DT_REORDER, cur_section);
+		l2 = find_ll_last(cur_col->section_list);
+		insque(l1, l2);
+		l3 = find_ll_last(cur_col->section_list);
+
+		fprintf(stderr, "reorder_after %p %p %p %s\n", l1, l2, l3, cur_section->name);
+	}
+#else
+	insque(new_ll_item(DT_REORDER, cur_section), find_ll_last(cur_col->section_list));
+#endif
+
+	cur_num_weights = cur_section->num_rules = save_cur_num_weights;
+	memcpy(cur_rule, save_cur_rule, MAX_COLLATION_WEIGHTS);
+	memcpy(cur_section->rules, save_cur_rule, MAX_COLLATION_WEIGHTS);
+
+
+#warning devel code
+/* 	fprintf(stderr, "reorder -- %s %d\n", ((weighted_item_t *)(lli->data))->symbol, w->num_weights); */
+
+#warning hack to get around hu_HU reorder-after problem
+/* 	if (!w->num_weights) { */
+
+/* 	} else { */
+/* 		cur_num_weights = w->num_weights; */
+/* 		memcpy(cur_rule, w->rule, MAX_COLLATION_WEIGHTS); */
+/* 	}	 */
+
+/* 	fprintf(stderr, "reorder_after succeeded for %s\n", t); */
+}
+
+static void do_reorder_end(void)
+{
+	if (!(order_state & IN_REORDER)) {
+		error_msg("reorder_end with no matching reorder_after");
+	}
+	order_state &= ~IN_REORDER;
+}
+
+static void do_reorder_sections_after(void)
+{
+	const char *t;
+	ll_item_t *lli;
+
+	if (order_state & ~IN_REORDER_SECTIONS) {
+		error_msg("reorder_sections_after following order_start or reorder_after");
+	}
+	order_state |= IN_REORDER_SECTIONS;
+
+	if (superset) {
+		error_msg("currently reorder_sections_after is not supported in supersets");
+	}
+
+	if (!(t = next_token())) {
+		error_msg("missing arg for reorder_sections_after");
+	}
+
+	t = xsymdup(t);
+
+	if (next_token() != NULL) {
+		error_msg("trailing text reorder_sections_after: %s", pos);
+	}
+
+	if (cur_col == cur_base) {
+		error_msg("sorry.. reorder_sections_after in base locale is not currently supported");
+	}
+
+	lli = cur_base->section_list;
+	do {
+/* 		fprintf(stderr, "hmm -- |%s|%d|\n", ((section_t *)(lli->data))->name, lli->data_type); */
+		if (lli->data_type & DT_SECTION) {
+/* 			fprintf(stderr, "checking |%s|%s|\n", ((section_t *)(lli->data))->name, t); */
+			if (!strcmp(((section_t *)(lli->data))->name, t)) {
+				reorder_section_ptr = lli;
+				return;
+			}
+		}
+		lli = lli->next;
+	} while (lli);
+
+	error_msg("reorder_sections_after for non-base item currently not supported: %s", t);
+}
+
+static void do_reorder_sections_end(void)
+{
+	if (!(order_state & IN_REORDER_SECTIONS)) {
+		error_msg("reorder_sections_end with no matching reorder_sections_after");
+	}
+	order_state &= ~IN_REORDER_SECTIONS;
+
+	reorder_section_ptr = NULL;
+}
+
+static ll_item_t *new_ll_item(int data_type, void *data)
+{
+	ll_item_t *p;
+
+	p = xmalloc(sizeof(ll_item_t));
+	p->next = p->prev = NULL;
+	p->data_type = data_type;
+	p->data = data;
+	p->idx = INT_MIN;
+
+	return p;
+}
+
+static int sym_cmp(const void *n1, const void *n2)
+{
+/* 	fprintf(stderr, "sym_cmp: |%s| |%s|\n", (const char *)n1, (const char *)n2); */
+    return strcmp((const char *) n1, (const char *) n2);
+}
+
+static char *xsymdup(const char *s)
+{
+	void *p;
+
+	if (!(p = tfind(s, &root_sym, sym_cmp))) { /* not a currently known symbol */
+		if (!(s = strdup(s)) || !(p = tsearch(s, &root_sym, sym_cmp))) {
+			error_msg("OUT OF MEMORY!");
+		}
+		++num_sym;
+		mem_sym += strlen(s) + 1;
+/* 		fprintf(stderr, "xsymdup: alloc |%s| %p |%s| %p\n", *(char **)p, p, s, s); */
+/* 	} else { */
+/* 		fprintf(stderr, "xsymdup: found |%s| %p\n", *(char **)p, p); */
+	}
+	return *(char **) p;
+}
+
+static int weight_cmp(const void *n1, const void *n2)
+{
+	const weight_t *w1 = (const weight_t *) n1;
+	const weight_t *w2 = (const weight_t *) n2;
+	int i, r;
+
+	if (w1->num_weights != w2->num_weights) {
+		return w1->num_weights - w2->num_weights;
+	}
+
+	for (i=0 ; i < w1->num_weights ; i++) {
+		if (w1->rule[i] != w2->rule[i]) {
+			return w1->rule[i] - w2->rule[i];
+		}
+		if ((r = strcmp(w1->colitem[i], w2->colitem[i])) != 0) {
+			return r;
+		}
+	}
+	return 0;
+}
+
+static weight_t *register_weight(weight_t *w)
+{
+	void *p;
+
+	if (!(p = tfind(w, &root_weight, weight_cmp))) { /* new weight */
+		p = xmalloc(sizeof(weight_t));
+		memcpy(p, w, sizeof(weight_t));
+		if (!(p = tsearch(p, &root_weight, weight_cmp))) {
+			error_msg("OUT OF MEMORY!");
+		}
+		++unique_weights;
+/* 	} else { */
+/* 		fprintf(stderr, "rw: found\n"); */
+	}
+	return *(weight_t **)p;
+}
+
+static size_t ll_len(ll_item_t *l)
+{
+	size_t n = 0;
+	ll_item_t *p = l;
+
+	while (p) {
+		++n;
+		p = p->next;
+		if (p == l) {			/* work for circular too */
+			break;
+		}
+	}
+	return n;
+}
+
+static size_t ll_count(ll_item_t *l, int mask)
+{
+	size_t n = 0;
+	ll_item_t *p = l;
+
+	while (p) {
+		if (p->data_type & mask) {
+			++n;
+		}
+		p = p->next;
+		if (p == l) {			/* work for circular too */
+			break;
+		}
+	}
+	return n;
+}
+
+
+static int wi_index_cmp(const void *n1, const void *n2)
+{
+	const char *s1 = ((weighted_item_t *)(((ll_item_t *) n1)->data))->symbol;
+	const char *s2 = ((weighted_item_t *)(((ll_item_t *) n2)->data))->symbol;
+
+    return strcmp(s1, s2);
+}
+
+static void add_wi_index(ll_item_t *l)
+{
+	assert(l->data_type == DT_WEIGHTED);
+
+	if (!strcmp(((weighted_item_t *)(l->data))->symbol, "UNDEFINED")) {
+		cur_col->undefined_idx = l;
+	}
+
+	if (!tfind(l, &cur_col->root_wi_index, wi_index_cmp)) { /* new wi_index */
+		if (!tsearch(l, &cur_col->root_wi_index, wi_index_cmp)) {
+			error_msg("OUT OF MEMORY!");
+		}
+	}
+
+	if (cur_base != cur_col) {
+		if (!tfind(l, &cur_base->root_wi_index, wi_index_cmp)) {/* not a base val */
+/* 			printf("derived: %s\n", ((weighted_item_t *)(l->data))->symbol); */
+			if (!tfind(l, &cur_base->root_derived_wi, wi_index_cmp)) { /* new derived */
+				if (!tsearch(l, &cur_base->root_derived_wi, wi_index_cmp)) {
+					error_msg("OUT OF MEMORY!");
+				}
+			}
+		}
+	}
+}
+
+static int final_index;
+
+
+static int is_ucode(const char *s)
+{
+	if ((s[0] == '<')
+		&& (s[1] == 'U')
+		&& isxdigit(s[2])
+		&& isxdigit(s[3])
+		&& isxdigit(s[4])
+		&& isxdigit(s[5])
+		&& (s[6] == '>')
+		) {
+		return 7;
+	} else {
+		return 0;
+	}
+}
+
+static void add_final_col_index(const char *s)
+{
+	ENTRY e;
+
+	e.key = (char *) s;
+	e.data = (void *)(final_index);
+	if (!hsearch(e, FIND)) {	/* not in the table */
+		if (!hsearch(e, ENTER)) {
+			error_msg("OUT OF MEMORY! (hsearch)");
+		}
+#if 0
+		{
+			int n;
+			void *v;
+			colitem_t ci;
+			colitem_t *p;
+			const char *t;
+
+			if (!strcmp(s, "UNDEFINED")) {
+				printf("%6d: %s\n", final_index, s);
+			} else {
+				assert(*s == '<');
+				if ((n = is_ucode(s)) != 0) {
+					assert(!s[n]);
+					printf("%6d: %s\n", final_index, s);
+				} else {
+					ci.string = (char *) s;
+					ci.element = NULL; /* don't care */
+					v = tfind(&ci, &cur_base->root_colitem, colitem_cmp);
+					if (!v) {
+						fprintf(stderr, "%s  NOT DEFINED!!!\n", s);
+					} else {
+						p = *((colitem_t **) v);
+						if (p->element != NULL) {
+							t = p->element;
+							assert(*t == '"');
+							++t;
+							n = is_ucode(t);
+							assert(n);
+							printf("%6d: %.*s | ", final_index, n, t);
+							do {
+								t += n;
+								assert(*t);
+								if (*t == '"') {
+									assert(!t[1]);
+									break;
+								}
+								n = is_ucode(t);
+								assert(n);
+								printf("%.*s", n, t);
+							} while (1);
+							printf("   collating-element %s\n", s);
+						} else {
+							printf("%6d: %s  (collating-symbol)\n", final_index, s);
+						}
+					}
+				}
+			}
+		}
+#endif
+		++final_index;
+	}
+
+}
+
+static int final_index_val0(const char *s)
+{
+	ENTRY *p;
+	ENTRY e;
+	e.key = (char *) s;
+
+	if (!(p = hsearch(e, FIND))) {	/* not in the table */
+		return 0;
+	}
+
+	return (int)(p->data);
+}
+
+static int final_index_val(const char *s)
+{
+	ENTRY *p;
+	ENTRY e;
+	e.key = (char *) s;
+
+	if (!(p = hsearch(e, FIND))) {	/* not in the table */
+		error_msg("can't find final index: %s", s);
+	}
+
+	return (int)(p->data);
+}
+
+static size_t num_tree_nodes;
+
+static void count_nodes(const void *ptr, VISIT order, int level)
+{
+    if ((order == postorder) || (order == leaf))  {
+		++num_tree_nodes;
+    }
+}
+
+static size_t tnumnodes(const void *root)
+{
+	num_tree_nodes = 0;
+
+	twalk(root, count_nodes);
+
+	return num_tree_nodes;
+
+}
+
+static ll_item_t *find_wi_index(const char *sym, col_locale_t *cl)
+{
+	weighted_item_t w;
+	ll_item_t l;
+	void *p;
+
+	w.symbol = sym;
+	l.data = &w;
+	l.data_type = DT_WEIGHTED;
+
+	p = tfind(&l, &cl->root_wi_index, wi_index_cmp);
+
+	if (p) {
+		p = *(ll_item_t **)p;
+	}
+
+	return (ll_item_t *) p;
+}
+
+static void mark_reordered(const char *sym)
+{
+	ll_item_t *lli;
+
+	lli = find_wi_index(sym, cur_base);
+
+	if (lli) {
+		if (!tsearch(lli, &cur_base->root_wi_index_reordered, wi_index_cmp)) {
+			error_msg("OUT OF MEMORY!");
+		}
+	}
+}
+
+static ll_item_t *find_wi_index_reordered(const char *sym)
+{
+	weighted_item_t w;
+	ll_item_t l;
+	void *p;
+
+	w.symbol = sym;
+	l.data = &w;
+	l.data_type = DT_WEIGHTED;
+
+	p = tfind(&l, &cur_base->root_wi_index_reordered, wi_index_cmp);
+
+	if (p) {
+		p = *(ll_item_t **)p;
+	}
+
+	return (ll_item_t *) p;
+}
+
+static ll_item_t *init_comm_ptr(void)
+{
+	assert(cur_base);
+	assert(cur_base->section_list);
+	/* at the moment, only support one section in comm */
+	assert(cur_base->section_list->next == NULL);
+
+	comm_cur_ptr = ((section_t *)(cur_base->section_list->data))->itm_list;
+
+	while (comm_cur_ptr && (comm_cur_ptr->data_type & DT_REORDER)) {
+		comm_cur_ptr = comm_cur_ptr->next;
+	}
+
+#warning devel code
+/* 	{ */
+/* 		ll_item_t *p = comm_cur_ptr; */
+/* 		fprintf(stderr, "init_comm_ptr\n"); */
+
+/* 		while (p != comm_cur_ptr) { */
+/* 			if (p->data_type & DT_WEIGHTED) { */
+/* 				fprintf(stderr, "%s", ((weighted_item_t *)p)->symbol); */
+/* 			} */
+/* 			p = p->next; */
+/* 		} */
+/* 	} */
+
+	assert(comm_cur_ptr);
+
+/* 	fprintf(stderr, "init_comm_ptr -- %s %p %p %p %d\n", */
+/* 			((weighted_item_t *)(comm_cur_ptr->data))->symbol, */
+/* 			comm_cur_ptr, comm_cur_ptr->prev, comm_cur_ptr->next, */
+/* 			ll_len(comm_cur_ptr)); */
+
+	comm_prev_ptr = NULL;
+	return comm_cur_ptr;
+}
+
+static ll_item_t *next_comm_ptr(void)
+{
+	/* at the moment, only support one section in comm */
+	assert(cur_base->section_list->next == NULL);
+
+	comm_prev_ptr = comm_cur_ptr;
+
+    while (comm_cur_ptr && ((comm_cur_ptr = comm_cur_ptr->next) != NULL)) {
+		if (!(comm_cur_ptr->data_type & DT_REORDER)) {
+			break;
+		}
+	}
+
+	return comm_cur_ptr;
+}
+
+static int dump_count;
+
+#if 0
+static void dump_section(section_t *s, int mask, col_locale_t *der)
+{
+	ll_item_t *lli;
+	ll_item_t *lli0;
+	weighted_item_t *w;
+	weight_t *p;
+	int i;
+
+	lli0 = lli = s->itm_list;
+
+	if (!lli0) {
+		return;
+	}
+
+	do {
+		if (!(lli->data_type & mask)) {
+			lli = lli->next;
+			continue;
+		}
+		if (lli->data_type & DT_WEIGHTED) {
+			++dump_count;
+			w = (weighted_item_t *)(lli->data);
+			p = w->weight;
+			printf("%6d: %s (%d) ", dump_count, w->symbol, p->num_weights);
+			for (i = 0 ; i < p->num_weights ; i++) {
+				if (p->rule[i] & R_FORWARD) {
+					printf("F");
+				}
+				if (p->rule[i] & R_BACKWARD) {
+					printf("B");
+				}
+				if (p->rule[i] & R_POSITION) {
+					printf("P");
+				}
+				printf(",");
+			}
+			for (i = 0 ; i < p->num_weights ; i++) {
+				printf("   %s", p->colitem[i]);
+			}
+			printf("\n");
+		} else if (lli->data_type & (DT_SECTION|DT_REORDER)) {
+
+			if (lli->data_type == DT_REORDER) {
+				assert(der);
+				if (strncmp(((section_t *)(lli->data))->name, der->name, strlen(der->name))) {
+					lli = lli->next;
+					continue;
+				}
+			}
+
+			if (lli->data_type & DT_SECTION) {
+				printf("SECTION -----------------\n");
+			} else {
+				printf("REORDER -----------------\n");
+			}
+
+			dump_section((section_t *)(lli->data), mask, der);
+			printf("DONE --------------------\n");
+		}
+		lli = lli->next;
+	} while (lli != lli0);
+}
+#else
+static int in_reorder_section = 0;
+
+static void dump_section(section_t *s, int mask, col_locale_t *der)
+{
+	ll_item_t *lli;
+	ll_item_t *lli0;
+	weighted_item_t *w;
+	weight_t *p;
+	int i;
+
+	lli0 = lli = s->itm_list;
+
+	if (!lli0) {
+		return;
+	}
+
+	do {
+		if (!(lli->data_type & mask)) {
+			lli = lli->next;
+			continue;
+		}
+		if (lli->data_type & DT_WEIGHTED) {
+			++dump_count;
+			w = (weighted_item_t *)(lli->data);
+			p = w->weight;
+#if 1
+			if (in_reorder_section) {
+				printf(" %p", w);
+			}
+#else
+			printf("%6d: %s (%d) ", dump_count, w->symbol, p->num_weights);
+			for (i = 0 ; i < p->num_weights ; i++) {
+				if (p->rule[i] & R_FORWARD) {
+					printf("F");
+				}
+				if (p->rule[i] & R_BACKWARD) {
+					printf("B");
+				}
+				if (p->rule[i] & R_POSITION) {
+					printf("P");
+				}
+				printf(",");
+			}
+			for (i = 0 ; i < p->num_weights ; i++) {
+				printf("   %s", p->colitem[i]);
+			}
+			printf("\n");
+#endif
+		} else if (lli->data_type & (DT_SECTION|DT_REORDER)) {
+
+			if (lli->data_type == DT_REORDER) {
+				assert(der);
+				if (strncmp(((section_t *)(lli->data))->name, der->name, strlen(der->name))) {
+					lli = lli->next;
+					continue;
+				}
+			}
+
+			if (lli->data_type & DT_SECTION) {
+/* 				printf("SECTION -----------------\n"); */
+				assert(0);
+			} else {
+/* 				printf("REORDER -----------------\n"); */
+				in_reorder_section = 1;
+			}
+
+			dump_section((section_t *)(lli->data), mask, der);
+/* 			printf("DONE --------------------\n"); */
+			printf("\n");
+			in_reorder_section = 0;
+		}
+		lli = lli->next;
+	} while (lli != lli0);
+}
+#endif
+
+static void dump_weights(const char *name)
+{
+	ll_item_t *lli;
+	col_locale_t *base;
+	col_locale_t *der;
+	col_locale_t cl;
+	void *p;
+
+	assert(name);
+
+	if (!*name) {				/* use last */
+		base = cur_base;
+		der = cur_derived;
+	} else {
+		cl.name = (char *) name;
+		if (!(p = tfind(&cl, &root_col_locale, col_locale_cmp))) {
+			error_msg("unknown locale: %s", name);
+		}
+		base = *((col_locale_t **) p);
+		der = NULL;
+		if (base->base_locale) { /* oops... really derived */
+			der = base;
+			base = der->base_locale;
+		}
+	}
+
+	dump_count = 0;
+
+	if (base) {
+/* 		printf("BASE - %s\n", base->name); */
+		for (lli = base->section_list ; lli ; lli = lli->next) {
+/* 			printf("SECTION %s\n", ((section_t *)(lli->data))->name); */
+			dump_section((section_t *)(lli->data), ~0, der);
+		}
+	}
+
+	assert(der != base);
+
+	if (der) {
+/* 		printf("DERIVED - %s\n", der->name); */
+		for (lli = der->section_list ; lli ; lli = lli->next) {
+			if (lli->data_type == DT_SECTION) {
+				dump_section((section_t *)(lli->data), DT_WEIGHTED, der);
+			}
+		}
+	}
+/* 	printf("DONE\n"); */
+}
+
+static void print_starter_node(const void *ptr, VISIT order, int level)
+{
+    if (order == postorder || order == leaf)  {
+		fprintf(stderr, "   %s\n", *(const char **) ptr);
+    }
+}
+
+static void finalize_base(void)
+{
+	ll_item_t *s;
+	ll_item_t *h;
+	ll_item_t *lli;
+	ll_item_t *h2;
+	ll_item_t *l2;
+	ll_item_t *cli;
+	ll_item_t *rli = NULL;
+	weighted_item_t *w;
+	weight_t *p;
+	int i, n, mr, r, mi;
+	col_locale_t *cl;
+	void *mm;
+
+	int num_invariant = 0;
+	int num_varying = 0;
+	int max_weight;
+	int index2weight_len_inc = 1;
+
+	assert(cur_base);
+	assert(base_locale_len+1 < BASE_LOCALE_LEN);
+
+	base_locale_array[base_locale_len].name = cur_base->name;
+	base_locale_array[base_locale_len].num_weights = 1;
+	base_locale_array[base_locale_len].index2weight_offset = index2weight_len;
+	base_locale_array[base_locale_len].index2ruleidx_offset = index2ruleidx_len;
+	if (!strcmp(cur_base->name,"ja_JP") || !strcmp(cur_base->name,"ko_KR")) {
+#warning fix the index2weight check!!
+		index2weight_len_inc = 0;
+	}
+/* 	printf("%s -- index2weight_len = %d\n", cur_base->name, index2weight_len); */
+
+	if (!hcreate(30000)) {
+		error_msg("OUT OF MEMORY!");
+	}
+
+	/* first pass ... set the fixed indexes */
+	final_index = i = 1;
+	mr = 0;
+	for (s = cur_base->section_list ; s ; s = s->next) {
+#if 1
+		if (s->data_type & DT_REORDER) { /* a reordered section */
+			fprintf(stderr, "pass1: reordered section %s - xxx\n", ((section_t *)(s->data))->name);
+			lli = ((section_t *)(s->data))->itm_list;
+			r = 0;
+			if (lli) {
+/* 				r = ll_len( ((section_t *)(lli->data))->itm_list ); */
+				r = ll_len(lli) + 1;
+			}
+			if (r > mr) {
+				mr = r;
+			}
+			fprintf(stderr, "pass1: reordered section %s - %d\n", ((section_t *)(s->data))->name, r);
+			continue;
+		}
+#endif
+		h = lli = ((section_t *)(s->data))->itm_list;
+		if (!lli) {
+			continue;
+		}
+		do {
+			if (lli->data_type & DT_RANGE) {
+				i += mr;
+				mr = 0;
+#warning check ko_kR and 9
+/* 				++i; */
+				lli->idx = i;
+				assert(!rli);
+				rli = lli;
+				fprintf(stderr, "range pre = %d  after = ", i);
+				i += ((range_item_t *)(lli->data))->length + 1;
+#warning check ko_kR and 9
+/* 				++i; */
+				fprintf(stderr, "%d\n", i);
+				if (!index2weight_len_inc) { /* ko_KR hack */
+					final_index += ((range_item_t *)(lli->data))->length + 1;
+				}
+/* 				add_final_col_index("RANGE"); */
+			} else if (lli->data_type & DT_WEIGHTED) {
+				i += mr;
+				mr = 0;
+				w = (weighted_item_t *)(lli->data);
+				if (find_wi_index_reordered(w->symbol)) { /* reordered symbol so skip on first pass */
+					++num_varying;
+					++i;
+					continue;
+				}
+				++num_invariant;
+				index2weight_buffer[index2weight_len] = lli->idx = i++;
+				index2weight_len += index2weight_len_inc;
+				add_final_col_index(w->symbol);
+				
+			} else {
+				assert(lli->data_type & DT_REORDER);
+				r = ll_len( ((section_t *)(lli->data))->itm_list );
+#warning check ko_kR and 9
+				if (r > mr) {
+					mr = r;
+				}
+/* 				r = 0; */
+			}
+		} while ((lli = lli->next) != h);
+	}
+
+	/* second pass ... set the reordered indexes */
+	mi = i + mr;
+	mr = i = 0;
+	for (s = cur_base->section_list ; s ; s = s->next) {
+		h = lli = ((section_t *)(s->data))->itm_list;
+		if (!lli) {
+			continue;
+		}
+		do {
+			if (lli->data_type & DT_RANGE) {
+				i += mr;
+				mr = 0;
+				i = lli->idx + ((range_item_t *)(lli->data))->length + 1;
+#warning check
+			} else if ((lli->data_type & DT_WEIGHTED) && !(s->data_type & DT_REORDER)) {
+				i += mr;
+				mr = 0;
+				w = (weighted_item_t *)(lli->data);
+				if (find_wi_index_reordered(w->symbol) /* reordered symbol skipped on first pass */
+#if 0
+					|| (s->data_type & DT_REORDER) /* or in a reordered section */
+#endif
+					) {
+					assert(!(s->data_type & DT_REORDER));
+					index2weight_buffer[index2weight_len] = lli->idx = ++i;
+					index2weight_len += index2weight_len_inc;
+					add_final_col_index(w->symbol);
+
+/* 					fprintf(stdout, "%11s: r %6d %6d %s\n", */
+/* 							cur_base->name, lli->idx, final_index_val(w->symbol), w->symbol); */
+
+					continue;
+				}
+				i = lli->idx;
+
+/* 				fprintf(stdout, "%11s: w %6d %6d %s\n", */
+/* 						cur_base->name, lli->idx, final_index_val(w->symbol), w->symbol); */
+
+			} else {
+/* 				fprintf(stderr, "section: %s  %d  %d\n", ((section_t *)(s->data))->name, */
+/* 						s->data_type, lli->data_type); */
+/* 					assert(!(s->data_type & DT_REORDER)); */
+/* 				assert(lli->data_type & DT_REORDER); */
+#if 1
+				if (s->data_type & DT_REORDER) {
+					h2 = l2 = lli;
+					if (!h2) {
+						continue;
+					}
+				} else {
+					assert(s->data_type & DT_SECTION);
+					h2 = l2 = ((section_t *)(lli->data))->itm_list;
+					if (!h2) {
+						continue;
+					}
+				}
+
+
+#else
+				h2 = l2 = ((section_t *)(lli->data))->itm_list;
+				if (!h2) {
+					continue;
+				}
+#endif
+				r = 0;
+				do {
+					assert(l2->data_type & DT_WEIGHTED);
+					++r;
+					l2->idx = i + r;
+
+/* 					fprintf(stdout, "%s: R %6d        %s\n", */
+/* 							((section_t *)(lli->data))->name, l2->idx, ((weighted_item_t *)(l2->data))->symbol); */
+
+				} while ((l2 = l2->next) != h2);
+				if (r > mr) {
+					mr = r;
+				}
+			}
+		} while ((lli = lli->next) != h);
+	}
+
+	/* finally, walk through all derived locales and set non-reordered section items */
+	mr = mi;
+	for (cli = cur_base->derived_list ; cli ; cli = cli->next) {
+		cl = (col_locale_t *)(cli->data);
+/* 		fprintf(stderr, "pass3: %d  %s\n", cli->data_type, cl->name); */
+
+/* 		fprintf(stdout, "pass3: %d  %s\n", cli->data_type, cl->name); */
+
+		assert(cli->data_type == DT_COL_LOCALE);
+
+		i = mi;
+		for (s = cl->section_list ; s ; s = s->next) {
+/* 			if (s->data_type & DT_REORDER) { */
+/* 				continue; */
+/* 			} */
+			h = lli = ((section_t *)(s->data))->itm_list;
+			if (!lli) {
+				continue;
+			}
+			do {
+				assert(!(lli->data_type & DT_RANGE));
+				if (lli->data_type & DT_WEIGHTED) {
+/* 					fprintf(stderr, "     %d %d %s\n", lli->data_type, lli->idx, ((weighted_item_t *)(lli->data))->symbol); */
+					add_final_col_index(((weighted_item_t *)(lli->data))->symbol);
+					if (s->data_type & DT_REORDER) {
+						continue;
+					}
+					assert(lli->idx == INT_MIN);
+					lli->idx = ++i;
+
+/* 					fprintf(stdout, "%11s: S %6d %6d %s\n", */
+/* 							cl->name, lli->idx, */
+/* 							final_index_val(((weighted_item_t *)(lli->data))->symbol), */
+/* 							((weighted_item_t *)(lli->data))->symbol); */
+
+				} else {
+					assert(0);
+					assert(lli->data_type & DT_SECTION);
+
+					h2 = l2 = ((section_t *)(lli->data))->itm_list;
+					if (!h2) {
+						continue;
+					}
+					do {
+						assert(l2->data_type & DT_WEIGHTED);
+						assert(l2->idx == INT_MIN);
+						l2->idx = ++i;
+						add_final_col_index(((weighted_item_t *)(l2->data))->symbol);
+					} while ((l2 = l2->next) != h2);
+				}
+			} while ((lli = lli->next) != h);
+		}
+		if (i > mr) {
+			mr = i;
+		}
+	}
+	max_weight = mr;
+
+	assert(num_varying == tnumnodes(cur_base->root_wi_index_reordered));
+
+	/* we can now initialize the wcs2index array */
+	{
+		ENTRY *p;
+		ENTRY e;
+		char buf[8];
+		static const char xd[] = "0123456789ABCDEF";
+		int starter_index = final_index;
+		int wcs2index_count = 0;
+
+		strcpy(buf, "<U....>");
+		memset(wcs2index, 0, sizeof(wcs2index));
+		e.key = (char *) buf;
+		for (i=1 ; i <= 0xffff ; i++) {
+			buf[5] = xd[ i & 0xf ];
+			buf[4] = xd[ (i >> 4) & 0xf ];
+			buf[3] = xd[ (i >> 8) & 0xf ];
+			buf[2] = xd[ (i >> 12) & 0xf ];
+
+			if ((p = hsearch(e, FIND)) != NULL) {
+				++wcs2index_count;
+				if ((tfind(buf, &cur_base->root_starter_char, sym_cmp)) != NULL) {
+					wcs2index[i] = ++starter_index;
+/* 					fprintf(stderr, "wcs2index[ %#06x ] = %d  (starter)\n", i, wcs2index[i]); */
+				} else {
+					wcs2index[i] = (int)(p->data);
+/* 					fprintf(stderr, "wcs2index[ %#06x ] = %d\n", i, wcs2index[i]); */
+				}
+			} else {
+				if ((tfind(buf, &cur_base->root_starter_char, sym_cmp)) != NULL) {
+					error_msg("marked starter but not in hash: %s", buf);
+				}
+			}
+		}
+
+
+	/* ---------------------------------------------------------------------- */
+		{
+			int i, n;
+			table_data table;
+			size_t t, smallest;
+
+			n = 0;
+			smallest = SIZE_MAX;
+			table.ii = NULL;
+			for (i=0 ; i < 14 ; i++) {
+				if ((RANGE >> i) < 4) {
+					break;
+				}
+				t = newopt(wcs2index, RANGE, i, &table);
+				if (smallest >= t) {
+					n = i;
+					smallest = t;
+					/*  			} else { */
+					/*  				break; */
+				}
+			}
+
+
+/* 			printf("smallest = %u  for range %#x (%u)\n", smallest, RANGE, RANGE); */
+			assert(smallest != SIZE_MAX);
+			if (smallest + wcs2colidt_len >= WCS2COLIDT_LEN) {
+				error_msg("WCS2COLIDT_LEN too small");
+			}
+			base_locale_array[base_locale_len].wcs2colidt_offset = wcs2colidt_len;
+			table.ii = wcs2colidt_buffer + wcs2colidt_len;
+			t = smallest;
+			smallest = SIZE_MAX;
+			smallest = newopt(wcs2index, RANGE, n, &table);
+			assert(t == smallest);
+			wcs2colidt_len += smallest;
+/* 			fprintf(stderr, "smallest = %d   wcs2colidt_len = %d\n", smallest, wcs2colidt_len); */
+
+#if 0
+			{
+				unsigned int sc, n, i0, i1;
+				unsigned int u = 0xe40;
+				table_data *tbl = &table;
+
+#define WCctype_TI_MASK ((1 << tbl->ti_shift)-1)
+#define WCctype_TI_SHIFT (tbl->ti_shift)
+#define WCctype_TI_LEN (tbl->ti_len)
+#define WCctype_II_MASK ((1 << tbl->ii_shift)-1)
+#define WCctype_II_SHIFT (tbl->ii_shift)
+#define WCctype_II_LEN (tbl->ii_len)
+
+				sc = u & WCctype_TI_MASK;
+				u >>= WCctype_TI_SHIFT;
+				n = u & WCctype_II_MASK;
+				u >>= WCctype_II_SHIFT;
+
+				i0 = tbl->ii[u];
+				fprintf(stderr, "i0 = %d\n", i0);
+				i0 <<= WCctype_II_SHIFT;
+				i1 = tbl->ii[WCctype_II_LEN + i0 + n];
+				/* 	i1 = tbl->ti[i0 + n]; */
+				fprintf(stderr, "i1 = %d\n", i1);
+				i1 <<= WCctype_TI_SHIFT;
+				/* 	return *(uint16_t *)(&(tbl->ii[WCctype_II_LEN + WCctype_TI_LEN + i1 + sc])); */
+				fprintf(stderr, "i2 = %d\n", WCctype_II_LEN + WCctype_TI_LEN + i1 + sc);
+				fprintf(stderr, "val = %d\n",  tbl->ii[WCctype_II_LEN + WCctype_TI_LEN + i1 + sc]);
+				/* 	return tbl->ut[i1 + sc]; */
+
+
+			}
+#endif
+			base_locale_array[base_locale_len].ii_shift = table.ii_shift;
+			base_locale_array[base_locale_len].ti_shift = table.ti_shift;
+			base_locale_array[base_locale_len].ii_len = table.ii_len;
+			base_locale_array[base_locale_len].ti_len = table.ti_len;
+		}
+	/* ---------------------------------------------------------------------- */
+
+		base_locale_array[base_locale_len].num_col_base = num_invariant + num_varying;
+		base_locale_array[base_locale_len].max_col_index = final_index;
+		base_locale_array[base_locale_len].max_weight = max_weight;
+
+		fprintf(stderr, "%s: %6u invariant  %6u varying  %6u derived  %6u total  %6u max weight  %6u wcs2\n",
+				cur_base->name, num_invariant, num_varying,
+				tnumnodes(cur_base->root_derived_wi), final_index, max_weight,
+				wcs2index_count);
+
+	}
+
+#if 1
+	/* ok, now we need to dump out the base and derived tables... */
+	/* don't forget to break up collating elements!!! */
+
+/* 	fprintf(stdout, "**************************************************\n"); */
+	/* first pass ... set the invariants */
+	for (s = cur_base->section_list ; s ; s = s->next) {
+#if 1
+		if (s->data_type & DT_REORDER) {
+			fprintf(stderr, "1: skipping reordered section %s\n", ((section_t *)(s->data))->name);
+			continue;
+		}
+#endif
+		h = lli = ((section_t *)(s->data))->itm_list;
+		if (!lli) {
+			continue;
+		}
+		do {
+			if (lli->data_type & DT_WEIGHTED) {
+				w = (weighted_item_t *)(lli->data);
+				if (find_wi_index_reordered(w->symbol)) { /* reordered symbol so skip on first pass */
+					continue;
+				}
+				if (index2weight_len_inc) {
+					index2ruleidx_buffer[index2ruleidx_len++] = 
+						add_rule((weighted_item_t *)(lli->data));
+				}
+/* 				fprintf(stdout, "%11s: w %6d %6d %s\n", */
+/* 						cur_base->name, lli->idx, final_index_val(w->symbol), w->symbol); */
+			}
+		} while ((lli = lli->next) != h);
+	}
+
+	/* second pass ... set varying */
+	for (s = cur_base->section_list ; s ; s = s->next) {
+#if 1
+		if (s->data_type & DT_REORDER) {
+			fprintf(stderr, "2: skipping reordered section %s\n", ((section_t *)(s->data))->name);
+			continue;
+		}
+#endif
+		h = lli = ((section_t *)(s->data))->itm_list;
+		if (!lli) {
+			continue;
+		}
+		do {
+			if (lli->data_type & DT_WEIGHTED) {
+				w = (weighted_item_t *)(lli->data);
+				if (find_wi_index_reordered(w->symbol)) { /* reordered symbol so skip on first pass */
+					if (index2weight_len_inc) {
+						index2ruleidx_buffer[index2ruleidx_len++] = 
+							add_rule((weighted_item_t *)(lli->data));
+					}
+/* 					fprintf(stdout, "%11s: r %6d %6d %s\n", */
+/* 							cur_base->name, lli->idx, final_index_val(w->symbol), w->symbol); */
+					continue;
+				}
+			}
+		} while ((lli = lli->next) != h);
+	}
+
+	do_starter_lists(cur_base);
+
+
+/* 	fprintf(stderr,"updated final_index = %d\n", final_index); */
+
+	if (rli) {
+		base_locale_array[base_locale_len].range_low
+			= strtoul(((range_item_t *)(rli->data))->symbol1 + 2, NULL, 16);
+		base_locale_array[base_locale_len].range_count
+			= ((range_item_t *)(rli->data))->length;
+		base_locale_array[base_locale_len].range_base_weight = rli->idx;
+		base_locale_array[base_locale_len].range_rule_offset = add_range_rule((range_item_t *)(rli->data));
+/* 		fprintf(stdout, "%11s:   %6d %6d %s %s (%d)\n", */
+/* 				"RANGE", rli->idx, -1, */
+/* 				((range_item_t *)(rli->data))->symbol1, */
+/* 				((range_item_t *)(rli->data))->symbol2, */
+/* 				((range_item_t *)(rli->data))->length); */
+	}
+
+/* 	fprintf(stdout,"\nDerived\n\n"); */
+
+	/* first, if base name is of the form ll_CC, add a derived locale for it */
+	if ((strlen(cur_base->name) == 5)
+		&& islower(cur_base->name[0])
+		&& islower(cur_base->name[1])
+		&& (cur_base->name[2] == '_')
+		&& isupper(cur_base->name[3])
+		&& isupper(cur_base->name[4])
+		) {
+
+		fprintf(stderr, "adding special derived for %s\n", cur_base->name);
+/* 	fprintf(stderr,"updated final_index = %d\n", final_index); */
+
+
+		assert(der_locale_len+1 < DER_LOCALE_LEN);
+
+		der_locale_array[der_locale_len].name = cur_base->name;
+		der_locale_array[der_locale_len].base_idx = base_locale_len;
+		
+		u16_buf[0] = 1;
+		u16_buf[1] = 0;
+		u16_buf_len = 2;
+
+		mm = NULL;
+		if ((u16_buf_len > override_len) ||
+			!(mm = memmem(override_buffer, override_len*sizeof(override_buffer[0]),
+						  u16_buf, u16_buf_len*sizeof(u16_buf[0])))
+			) {
+			assert(override_len + u16_buf_len < OVERRIDE_LEN);
+			memcpy(override_buffer + override_len, u16_buf, u16_buf_len*sizeof(u16_buf[0]));
+			der_locale_array[der_locale_len].overrides_offset = override_len;
+			override_len += u16_buf_len;
+/* 			printf("%s: override_len = %d   u16_buf_len = %d\n", cl->name, override_len, u16_buf_len); */
+		} else if (!(u16_buf_len > override_len)) {
+			assert(mm);
+			der_locale_array[der_locale_len].overrides_offset = ((uint16_t *)(mm)) - override_buffer;
+/* 			printf("%s: memmem found a match with u16_buf_len = %d\n", cl->name, u16_buf_len); */
+		}
+		der_locale_array[der_locale_len].multistart_offset
+			= base_locale_array[base_locale_len].multistart_offset;
+		der_locale_array[der_locale_len].undefined_idx = final_index_val0("UNDEFINED");
+
+		if (!der_locale_array[der_locale_len].undefined_idx) {
+			error_msg("no UNDEFINED definition for %s", cur_base->name);
+		}
+
+		++der_locale_len;
+	} else {
+		fprintf(stderr, "NOT adding special derived for %s\n", cur_base->name);
+	}
+
+	/* now all the derived... */
+	for (cli = cur_base->derived_list ; cli ; cli = cli->next) {
+		cl = (col_locale_t *)(cli->data);
+		assert(cli->data_type == DT_COL_LOCALE);
+
+		assert(der_locale_len+1 < DER_LOCALE_LEN);
+
+		der_locale_array[der_locale_len].name = cl->name;
+		der_locale_array[der_locale_len].base_idx = base_locale_len;
+
+		u16_buf_len = 0;
+
+		for (i = 0 ; i < 2 ; i++) {
+			if (i) {
+/* 				fprintf(stdout, "   section --- (singles)\n"); */
+				u16_buf[u16_buf_len++] = 1;	/* single */
+			}
+			/* we do this in two passes... first all sequences, then all single reorders */
+			for (s = cl->section_list ; s ; s = s->next) {
+				h = lli = ((section_t *)(s->data))->itm_list;
+				if (!lli) {
+					continue;
+				}
+				assert(u16_buf_len +4 < sizeof(u16_buf)/sizeof(u16_buf[0]));
+				if ((!i && (ll_len(h) > 1) ) || (ll_len(h) == i)) {
+					if (!i) {
+/* 						fprintf(stdout, "   section ----------------- %d %d\n", i, ll_len(h)); */
+						u16_buf[u16_buf_len++] = ll_len(h);	/* multi */
+						assert(lli->data_type & DT_WEIGHTED);
+#if 0
+						u16_buf[u16_buf_len++] = final_index_val(((weighted_item_t *)(lli->data))->symbol);	/* start index */
+#endif
+						u16_buf[u16_buf_len++] = lli->idx; /* start weight */
+					}
+					do {
+						if (lli->data_type & DT_WEIGHTED) {
+/* 							fprintf(stdout, "%11s: S %6d %6d %s\n", */
+/* 									cl->name, lli->idx, */
+/* 									final_index_val(((weighted_item_t *)(lli->data))->symbol), */
+/* 									((weighted_item_t *)(lli->data))->symbol); */
+#if 0
+							if (i) {
+								assert(u16_buf_len +4 < sizeof(u16_buf)/sizeof(u16_buf[0]));
+								u16_buf[u16_buf_len++] = final_index_val(((weighted_item_t *)(lli->data))->symbol);
+								assert(u16_buf[u16_buf_len-1]);
+								u16_buf[u16_buf_len++] = lli->idx; /* weight */
+							}
+#else
+							assert(u16_buf_len +4 < sizeof(u16_buf)/sizeof(u16_buf[0]));
+							u16_buf[u16_buf_len++] = final_index_val(((weighted_item_t *)(lli->data))->symbol);
+							assert(u16_buf[u16_buf_len-1]);
+							if (i) {
+								u16_buf[u16_buf_len++] = lli->idx; /* weight */
+							}
+#endif
+							u16_buf[u16_buf_len++] = add_rule((weighted_item_t *)(lli->data));
+
+						}
+					} while ((lli = lli->next) != h);
+				}
+			}
+		}
+		u16_buf[u16_buf_len++] = 0;
+
+		mm = NULL;
+		if ((u16_buf_len > override_len) ||
+			!(mm = memmem(override_buffer, override_len*sizeof(override_buffer[0]),
+						  u16_buf, u16_buf_len*sizeof(u16_buf[0])))
+			) {
+			assert(override_len + u16_buf_len < OVERRIDE_LEN);
+			memcpy(override_buffer + override_len, u16_buf, u16_buf_len*sizeof(u16_buf[0]));
+			der_locale_array[der_locale_len].overrides_offset = override_len;
+			override_len += u16_buf_len;
+/* 			printf("%s: override_len = %d   u16_buf_len = %d\n", cl->name, override_len, u16_buf_len); */
+		} else if (!(u16_buf_len > override_len)) {
+			assert(mm);
+			der_locale_array[der_locale_len].overrides_offset = ((uint16_t *)(mm)) - override_buffer;
+/* 			printf("%s: memmem found a match with u16_buf_len = %d\n", cl->name, u16_buf_len); */
+		}
+
+		do_starter_lists(cl);
+
+		der_locale_array[der_locale_len].undefined_idx = final_index_val0("UNDEFINED");
+#if 0
+		assert(der_locale_array[der_locale_len].undefined_idx);
+		if (!der_locale_array[der_locale_len].undefined_idx) {
+			der_locale_array[der_locale_len].undefined_idx = base_locale_array[base_locale_len].undefined_idx;
+		}
+#endif
+
+		if (!der_locale_array[der_locale_len].undefined_idx) {
+			error_msg("no UNDEFINED definition for %s", cl->name);
+		}
+
+		++der_locale_len;
+	}
+
+#endif
+
+#warning handle UNDEFINED idx specially?  what if in only some of derived?
+/* 	base_locale_array[base_locale_len].undefined_idx = final_index_val0("UNDEFINED"); */
+	base_locale_array[base_locale_len].undefined_idx = 0;
+
+
+	hdestroy();
+
+	++base_locale_len;
+
+/* 	if (tnumnodes(cur_base->root_starter_char)) { */
+/* 		fprintf(stderr, "starter nodes\n"); */
+/* 		twalk(cur_base->root_starter_char, print_starter_node); */
+/* 	} */
+}
+
+static int starter_all_cmp(const void *n1, const void *n2)
+{
+	const char *s1 = ((weighted_item_t *) n1)->symbol;
+	const char *s2 = ((weighted_item_t *) n2)->symbol;
+	colitem_t x;
+	colitem_t *p;
+	int n;
+
+	/* sort by 1st char ... then inverse for string */
+
+	x.element = NULL;
+	if (!is_ucode(s1)) {
+		x.string = s1;
+		p = tfind(&x, &cur_base->root_colitem, colitem_cmp);
+		s1 = (*((colitem_t **) p))->element + 1;
+	}
+	if (!is_ucode(s2)) {
+		x.string = s2;
+		p = tfind(&x, &cur_base->root_colitem, colitem_cmp);
+		s2 = (*((colitem_t **) p))->element + 1;
+	}
+
+	/* <U####>< */
+	/* 01234567 */
+
+	assert(is_ucode(s1));
+	assert(is_ucode(s2));
+
+	n = strncmp(s1+2, s2+2, 4);
+	if (n) {
+		return n;
+	}
+
+	s1 += 7;
+	s2 += 7;
+
+	return strcmp(s2, s1);
+}
+
+static void print_starter_all_node(const void *ptr, VISIT order, int level)
+{
+    const weighted_item_t *w = *(const weighted_item_t **) ptr;
+	colitem_t *ci;
+	void *p;
+	int n;
+	colitem_t x;
+
+    if (order == postorder || order == leaf)  {
+#if 0
+		if ((n = is_ucode(w->symbol)) != 0) {
+			printf(" %s\n", w->symbol);
+		} else {
+			x.string = w->symbol;
+			x.element = NULL;
+			p = tfind(&x, &cur_base->root_colitem, colitem_cmp);
+			assert(p);
+			ci = *((colitem_t **) p);
+			printf("%s = %s\n", ci->element, w->symbol);
+		}
+#else
+		printf("%s|", w->symbol);
+/* 		if ((n = is_ucode(w->symbol)) != 0) { */
+/* 			printf("\n"); */
+/* 		} */
+#endif
+	}
+}
+
+static void process_starter_node(const void *ptr, VISIT order, int level)
+{
+    const weighted_item_t *w = *(const weighted_item_t **) ptr;
+	colitem_t *ci;
+	void *p;
+	int n;
+	colitem_t x;
+	const char *s;
+	char buf[32];
+
+	/* store index of collation item followed by (unprefixed) nul-terminated string */
+    if (order == postorder || order == leaf)  {
+		if ((n = is_ucode(w->symbol)) != 0) {
+			u16_buf[u16_buf_len++] = final_index_val(w->symbol);
+			assert(u16_buf[u16_buf_len-1]);
+			u16_buf[u16_buf_len++] = 0;
+			if (++u16_starter < base_locale_array[base_locale_len].num_starters) {
+				u16_buf[u16_starter] = u16_buf_len;
+			}
+/* 			fprintf(stderr, "ucode - %d %d\n", u16_buf[u16_starter-1], u16_buf_len); */
+		} else {
+			x.string = w->symbol;
+			x.element = NULL;
+			p = tfind(&x, &cur_base->root_colitem, colitem_cmp);
+			assert(p);
+			ci = *((colitem_t **) p);
+			s = ci->element;
+			u16_buf[u16_buf_len++] = final_index_val(w->symbol);
+			assert(u16_buf[u16_buf_len-1]);
+			assert(*s == '"');
+			n = is_ucode(++s);
+/* 			fprintf(stderr, "s is |%s| with len %d (%d)\n", s, strlen(s), n); */
+			assert(n);
+			s += n;
+			while (*s != '"') {
+				n = is_ucode(s);
+				assert(n);
+				strncpy(buf, s, n+1);
+				buf[n] = 0;
+/* 				fprintf(stderr, "buf is |%s| with len %d (%d)\n", buf, strlen(buf), n); */
+				u16_buf[u16_buf_len++] = final_index_val(buf);
+				assert(u16_buf[u16_buf_len-1]);
+				s += n;
+			}
+			u16_buf[u16_buf_len++] = 0;
+		}
+	}
+}
+
+static void **p_cl_root_starter_all;
+
+static void complete_starter_node(const void *ptr, VISIT order, int level)
+{
+	weighted_item_t w;
+	weighted_item_t *p;
+
+    if (order == postorder || order == leaf)  {
+		w.symbol = *(const char **) ptr;
+		w.weight = NULL;
+		if (!tfind(&w, p_cl_root_starter_all, starter_all_cmp)) {
+			p = xmalloc(sizeof(weighted_item_t));
+			p->symbol = w.symbol;
+			p->weight = NULL;
+/* 			fprintf(stderr, "complete_starter_node: %s\n", *(const char **) ptr); */
+			if (!tsearch(p, p_cl_root_starter_all, starter_all_cmp)) {
+				error_msg("OUT OF MEMORY");
+			}
+		}
+    }
+}
+
+static void do_starter_lists(col_locale_t *cl)
+{
+	ll_item_t *s;
+	ll_item_t *h;
+	ll_item_t *lli;
+	col_locale_t *c;
+	colitem_t *ci;
+	weighted_item_t *w;
+	void *p;
+	char buf[32];
+	int n;
+	colitem_t x;
+	void *mm;
+
+	c = cl;
+	if (c != cur_base) {
+		c = cur_base;
+	}
+
+/* 	printf("STARTERS %s --------------------\n", cl->name); */
+ LOOP:
+	for (s = c->section_list ; s ; s = s->next) {
+		h = lli = ((section_t *)(s->data))->itm_list;
+		if (!lli) {
+			continue;
+		}
+		do {
+			if (lli->data_type & DT_WEIGHTED) {
+				w = (weighted_item_t *)(lli->data);
+				ci = NULL;
+				if ((n = is_ucode(w->symbol)) != 0) {
+					strcpy(buf, w->symbol);
+				} else {
+/* 					fprintf(stdout, "looking for |%s|\n", w->symbol); */
+					x.string = w->symbol;
+					x.element = NULL;
+					p = tfind(&x, &cur_base->root_colitem, colitem_cmp);
+					if (!p) {
+/* 						fprintf(stderr, "Whoa... processing starters for %s and couldn't find %s\n", */
+/* 								cl->name, w->symbol); */
+						continue;
+					}
+					ci = *((colitem_t **) p);
+					if (!ci->element) {	/* just a collating symbol */
+						continue;
+					}
+					assert(ci->element[0] == '"');
+					n = is_ucode(ci->element + 1);
+					assert(n);
+					strncpy(buf, ci->element + 1, n);
+				}
+				if ((tfind(buf, &cur_base->root_starter_char, sym_cmp)) != NULL) {
+/* 					fprintf(stdout, "adding from %s: %s", c->name, w->symbol); */
+/* 					if (ci) { */
+/* 						fprintf(stdout, " = %s", ci->element); */
+/* 					} */
+/* 					fprintf(stdout, "\n"); */
+
+					if (!tsearch(w, &cl->root_starter_all, starter_all_cmp)) {
+						error_msg("OUT OF MEMORY");
+					}
+				}
+			}
+		} while ((lli = lli->next) != h);
+	}
+
+	if (c != cl) {
+		c = cl;
+		goto LOOP;
+	}
+
+	p_cl_root_starter_all = &cl->root_starter_all;
+	twalk(cur_base->root_starter_char, complete_starter_node);
+
+	if (cl == cur_base) {
+		base_locale_array[base_locale_len].num_starters	= tnumnodes(cur_base->root_starter_char);
+	}
+
+#if 0
+	printf("\nNow walking tree...\n\n");
+	twalk(cl->root_starter_all, print_starter_all_node);
+	printf("\n\n");
+
+#endif
+	u16_starter = 0;
+	u16_buf[0] = u16_buf_len = base_locale_array[base_locale_len].num_starters;
+	twalk(cl->root_starter_all, process_starter_node);
+/* 	fprintf(stderr, "s=%d n=%d\n", u16_starter,  base_locale_array[base_locale_len].num_starters); */
+	assert(u16_starter == base_locale_array[base_locale_len].num_starters);
+
+#if 0
+	{ int i;
+	for (i=0 ; i < u16_buf_len ; i++) {
+		fprintf(stderr, "starter %2d: %d - %#06x\n", i, u16_buf[i], u16_buf[i]);
+	}}
+#endif
+
+	mm = NULL;
+	if (u16_buf_len) {
+/* 		assert(base_locale_array[base_locale_len].num_starters); */
+		if ((u16_buf_len > multistart_len) ||
+			!(mm = memmem(multistart_buffer, multistart_len*sizeof(multistart_buffer[0]),
+						  u16_buf, u16_buf_len*sizeof(u16_buf[0])))
+			) {
+			assert(multistart_len + u16_buf_len < MULTISTART_LEN);
+			memcpy(multistart_buffer + multistart_len, u16_buf, u16_buf_len*sizeof(u16_buf[0]));
+			if (cl == cur_base) {
+				base_locale_array[base_locale_len].multistart_offset = multistart_len;
+			} else {
+				der_locale_array[der_locale_len].multistart_offset = multistart_len;
+			}
+			multistart_len += u16_buf_len;
+/* 			fprintf(stderr, "%s: multistart_len = %d   u16_buf_len = %d\n", cl->name, multistart_len, u16_buf_len); */
+		} else if (!(u16_buf_len > multistart_len)) {
+			assert(mm);
+			if (cl == cur_base) {
+				base_locale_array[base_locale_len].multistart_offset = ((uint16_t *)(mm)) - multistart_buffer;
+			} else {
+				der_locale_array[der_locale_len].multistart_offset = ((uint16_t *)(mm)) - multistart_buffer;
+			}
+/* 			fprintf(stderr, "%s: memmem found a match with u16_buf_len = %d\n", cl->name, u16_buf_len); */
+		}
+	} else {
+		assert(!base_locale_array[base_locale_len].num_starters);
+	}
+
+/* 	printf("u16_buf_len = %d\n", u16_buf_len); */
+
+/* 	printf("STARTERS %s DONE ---------------\n", cl->name); */
+}
+
+
+/* For sorting the blocks of unsigned chars. */
+static size_t nu_val;
+
+int nu_memcmp(const void *a, const void *b)
+{
+	return memcmp(*(unsigned char**)a, *(unsigned char**)b, nu_val * sizeof(tbl_item));
+}
+
+
+size_t newopt(tbl_item *ut, size_t usize, int shift, table_data *tbl)
+{
+	static int recurse = 0;
+	tbl_item *ti[RANGE];	/* table index */
+	size_t numblocks;
+	size_t blocksize;
+	size_t uniq;
+	size_t i, j;
+	size_t smallest, t;
+	tbl_item *ii_save;
+	int uniqblock[1 << (8*sizeof(tbl_item) - 1)];
+	tbl_item uit[RANGE];
+	int shift2;
+
+	if (shift > 15) {
+		return SIZE_MAX;
+	}
+
+	ii_save = NULL;
+	blocksize = 1 << shift;
+	numblocks = usize >> shift;
+
+	/* init table index */
+	for (i=j=0 ; i < numblocks ; i++) {
+		ti[i] = ut + j;
+		j += blocksize;
+	}
+
+	/* sort */
+	nu_val = blocksize;
+	qsort(ti, numblocks, sizeof(unsigned char *), nu_memcmp);
+	
+	uniq = 1;
+	uit[(ti[0]-ut)/blocksize] = 0;
+	for (i=1 ; i < numblocks ; i++) {
+		if (memcmp(ti[i-1], ti[i], blocksize*sizeof(tbl_item)) < 0) {
+			if (++uniq > (1 << (8*sizeof(tbl_item) - 1))) {
+				break;
+			}
+			uniqblock[uniq - 1] = i;
+		}
+#if 1
+		else if (memcmp(ti[i-1], ti[i], blocksize*sizeof(tbl_item)) > 0) {
+			printf("bad sort %i!\n", i);
+			abort();
+		}
+#endif
+		uit[(ti[i]-ut)/blocksize] = uniq - 1;
+	}
+
+	smallest = SIZE_MAX;
+	shift2 = -1;
+	if (uniq <= (1 << (8*sizeof(tbl_item) - 1))) {
+		smallest = numblocks + uniq * blocksize;
+		if (!recurse) {
+			++recurse;
+			for (j=1 ; j < 14 ; j++) {
+				if ((numblocks >> j) < 2) break;
+				if (tbl) {
+					ii_save = tbl->ii;
+					tbl->ii = NULL;
+				}
+				if ((t = newopt(uit, numblocks, j, tbl)) < SIZE_MAX) {
+					t += uniq * blocksize;
+				}
+				if (tbl) {
+					tbl->ii = ii_save;
+				}
+				if (smallest >= t) {
+					shift2 = j;
+					smallest = t;
+/* 					if (!tbl->ii) { */
+/* 						printf("ishift %u  tshift %u  size %u\n", */
+/* 							   shift2, shift, t); */
+/* 					} */
+/*  				} else { */
+/*  					break; */
+				}
+			}
+			--recurse;
+		}
+	} else {
+		return SIZE_MAX;
+	}
+
+	if (tbl->ii) {
+		if (recurse) {
+			tbl->ii_shift = shift;
+			tbl->ii_len = numblocks;
+			memcpy(tbl->ii, uit, numblocks*sizeof(tbl_item));
+			tbl->ti = tbl->ii + tbl->ii_len;
+			tbl->ti_len = uniq * blocksize;
+			for (i=0 ; i < uniq ; i++) {
+				memcpy(tbl->ti + i * blocksize, ti[uniqblock[i]], blocksize*sizeof(tbl_item));
+			}
+		} else {
+			++recurse;
+/* 			printf("setting ishift %u  tshift %u\n", shift2, shift); */
+			newopt(uit, numblocks, shift2, tbl);
+			--recurse;
+			tbl->ti_shift = shift;
+			tbl->ut_len = uniq * blocksize;
+			tbl->ut = tbl->ti + tbl->ti_len;
+			for (i=0 ; i < uniq ; i++) {
+				memcpy(tbl->ut + i * blocksize, ti[uniqblock[i]], blocksize*sizeof(tbl_item));
+			}
+		}
+	}
+	return smallest;
+}
+
+static const int rule2val[8] = {
+	-1,
+	(1 << 14),					/* forward */
+	(2 << 14),					/* position */
+	(3 << 14),					/* forward,position */
+	0,							/* backward */
+	-1,
+	-1,
+	-1,
+};
+
+
+static int final_index_val_x(const char *s, const char *sym)
+{
+	int r;
+
+	if (!(r = final_index_val0(s))) {
+		if (!strcmp(s, "IGNORE")) {
+			r = 0;
+		} else if (!strcmp(s, "..") || !strcmp(sym, "RANGE")) {
+			if (*sym == '.') {
+				final_index_val(sym); /* make sure it's known */
+			}
+			r = 0x3fff;
+		} else if (!strcmp(s, ".")) {
+			r = 0x3ffe;
+		} else {
+			error_msg("can't find final index: %s", s);
+		}
+	}
+	return r;
+}
+
+/* store rule2val in 2 high bits and collation index in lower.
+ * for sort strings, store (offset from base) + max colindex as index.
+ */
+static unsigned int add_rule(weighted_item_t *wi)
+{
+	weight_t *w = wi->weight;
+	int i, j, r, n;
+	uint16_t rbuf[MAX_COLLATION_WEIGHTS];
+	uint16_t ws_buf[32];
+	void *mm;
+	char buf[32];
+	const char *s;
+	const char *e;
+
+	for (i=0 ; i < MAX_COLLATION_WEIGHTS ; i++) {
+		rbuf[i] = rule2val[R_FORWARD]; /* set a default to forward-ignore */
+	}
+
+	if (base_locale_array[base_locale_len].num_weights < w->num_weights) {
+		base_locale_array[base_locale_len].num_weights = w->num_weights;
+	}
+
+	for (i=0 ; i < w->num_weights ; i++) {
+		assert(rule2val[(int)(w->rule[i])] >= 0);
+		assert(w->colitem[i] && *w->colitem[i]);
+		if (*w->colitem[i] == '"') { /* string... */
+			s = w->colitem[i] + 1;
+			assert(*s == '<');
+			n = 0;
+			do {
+				e = s;
+				do {
+					if (*e == '/') {
+						e += 2;
+						continue;
+					}
+				} while (*e++ != '>');
+				assert(((size_t)(e-s) < sizeof(buf)));
+				memcpy(buf, s, (size_t)(e-s));
+				buf[(size_t)(e-s)] = 0;
+
+				r = final_index_val_x(buf, wi->symbol);
+				assert(n + 1 < sizeof(ws_buf)/sizeof(ws_buf[0]));
+				ws_buf[n++] = r | rule2val[(int)(w->rule[i])];
+
+				s = e;
+			} while (*s != '"');
+			ws_buf[n++] = 0;	/* terminator */
+
+			mm = memmem(weightstr_buffer, weightstr_len*sizeof(weightstr_buffer[0]),
+						ws_buf, n*sizeof(ws_buf[0]));
+
+			if (!mm) {
+				assert(weightstr_len + n < WEIGHTSTR_LEN);
+				memcpy(weightstr_buffer + weightstr_len, ws_buf, n*sizeof(ws_buf[0]));
+				mm = weightstr_buffer + weightstr_len;
+				weightstr_len += n;
+			}
+			r = (((uint16_t *)(mm)) - weightstr_buffer)
+				+ base_locale_array[base_locale_len].max_col_index + 2;
+			assert(r < (1 << 14));
+			rbuf[i] = r | rule2val[(int)(w->rule[i])];
+		} else {				/* item */
+			r = final_index_val_x(w->colitem[i], wi->symbol); 
+			rbuf[i] = r | rule2val[(int)(w->rule[i])];
+		}
+	}
+
+	for (i=0 ; i < ruletable_len ; i += MAX_COLLATION_WEIGHTS) {
+		if (!memcmp(ruletable_buffer + i, rbuf, MAX_COLLATION_WEIGHTS*sizeof(ruletable_buffer[0]))) {
+			return i/MAX_COLLATION_WEIGHTS;
+		}
+	}
+
+	memcpy(ruletable_buffer + ruletable_len, rbuf, MAX_COLLATION_WEIGHTS*sizeof(ruletable_buffer[0]));
+	ruletable_len += MAX_COLLATION_WEIGHTS;
+
+	return  (ruletable_len / MAX_COLLATION_WEIGHTS)-1;
+}
+
+static unsigned int add_range_rule(range_item_t *ri)
+{
+	weight_t *w = ri->weight;
+	int i, j, r, n;
+	uint16_t rbuf[MAX_COLLATION_WEIGHTS];
+	uint16_t ws_buf[32];
+	void *mm;
+	char buf[32];
+	const char *s;
+	const char *e;
+
+	for (i=0 ; i < MAX_COLLATION_WEIGHTS ; i++) {
+		rbuf[i] = rule2val[R_FORWARD]; /* set a default to forward-ignore */
+	}
+
+	if (base_locale_array[base_locale_len].num_weights < w->num_weights) {
+		base_locale_array[base_locale_len].num_weights = w->num_weights;
+	}
+
+	for (i=0 ; i < w->num_weights ; i++) {
+		assert(rule2val[(int)(w->rule[i])] >= 0);
+		assert(w->colitem[i] && *w->colitem[i]);
+		if (*w->colitem[i] == '"') { /* string... */
+			s = w->colitem[i] + 1;
+			assert(*s == '<');
+			n = 0;
+			do {
+				e = s;
+				do {
+					if (*e == '/') {
+						e += 2;
+						continue;
+					}
+				} while (*e++ != '>');
+				assert(((size_t)(e-s) < sizeof(buf)));
+				memcpy(buf, s, (size_t)(e-s));
+				buf[(size_t)(e-s)] = 0;
+
+				r = final_index_val_x(buf, "RANGE");
+				assert(n + 1 < sizeof(ws_buf)/sizeof(ws_buf[0]));
+				ws_buf[n++] = r | rule2val[(int)(w->rule[i])];
+
+				s = e;
+			} while (*s != '"');
+			ws_buf[n++] = 0;	/* terminator */
+
+			mm = memmem(weightstr_buffer, weightstr_len*sizeof(weightstr_buffer[0]),
+						ws_buf, n*sizeof(ws_buf[0]));
+
+			if (!mm) {
+				assert(weightstr_len + n < WEIGHTSTR_LEN);
+				memcpy(weightstr_buffer + weightstr_len, ws_buf, n*sizeof(ws_buf[0]));
+				mm = weightstr_buffer + weightstr_len;
+				weightstr_len += n;
+			}
+			r = (((uint16_t *)(mm)) - weightstr_buffer)
+				+ base_locale_array[base_locale_len].max_col_index + 2;
+			assert(r < (1 << 14));
+			rbuf[i] = r | rule2val[(int)(w->rule[i])];
+		} else {				/* item */
+			r = final_index_val_x(w->colitem[i], "RANGE");
+			rbuf[i] = r | rule2val[(int)(w->rule[i])];
+		}
+	}
+
+	for (i=0 ; i < ruletable_len ; i += MAX_COLLATION_WEIGHTS) {
+		if (!memcmp(ruletable_buffer + i, rbuf, MAX_COLLATION_WEIGHTS*sizeof(ruletable_buffer[0]))) {
+			return i/MAX_COLLATION_WEIGHTS;
+		}
+	}
+
+	memcpy(ruletable_buffer + ruletable_len, rbuf, MAX_COLLATION_WEIGHTS*sizeof(ruletable_buffer[0]));
+	ruletable_len += MAX_COLLATION_WEIGHTS;
+
+	return  (ruletable_len / MAX_COLLATION_WEIGHTS)-1;
+}
+
+#define DUMPn(X) fprintf(stderr, "%10d-%-.20s", base_locale_array[n]. ## X, #X);
+
+static void dump_base_locale(int n)
+{
+	assert(n < base_locale_len);
+
+	fprintf(stderr, "Base Locale: %s\n", base_locale_array[n].name);
+
+	DUMPn(num_weights);
+
+	DUMPn(ii_shift);
+	DUMPn(ti_shift);
+	DUMPn(ii_len);
+	DUMPn(ti_len);
+	DUMPn(max_weight);
+	fprintf(stderr, "\n");
+	DUMPn(num_col_base);
+	DUMPn(max_col_index);
+	DUMPn(undefined_idx);
+	DUMPn(range_low);
+	DUMPn(range_count);
+	fprintf(stderr, "\n");
+	DUMPn(range_base_weight);
+	DUMPn(num_starters);
+
+	fprintf(stderr, "\n");
+	DUMPn(range_rule_offset);
+	DUMPn(wcs2colidt_offset);
+	DUMPn(index2weight_offset);
+	fprintf(stderr, "\n");
+	DUMPn(index2ruleidx_offset);
+	DUMPn(multistart_offset);
+	fprintf(stderr, "\n");
+}
+
+#undef DUMPn
+#define DUMPn(X) fprintf(stderr, "%10d-%s", der_locale_array[n]. ## X, #X);
+
+static void dump_der_locale(int n)
+{
+	assert(n < der_locale_len);
+
+	fprintf(stderr, "Derived Locale: %s (%.12s)",
+			der_locale_array[n].name,
+			base_locale_array[der_locale_array[n].base_idx].name);
+
+
+	DUMPn(base_idx);
+
+	DUMPn(undefined_idx);
+	
+	DUMPn(overrides_offset);
+	DUMPn(multistart_offset);
+
+	fprintf(stderr, "\n");
+}
+
+
+static unsigned long collate_pos;
+
+static void dump_u16_array(FILE *fp, uint16_t *u, int len, const char *name)
+{
+	int i;
+
+	fprintf(fp, "\t/* %8lu %s */\n", collate_pos, name);
+	for (i=0 ; i < len ; i++) {
+		if (!(i & 7)) {
+			fprintf(fp, "\n\t");
+		}
+		fprintf(fp,"  %#06x,", (unsigned int)(u[i]));
+	}
+	fprintf(fp,"\n");
+	collate_pos += len;
+}
+
+#define OUT_U16C(X,N) fprintf(fp,"\t%10d, /* %8lu %s */\n", X, collate_pos++, N); 
+
+static void dump_collate(FILE *fp)
+{
+	int n;
+
+	fprintf(fp, "const uint16_t __locale_collate_tbl[] = {\n");
+
+	OUT_U16C(base_locale_len, "numbef of base locales");
+	OUT_U16C(der_locale_len, "number of derived locales");
+	OUT_U16C(MAX_COLLATION_WEIGHTS, "max collation weights");
+	OUT_U16C(index2weight_len, "number of index2{weight|ruleidx} elements");
+	OUT_U16C(weightstr_len, "number of weightstr elements");
+	OUT_U16C(multistart_len, "number of multistart elements");
+	OUT_U16C(override_len, "number of override elements");
+	OUT_U16C(ruletable_len, "number of ruletable elements");
+
+#undef DUMPn
+#define DUMPn(X) fprintf(fp, "\t%10d, /* %8lu %s */\n", base_locale_array[n]. ## X, collate_pos++, #X);
+	for (n=0 ; n < base_locale_len ; n++) {
+		unsigned wcs2colidt_offset_low = base_locale_array[n].wcs2colidt_offset & 0xffffU;
+		unsigned wcs2colidt_offset_hi = base_locale_array[n].wcs2colidt_offset >> 16;
+		fprintf(fp, "\t/* Base Locale %2d: %s */\n", n, base_locale_array[n].name);
+		DUMPn(num_weights);
+		DUMPn(num_starters);
+		DUMPn(ii_shift);
+		DUMPn(ti_shift);
+		DUMPn(ii_len);
+		DUMPn(ti_len);
+		DUMPn(max_weight);
+		DUMPn(num_col_base);
+		DUMPn(max_col_index);
+		DUMPn(undefined_idx);
+		DUMPn(range_low);
+		DUMPn(range_count);
+		DUMPn(range_base_weight);
+		DUMPn(range_rule_offset);
+		DUMPn(index2weight_offset);
+		DUMPn(index2ruleidx_offset);
+		DUMPn(multistart_offset);
+#undef DUMPn
+#define DUMPn(X) fprintf(fp, "\t%10d, /* %8lu %s */\n", X, collate_pos++, #X);
+		DUMPn(wcs2colidt_offset_low);
+		DUMPn(wcs2colidt_offset_hi);
+	}
+#undef DUMPn		
+
+
+	fprintf(fp, "#define COL_IDX_C     %5d\n", 0);
+#define DUMPn(X) fprintf(fp, "\t%10d, /* %8lu %s */\n", der_locale_array[n]. ## X, collate_pos++, #X);
+	for (n=0 ; n < der_locale_len ; n++) {
+		fprintf(fp, "#define COL_IDX_%s %5d\n", der_locale_array[n].name, n+1);
+		fprintf(fp, "\t/* Derived Locale %4d: %s (%.12s) */\n",
+				n, der_locale_array[n].name,
+				base_locale_array[der_locale_array[n].base_idx].name);
+		DUMPn(base_idx);
+		DUMPn(undefined_idx);
+		DUMPn(overrides_offset);
+		DUMPn(multistart_offset);
+	}
+#undef DUMPn
+
+	fprintf(fp, "\n");
+
+	dump_u16_array(fp, index2weight_buffer, index2weight_len, "index2weight");
+	dump_u16_array(fp, index2ruleidx_buffer, index2ruleidx_len, "index2ruleidx");
+	dump_u16_array(fp, multistart_buffer, multistart_len, "multistart");
+	dump_u16_array(fp, override_buffer, override_len, "override");
+	dump_u16_array(fp, ruletable_buffer, ruletable_len, "ruletable");
+	dump_u16_array(fp, weightstr_buffer, weightstr_len, "weightstr");
+	dump_u16_array(fp, wcs2colidt_buffer, wcs2colidt_len, "wcs2colidt");
+
+
+	fprintf(fp,"}; /* %8lu */\n", collate_pos);
+
+	fprintf(fp,"#define __lc_collate_data_LEN  %d\n\n", collate_pos);
+}