New locale support (in development). Supports LC_CTYPE, LC_NUMERIC,

  LC_TIME, LC_MONETARY, and LC_MESSAGES for the SUSv3 items.  Also,
  nl_langinfo() when real locale support is enabled.
New implementation of ctype.h.
New implementation of wctype.h.
New implementation of most of the string functions (smaller).
New implementation of the wcs/wmem functions.  These are untested, but
  they're also just preprocessor-modified versions ot the corresponding
  str/mem functions.
Tweaked qsort and new bsearch.

Stuff still pending:
stdlib.h and wchar.h  mb<->wc functions.  I actually have working
   versions of the stdlib ones, but the reentrant versions from
   wchar.h require some reworking.
Basic replacement and translit support for wc->mb conversions.
   (groundwork laid).
Simple-minded collate support such as was provided by the previous
   locale implementation.  (mostly done -- 8-bit codesets only)
Shared mmaping of the locale data and strerror message text.

30 files changed, 8180 insertions, 506 deletions

diff --git a/extra/locale/.cvsignore b/extra/locale/.cvsignore
new file mode 100644
index 000000000..e020fa9e3
--- /dev/null
+++ b/extra/locale/.cvsignore
@@ -0,0 +1,14 @@
+lmmtolso
+gen_wctype
+wctables.h
+gen_wc8bit
+c8tables.h
+gen_locale
+locale_tables.h
+locale.mmap
+gen_mmap
+lt_defines.h
+locale_data.c
+uClibc_locale_data.h
+codesets.txt
+locales.txt
diff --git a/extra/locale/LOCALES b/extra/locale/LOCALES
new file mode 100644
index 000000000..739ecb95a
--- /dev/null
+++ b/extra/locale/LOCALES
@@ -0,0 +1,288 @@
+# First some @modifier mappings.  Internally, the modifier is signaled by
+# replacing '_' in the locale name with a unique identifying character.
+# For example, internally we map "ca_ES@euro" to "caeES".  This allows for
+# smaller code and easier processing of locale names.
+
+@euro e
+@cyrillic c
+
+#---------------------------------------------------------------------------
+# Next, set to {y}es to enable and {n}o to disable the UTF-8 and the 8-bit
+# codeset locales.  Of course, you must have built the c8tables.h and
+# the wctables.h files appropriately.
+
+
+UTF-8 yes
+8-BIT yes
+
+#---------------------------------------------------------------------------
+# Now the locales
+
+af_ZA ISO-8859-1
+ar_AE ISO-8859-6
+ar_BH ISO-8859-6
+ar_DZ ISO-8859-6
+ar_EG ISO-8859-6
+ar_IN UTF-8
+ar_IQ ISO-8859-6
+ar_JO ISO-8859-6
+ar_KW ISO-8859-6
+ar_LB ISO-8859-6
+ar_LY ISO-8859-6
+ar_MA ISO-8859-6
+ar_OM ISO-8859-6
+ar_QA ISO-8859-6
+ar_SA ISO-8859-6
+ar_SD ISO-8859-6
+ar_SY ISO-8859-6
+ar_TN ISO-8859-6
+ar_YE ISO-8859-6
+be_BY CP1251
+bg_BG CP1251
+ca_ES ISO-8859-1
+ca_ES@euro ISO-8859-15
+cs_CZ ISO-8859-2
+da_DK ISO-8859-1
+de_AT ISO-8859-1
+de_AT@euro ISO-8859-15
+de_BE ISO-8859-1
+de_BE@euro ISO-8859-15
+de_CH ISO-8859-1
+de_DE ISO-8859-1
+de_DE@euro ISO-8859-15
+de_LU ISO-8859-1
+de_LU@euro ISO-8859-15
+el_GR ISO-8859-7
+en_AU ISO-8859-1
+en_BW ISO-8859-1
+en_CA ISO-8859-1
+en_DK ISO-8859-1
+en_GB ISO-8859-1
+en_HK ISO-8859-1
+en_IE ISO-8859-1
+en_IE@euro ISO-8859-15
+en_IN UTF-8
+en_NZ ISO-8859-1
+en_PH ISO-8859-1
+en_SG ISO-8859-1
+en_US ISO-8859-1
+en_ZA ISO-8859-1
+en_ZW ISO-8859-1
+es_AR ISO-8859-1
+es_BO ISO-8859-1
+es_CL ISO-8859-1
+es_CO ISO-8859-1
+es_CR ISO-8859-1
+es_DO ISO-8859-1
+es_EC ISO-8859-1
+es_ES ISO-8859-1
+es_ES@euro ISO-8859-15
+es_GT ISO-8859-1
+es_HN ISO-8859-1
+es_MX ISO-8859-1
+es_NI ISO-8859-1
+es_PA ISO-8859-1
+es_PE ISO-8859-1
+es_PR ISO-8859-1
+es_PY ISO-8859-1
+es_SV ISO-8859-1
+es_US ISO-8859-1
+es_UY ISO-8859-1
+es_VE ISO-8859-1
+et_EE ISO-8859-1
+eu_ES ISO-8859-1
+eu_ES@euro ISO-8859-15
+fa_IR UTF-8
+fi_FI ISO-8859-1
+fi_FI@euro ISO-8859-15
+fo_FO ISO-8859-1
+fr_BE ISO-8859-1
+fr_BE@euro ISO-8859-15
+fr_CA ISO-8859-1
+fr_CH ISO-8859-1
+fr_FR ISO-8859-1
+fr_FR@euro ISO-8859-15
+fr_LU ISO-8859-1
+fr_LU@euro ISO-8859-15
+ga_IE ISO-8859-1
+ga_IE@euro ISO-8859-15
+gl_ES ISO-8859-1
+gl_ES@euro ISO-8859-15
+gv_GB ISO-8859-1
+he_IL ISO-8859-8
+hi_IN UTF-8
+hr_HR ISO-8859-2
+hu_HU ISO-8859-2
+id_ID ISO-8859-1
+is_IS ISO-8859-1
+it_CH ISO-8859-1
+it_IT ISO-8859-1
+it_IT@euro ISO-8859-15
+iw_IL ISO-8859-8
+kl_GL ISO-8859-1
+ko_KR.UTF-8 UTF-8
+kw_GB ISO-8859-1
+lt_LT ISO-8859-13
+lv_LV ISO-8859-13
+mk_MK ISO-8859-5
+mr_IN UTF-8
+ms_MY ISO-8859-1
+mt_MT ISO-8859-3
+nl_BE ISO-8859-1
+nl_BE@euro ISO-8859-15
+nl_NL ISO-8859-1
+nl_NL@euro ISO-8859-15
+nn_NO ISO-8859-1
+no_NO ISO-8859-1
+pl_PL ISO-8859-2
+pt_BR ISO-8859-1
+pt_PT ISO-8859-1
+pt_PT@euro ISO-8859-15
+ro_RO ISO-8859-2
+ru_RU ISO-8859-5
+ru_RU.KOI8-R KOI8-R
+ru_UA KOI8-U
+sk_SK ISO-8859-2
+sl_SI ISO-8859-2
+sq_AL ISO-8859-1
+sr_YU ISO-8859-2
+sr_YU@cyrillic ISO-8859-5
+sv_FI ISO-8859-1
+sv_FI@euro ISO-8859-15
+sv_SE ISO-8859-1
+ta_IN UTF-8
+te_IN UTF-8
+th_TH TIS-620
+tr_TR ISO-8859-9
+uk_UA KOI8-U
+vi_VN UTF-8
+ar_AE.UTF-8 UTF-8
+ar_BH.UTF-8 UTF-8
+ar_DZ.UTF-8 UTF-8
+ar_EG.UTF-8 UTF-8
+ar_IQ.UTF-8 UTF-8
+ar_JO.UTF-8 UTF-8
+ar_KW.UTF-8 UTF-8
+ar_LB.UTF-8 UTF-8
+ar_LY.UTF-8 UTF-8
+ar_MA.UTF-8 UTF-8
+ar_OM.UTF-8 UTF-8
+ar_QA.UTF-8 UTF-8
+ar_SA.UTF-8 UTF-8
+ar_SD.UTF-8 UTF-8
+ar_SY.UTF-8 UTF-8
+ar_TN.UTF-8 UTF-8
+ar_YE.UTF-8 UTF-8
+be_BY.UTF-8 UTF-8
+bg_BG.UTF-8 UTF-8
+ca_ES.UTF-8 UTF-8
+ca_ES.UTF-8@euro UTF-8
+cs_CZ.UTF-8 UTF-8
+da_DK.UTF-8 UTF-8
+de_AT.UTF-8 UTF-8
+de_AT.UTF-8@euro UTF-8
+de_BE.UTF-8 UTF-8
+de_BE.UTF-8@euro UTF-8
+de_CH.UTF-8 UTF-8
+de_DE.UTF-8 UTF-8
+de_DE.UTF-8@euro UTF-8
+de_LU.UTF-8 UTF-8
+de_LU.UTF-8@euro UTF-8
+el_GR.UTF-8 UTF-8
+en_AU.UTF-8 UTF-8
+en_BW.UTF-8 UTF-8
+en_CA.UTF-8 UTF-8
+en_DK.UTF-8 UTF-8
+en_GB.UTF-8 UTF-8
+en_HK.UTF-8 UTF-8
+en_IE.UTF-8 UTF-8
+en_IE.UTF-8@euro UTF-8
+en_NZ.UTF-8 UTF-8
+en_PH.UTF-8 UTF-8
+en_SG.UTF-8 UTF-8
+en_US.UTF-8 UTF-8
+en_ZA.UTF-8 UTF-8
+en_ZW.UTF-8 UTF-8
+es_AR.UTF-8 UTF-8
+es_BO.UTF-8 UTF-8
+es_CL.UTF-8 UTF-8
+es_CO.UTF-8 UTF-8
+es_CR.UTF-8 UTF-8
+es_DO.UTF-8 UTF-8
+es_EC.UTF-8 UTF-8
+es_ES.UTF-8 UTF-8
+es_ES.UTF-8@euro UTF-8
+es_GT.UTF-8 UTF-8
+es_HN.UTF-8 UTF-8
+es_MX.UTF-8 UTF-8
+es_NI.UTF-8 UTF-8
+es_PA.UTF-8 UTF-8
+es_PE.UTF-8 UTF-8
+es_PR.UTF-8 UTF-8
+es_PY.UTF-8 UTF-8
+es_SV.UTF-8 UTF-8
+es_US.UTF-8 UTF-8
+es_UY.UTF-8 UTF-8
+es_VE.UTF-8 UTF-8
+et_EE.UTF-8 UTF-8
+eu_ES.UTF-8 UTF-8
+eu_ES.UTF-8@euro UTF-8
+fi_FI.UTF-8 UTF-8
+fi_FI.UTF-8@euro UTF-8
+fo_FO.UTF-8 UTF-8
+fr_BE.UTF-8 UTF-8
+fr_BE.UTF-8@euro UTF-8
+fr_CA.UTF-8 UTF-8
+fr_CH.UTF-8 UTF-8
+fr_FR.UTF-8 UTF-8
+fr_FR.UTF-8@euro UTF-8
+fr_LU.UTF-8 UTF-8
+fr_LU.UTF-8@euro UTF-8
+ga_IE.UTF-8 UTF-8
+ga_IE.UTF-8@euro UTF-8
+gl_ES.UTF-8 UTF-8
+gl_ES.UTF-8@euro UTF-8
+gv_GB.UTF-8 UTF-8
+he_IL.UTF-8 UTF-8
+hr_HR.UTF-8 UTF-8
+hu_HU.UTF-8 UTF-8
+id_ID.UTF-8 UTF-8
+is_IS.UTF-8 UTF-8
+it_CH.UTF-8 UTF-8
+it_IT.UTF-8 UTF-8
+it_IT.UTF-8@euro UTF-8
+iw_IL.UTF-8 UTF-8
+kl_GL.UTF-8 UTF-8
+kw_GB.UTF-8 UTF-8
+lt_LT.UTF-8 UTF-8
+lv_LV.UTF-8 UTF-8
+mk_MK.UTF-8 UTF-8
+ms_MY.UTF-8 UTF-8
+mt_MT.UTF-8 UTF-8
+nl_BE.UTF-8 UTF-8
+nl_BE.UTF-8@euro UTF-8
+nl_NL.UTF-8 UTF-8
+nl_NL.UTF-8@euro UTF-8
+nn_NO.UTF-8 UTF-8
+no_NO.UTF-8 UTF-8
+pl_PL.UTF-8 UTF-8
+pt_BR.UTF-8 UTF-8
+pt_PT.UTF-8 UTF-8
+pt_PT.UTF-8@euro UTF-8
+ro_RO.UTF-8 UTF-8
+ru_RU.UTF-8 UTF-8
+ru_UA.UTF-8 UTF-8
+sk_SK.UTF-8 UTF-8
+sl_SI.UTF-8 UTF-8
+sq_AL.UTF-8 UTF-8
+sr_YU.UTF-8 UTF-8
+sr_YU.UTF-8@cyrillic UTF-8
+sv_FI.UTF-8 UTF-8
+sv_FI.UTF-8@euro UTF-8
+sv_SE.UTF-8 UTF-8
+th_TH.UTF-8 UTF-8
+tr_TR.UTF-8 UTF-8
+uk_UA.UTF-8 UTF-8
+zh_CN.UTF-8 UTF-8
+zh_HK.UTF-8 UTF-8
+zh_TW.UTF-8 UTF-8
diff --git a/extra/locale/README b/extra/locale/README
index 6ac487c9a..63723f97f 100644
--- a/extra/locale/README
+++ b/extra/locale/README
@@ -1,29 +1,71 @@
+Warning!!!   Warning!!!   Warning!!!   Warning!!!   Warning!!!   Warning!!!
+Warning!!!   Warning!!!   Warning!!!   Warning!!!   Warning!!!   Warning!!!
 
-The programs gen_ctype_from_glibc.c and gen_collate_from_glibc.c
-will generate data files which can be
-used by uClibc ctype, strcoll and setlocale functions to support locales.
-From the comments:
+The programs in this directory are strictly cut-and-paste hack jobs to
+extract the data I needed from glibc's locale database.  I'm ashamed to
+even let them into the light of day, and I consider them complete garbage.
 
-/*
- * Generator locale ctype tables
- * You must have already setuped locale for worked libc (libc5 or glibc)
- *
- * This programm scan /usr/share/locale directories and write
- * ./LOCALE/LC_CTYPE files for system with uclibc
- *
- * Written by Vladimir Oleynik <vodz@usa.net> 2001
- */
+However, they are currently necessary to build the data needed for the
+locale support I've implemented, so I'm forced to include them here.
 
+NOTE: While its possible to use this stuff for native != target arch,
+you'll have to either write a converter to account for endianess and
+struct padding issues, or run the mmap file generator on your target
+arch.  But all these programs will be rewritten at some point.
 
-Sample usage to dump all the data files in a tmp directory:
+All that being said, LC_CTYPE support has been added and supports the
+ctype.h and wctype.h function.  Also, LC_TIME, LC_MONETARY, LC_NUMERIC,
+and LC_MESSAGES are supported wrt SUSv3.  localeconv() works in both
+real and stub locale modes.  nl_langinfo() currently only works with
+real locales enabled.  That will be fixed though.  wc->mb unsupported
+char replacement and basic translit support is on the way as well.
+Finally, some basic 8-bit codeset LC_COLLATE support should be in place
+in the next week or two (similar to what was in the previous locale
+implementation).
 
-gcc gen_ctype_from_glibc.c -o gen_ctype_from_glibc
-gcc gen_collate_from_glibc.c -o gen_ctype_from_glibc
+Also, as one can probably guess, I'm working towards having the locale
+data accessed via a shared mmap.  That will allow non-mmu platforms
+to use this without the current bloat.
 
-mkdir tmp
-cd tmp
-../gen_ctype_from_glibc -d /usr/share/locale -c
-../gen_collate_from_glibc
+Currently, the output of size for my locale_data.o file is
+  text    data     bss     dec     hex filename
+  59072       4       0   59076    e6c4 extra/locale/locale_data.o
+which is for the C locale (automatic of course) + all codesets in
+charmaps/ and all 268 locales in LOCALES.  I estimate that the
+translit support for those 8-bit codesets will add another 7-10k.
 
-Then just move the directory or directories you need (not the .c files)
-to the uClibc locale file directory you set in Config.
+One difference of note is that the special case upper/lower mappings
+in the turkish locale are currently not implemented.  That will be
+fixed.
+
+Manuel
+
+Warning!!!   Warning!!!   Warning!!!   Warning!!!   Warning!!!   Warning!!!
+Warning!!!   Warning!!!   Warning!!!   Warning!!!   Warning!!!   Warning!!!
+
+
+1) In the toplevel dir,  'make headers uClibc_config'  .
+
+2) Create a codesets.txt file in this dir listing the codesets you want
+   to support.  The easiest way to do this is to edit the output of
+	'find ./charmaps -name "*.pairs" > codesets.txt'.
+   NOTE: UTF-8 support is always included if you build with wide chars enabled.
+   NOTE: The files in charmaps/ were created from glibc's charmap files
+         with the awk script at the end of this file.  You can add others
+	 but only single byte codesets are supported.
+
+3) Create a locales.txt file to select the locales you want to support.
+   You can copy and edit the LOCALES file for example.  Other locales could
+   added provided you've included the appropriate codesets in step 2.
+   NOTE: You have to have the approprite locales available for glibc!
+
+4) Run make here.
+
+5) Continue building uClibc from the toplevel dir.
+
+
+
+
+Script used to generate the charmaps/*.pairs files:
+
+cat $1 | awk 'BEGIN { i = 0 } ; { if ($1 == "CHARMAP") i=1 ; else if ($1 == "END") i=0 ; else if (i==1) { sub("/","0",$2) ; sub("<U","0x",$1) ; sub(">","",$1) ; print "{", $2, ",", $1, "}," } }'
diff --git a/extra/locale/charmaps/ASCII.pairs b/extra/locale/charmaps/ASCII.pairs
new file mode 100644
index 000000000..038314d1f
--- /dev/null
+++ b/extra/locale/charmaps/ASCII.pairs
@@ -0,0 +1,128 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
diff --git a/extra/locale/charmaps/CP1251.pairs b/extra/locale/charmaps/CP1251.pairs
new file mode 100644
index 000000000..a5a80d61d
--- /dev/null
+++ b/extra/locale/charmaps/CP1251.pairs
@@ -0,0 +1,255 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
+{ 0x80 , 0x0402 },
+{ 0x81 , 0x0403 },
+{ 0x82 , 0x201A },
+{ 0x83 , 0x0453 },
+{ 0x84 , 0x201E },
+{ 0x85 , 0x2026 },
+{ 0x86 , 0x2020 },
+{ 0x87 , 0x2021 },
+{ 0x88 , 0x20AC },
+{ 0x89 , 0x2030 },
+{ 0x8a , 0x0409 },
+{ 0x8b , 0x2039 },
+{ 0x8c , 0x040A },
+{ 0x8d , 0x040C },
+{ 0x8e , 0x040B },
+{ 0x8f , 0x040F },
+{ 0x90 , 0x0452 },
+{ 0x91 , 0x2018 },
+{ 0x92 , 0x2019 },
+{ 0x93 , 0x201C },
+{ 0x94 , 0x201D },
+{ 0x95 , 0x2022 },
+{ 0x96 , 0x2013 },
+{ 0x97 , 0x2014 },
+{ 0x99 , 0x2122 },
+{ 0x9a , 0x0459 },
+{ 0x9b , 0x203A },
+{ 0x9c , 0x045A },
+{ 0x9d , 0x045C },
+{ 0x9e , 0x045B },
+{ 0x9f , 0x045F },
+{ 0xa0 , 0x00A0 },
+{ 0xa1 , 0x040E },
+{ 0xa2 , 0x045E },
+{ 0xa3 , 0x0408 },
+{ 0xa4 , 0x00A4 },
+{ 0xa5 , 0x0490 },
+{ 0xa6 , 0x00A6 },
+{ 0xa7 , 0x00A7 },
+{ 0xa8 , 0x0401 },
+{ 0xa9 , 0x00A9 },
+{ 0xaa , 0x0404 },
+{ 0xab , 0x00AB },
+{ 0xac , 0x00AC },
+{ 0xad , 0x00AD },
+{ 0xae , 0x00AE },
+{ 0xaf , 0x0407 },
+{ 0xb0 , 0x00B0 },
+{ 0xb1 , 0x00B1 },
+{ 0xb2 , 0x0406 },
+{ 0xb3 , 0x0456 },
+{ 0xb4 , 0x0491 },
+{ 0xb5 , 0x00B5 },
+{ 0xb6 , 0x00B6 },
+{ 0xb7 , 0x00B7 },
+{ 0xb8 , 0x0451 },
+{ 0xb9 , 0x2116 },
+{ 0xba , 0x0454 },
+{ 0xbb , 0x00BB },
+{ 0xbc , 0x0458 },
+{ 0xbd , 0x0405 },
+{ 0xbe , 0x0455 },
+{ 0xbf , 0x0457 },
+{ 0xc0 , 0x0410 },
+{ 0xc1 , 0x0411 },
+{ 0xc2 , 0x0412 },
+{ 0xc3 , 0x0413 },
+{ 0xc4 , 0x0414 },
+{ 0xc5 , 0x0415 },
+{ 0xc6 , 0x0416 },
+{ 0xc7 , 0x0417 },
+{ 0xc8 , 0x0418 },
+{ 0xc9 , 0x0419 },
+{ 0xca , 0x041A },
+{ 0xcb , 0x041B },
+{ 0xcc , 0x041C },
+{ 0xcd , 0x041D },
+{ 0xce , 0x041E },
+{ 0xcf , 0x041F },
+{ 0xd0 , 0x0420 },
+{ 0xd1 , 0x0421 },
+{ 0xd2 , 0x0422 },
+{ 0xd3 , 0x0423 },
+{ 0xd4 , 0x0424 },
+{ 0xd5 , 0x0425 },
+{ 0xd6 , 0x0426 },
+{ 0xd7 , 0x0427 },
+{ 0xd8 , 0x0428 },
+{ 0xd9 , 0x0429 },
+{ 0xda , 0x042A },
+{ 0xdb , 0x042B },
+{ 0xdc , 0x042C },
+{ 0xdd , 0x042D },
+{ 0xde , 0x042E },
+{ 0xdf , 0x042F },
+{ 0xe0 , 0x0430 },
+{ 0xe1 , 0x0431 },
+{ 0xe2 , 0x0432 },
+{ 0xe3 , 0x0433 },
+{ 0xe4 , 0x0434 },
+{ 0xe5 , 0x0435 },
+{ 0xe6 , 0x0436 },
+{ 0xe7 , 0x0437 },
+{ 0xe8 , 0x0438 },
+{ 0xe9 , 0x0439 },
+{ 0xea , 0x043A },
+{ 0xeb , 0x043B },
+{ 0xec , 0x043C },
+{ 0xed , 0x043D },
+{ 0xee , 0x043E },
+{ 0xef , 0x043F },
+{ 0xf0 , 0x0440 },
+{ 0xf1 , 0x0441 },
+{ 0xf2 , 0x0442 },
+{ 0xf3 , 0x0443 },
+{ 0xf4 , 0x0444 },
+{ 0xf5 , 0x0445 },
+{ 0xf6 , 0x0446 },
+{ 0xf7 , 0x0447 },
+{ 0xf8 , 0x0448 },
+{ 0xf9 , 0x0449 },
+{ 0xfa , 0x044A },
+{ 0xfb , 0x044B },
+{ 0xfc , 0x044C },
+{ 0xfd , 0x044D },
+{ 0xfe , 0x044E },
+{ 0xff , 0x044F },
diff --git a/extra/locale/charmaps/ISO-8859-1.pairs b/extra/locale/charmaps/ISO-8859-1.pairs
new file mode 100644
index 000000000..97812a0d5
--- /dev/null
+++ b/extra/locale/charmaps/ISO-8859-1.pairs
@@ -0,0 +1,256 @@
+{ 0x00 , 0x0000 },
+{ 0x01 , 0x0001 },
+{ 0x02 , 0x0002 },
+{ 0x03 , 0x0003 },
+{ 0x04 , 0x0004 },
+{ 0x05 , 0x0005 },
+{ 0x06 , 0x0006 },
+{ 0x07 , 0x0007 },
+{ 0x08 , 0x0008 },
+{ 0x09 , 0x0009 },
+{ 0x0a , 0x000A },
+{ 0x0b , 0x000B },
+{ 0x0c , 0x000C },
+{ 0x0d , 0x000D },
+{ 0x0e , 0x000E },
+{ 0x0f , 0x000F },
+{ 0x10 , 0x0010 },
+{ 0x11 , 0x0011 },
+{ 0x12 , 0x0012 },
+{ 0x13 , 0x0013 },
+{ 0x14 , 0x0014 },
+{ 0x15 , 0x0015 },
+{ 0x16 , 0x0016 },
+{ 0x17 , 0x0017 },
+{ 0x18 , 0x0018 },
+{ 0x19 , 0x0019 },
+{ 0x1a , 0x001A },
+{ 0x1b , 0x001B },
+{ 0x1c , 0x001C },
+{ 0x1d , 0x001D },
+{ 0x1e , 0x001E },
+{ 0x1f , 0x001F },
+{ 0x20 , 0x0020 },
+{ 0x21 , 0x0021 },
+{ 0x22 , 0x0022 },
+{ 0x23 , 0x0023 },
+{ 0x24 , 0x0024 },
+{ 0x25 , 0x0025 },
+{ 0x26 , 0x0026 },
+{ 0x27 , 0x0027 },
+{ 0x28 , 0x0028 },
+{ 0x29 , 0x0029 },
+{ 0x2a , 0x002A },
+{ 0x2b , 0x002B },
+{ 0x2c , 0x002C },
+{ 0x2d , 0x002D },
+{ 0x2e , 0x002E },
+{ 0x2f , 0x002F },
+{ 0x30 , 0x0030 },
+{ 0x31 , 0x0031 },
+{ 0x32 , 0x0032 },
+{ 0x33 , 0x0033 },
+{ 0x34 , 0x0034 },
+{ 0x35 , 0x0035 },
+{ 0x36 , 0x0036 },
+{ 0x37 , 0x0037 },
+{ 0x38 , 0x0038 },
+{ 0x39 , 0x0039 },
+{ 0x3a , 0x003A },
+{ 0x3b , 0x003B },
+{ 0x3c , 0x003C },
+{ 0x3d , 0x003D },
+{ 0x3e , 0x003E },
+{ 0x3f , 0x003F },
+{ 0x40 , 0x0040 },
+{ 0x41 , 0x0041 },
+{ 0x42 , 0x0042 },
+{ 0x43 , 0x0043 },
+{ 0x44 , 0x0044 },
+{ 0x45 , 0x0045 },
+{ 0x46 , 0x0046 },
+{ 0x47 , 0x0047 },
+{ 0x48 , 0x0048 },
+{ 0x49 , 0x0049 },
+{ 0x4a , 0x004A },
+{ 0x4b , 0x004B },
+{ 0x4c , 0x004C },
+{ 0x4d , 0x004D },
+{ 0x4e , 0x004E },
+{ 0x4f , 0x004F },
+{ 0x50 , 0x0050 },
+{ 0x51 , 0x0051 },
+{ 0x52 , 0x0052 },
+{ 0x53 , 0x0053 },
+{ 0x54 , 0x0054 },
+{ 0x55 , 0x0055 },
+{ 0x56 , 0x0056 },
+{ 0x57 , 0x0057 },
+{ 0x58 , 0x0058 },
+{ 0x59 , 0x0059 },
+{ 0x5a , 0x005A },
+{ 0x5b , 0x005B },
+{ 0x5c , 0x005C },
+{ 0x5d , 0x005D },
+{ 0x5e , 0x005E },
+{ 0x5f , 0x005F },
+{ 0x60 , 0x0060 },
+{ 0x61 , 0x0061 },
+{ 0x62 , 0x0062 },
+{ 0x63 , 0x0063 },
+{ 0x64 , 0x0064 },
+{ 0x65 , 0x0065 },
+{ 0x66 , 0x0066 },
+{ 0x67 , 0x0067 },
+{ 0x68 , 0x0068 },
+{ 0x69 , 0x0069 },
+{ 0x6a , 0x006A },
+{ 0x6b , 0x006B },
+{ 0x6c , 0x006C },
+{ 0x6d , 0x006D },
+{ 0x6e , 0x006E },
+{ 0x6f , 0x006F },
+{ 0x70 , 0x0070 },
+{ 0x71 , 0x0071 },
+{ 0x72 , 0x0072 },
+{ 0x73 , 0x0073 },
+{ 0x74 , 0x0074 },
+{ 0x75 , 0x0075 },
+{ 0x76 , 0x0076 },
+{ 0x77 , 0x0077 },
+{ 0x78 , 0x0078 },
+{ 0x79 , 0x0079 },
+{ 0x7a , 0x007A },
+{ 0x7b , 0x007B },
+{ 0x7c , 0x007C },
+{ 0x7d , 0x007D },
+{ 0x7e , 0x007E },
+{ 0x7f , 0x007F },
+{ 0x80 , 0x0080 },
+{ 0x81 , 0x0081 },
+{ 0x82 , 0x0082 },
+{ 0x83 , 0x0083 },
+{ 0x84 , 0x0084 },
+{ 0x85 , 0x0085 },
+{ 0x86 , 0x0086 },
+{ 0x87 , 0x0087 },
+{ 0x88 , 0x0088 },
+{ 0x89 , 0x0089 },
+{ 0x8a , 0x008A },
+{ 0x8b , 0x008B },
+{ 0x8c , 0x008C },
+{ 0x8d , 0x008D },
+{ 0x8e , 0x008E },
+{ 0x8f , 0x008F },
+{ 0x90 , 0x0090 },
+{ 0x91 , 0x0091 },
+{ 0x92 , 0x0092 },
+{ 0x93 , 0x0093 },
+{ 0x94 , 0x0094 },
+{ 0x95 , 0x0095 },
+{ 0x96 , 0x0096 },
+{ 0x97 , 0x0097 },
+{ 0x98 , 0x0098 },
+{ 0x99 , 0x0099 },
+{ 0x9a , 0x009A },
+{ 0x9b , 0x009B },
+{ 0x9c , 0x009C },
+{ 0x9d , 0x009D },
+{ 0x9e , 0x009E },
+{ 0x9f , 0x009F },
+{ 0xa0 , 0x00A0 },
+{ 0xa1 , 0x00A1 },
+{ 0xa2 , 0x00A2 },
+{ 0xa3 , 0x00A3 },
+{ 0xa4 , 0x00A4 },
+{ 0xa5 , 0x00A5 },
+{ 0xa6 , 0x00A6 },
+{ 0xa7 , 0x00A7 },
+{ 0xa8 , 0x00A8 },
+{ 0xa9 , 0x00A9 },
+{ 0xaa , 0x00AA },