1 // helper program is in ~me/encodings.d to make more tables from wikipedia 2 3 /** 4 This is meant to help get data from the wild into utf8 strings 5 so you can work with them easily inside D. 6 7 The main function is convertToUtf8(), which takes a byte array 8 of your raw data (a byte array because it isn't really a D string 9 yet until it is utf8), and a runtime string telling it's current 10 encoding. 11 12 The current encoding argument is meant to come from the data's 13 metadata, and is flexible on exact format - it is case insensitive 14 and takes several variations on the names. 15 16 This way, you should be able to send it the encoding string directly 17 from an XML document, a HTTP header, or whatever you have, and it 18 ought to just work. 19 20 Example: 21 --- 22 auto data = cast(immutable(ubyte)[]) 23 std.file.read("my-windows-file.txt"); 24 string utf8String = convertToUtf8(data, "windows-1252"); 25 // utf8String can now be used 26 --- 27 28 29 The encodings currently implemented for decoding are: 30 $(LIST 31 * UTF-8 (a no-op; it simply casts the array to string) 32 * UTF-16, 33 * UTF-32, 34 * Windows-1252, 35 * ISO 8859 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, and 16. 36 * KOI8-R 37 ) 38 39 It treats ISO 8859-1, Latin-1, and Windows-1252 the same way, since 40 those labels are pretty much de-facto the same thing in wild documents (people mislabel them a lot and I found it more useful to just deal with it than to be pedantic). 41 42 This module currently makes no attempt to look at control characters. 43 */ 44 module arsd.characterencodings; 45 46 import std.string; 47 import std.array; 48 import std.conv; 49 50 /// Like convertToUtf8, but if the encoding is unknown, it just strips all chars > 127 and calls it done instead of throwing 51 string convertToUtf8Lossy(immutable(ubyte)[] data, string dataCharacterEncoding) { 52 try { 53 auto ret = convertToUtf8(data, dataCharacterEncoding); 54 import std.utf; 55 validate(ret); 56 return ret; 57 } catch(Exception e) { 58 string ret; 59 foreach(b; data) 60 if(b < 128) 61 ret ~= b; 62 return ret; 63 } 64 } 65 66 /// Takes data from a given character encoding and returns it as UTF-8 67 string convertToUtf8(immutable(ubyte)[] data, string dataCharacterEncoding) { 68 // just to normalize the passed string... 69 auto encoding = dataCharacterEncoding.toLower(); 70 encoding = encoding.replace(" ", ""); 71 encoding = encoding.replace("-", ""); 72 encoding = encoding.replace("_", ""); 73 // should be good enough. 74 75 switch(encoding) { 76 default: 77 throw new Exception("I don't know how to convert " ~ dataCharacterEncoding ~ " to UTF-8"); 78 // since the input is immutable, these are ok too. 79 // just want to cover all the bases with one runtime function. 80 case "utf16": 81 case "utf16le": 82 return to!string(cast(wstring) data); 83 case "utf32": 84 case "utf32le": 85 return to!string(cast(dstring) data); 86 // FIXME: does the big endian to little endian conversion work? 87 case "ascii": 88 case "usascii": // utf-8 is a superset of ascii 89 case "utf8": 90 return cast(string) data; 91 // and now the various 8 bit encodings we support. 92 case "windows1252": 93 return decodeImpl(data, ISO_8859_1, Windows_1252); 94 case "windows1251": 95 return decodeImpl(data, Windows_1251, Windows_1251_Lower); 96 case "koi8r": 97 return decodeImpl(data, KOI8_R, KOI8_R_Lower); 98 case "latin1": 99 case "iso88591": 100 // Why am I putting Windows_1252 here? A lot of 101 // stuff in the wild is mislabeled, so this will 102 // do some good in the Just Works department. 103 // Regardless, I don't handle the 104 // control char set in that zone anyway right now. 105 return decodeImpl(data, ISO_8859_1, Windows_1252); 106 case "iso88592": 107 return decodeImpl(data, ISO_8859_2); 108 case "iso88593": 109 return decodeImpl(data, ISO_8859_3); 110 case "iso88594": 111 return decodeImpl(data, ISO_8859_4); 112 case "iso88595": 113 return decodeImpl(data, ISO_8859_5); 114 case "iso88596": 115 return decodeImpl(data, ISO_8859_6); 116 case "iso88597": 117 return decodeImpl(data, ISO_8859_7); 118 case "iso88598": 119 return decodeImpl(data, ISO_8859_8); 120 case "iso88599": 121 return decodeImpl(data, ISO_8859_9); 122 case "iso885910": 123 return decodeImpl(data, ISO_8859_10); 124 case "iso885911": 125 return decodeImpl(data, ISO_8859_11); 126 case "iso885913": 127 return decodeImpl(data, ISO_8859_13); 128 case "iso885914": 129 return decodeImpl(data, ISO_8859_14); 130 case "iso885915": 131 return decodeImpl(data, ISO_8859_15); 132 case "iso885916": 133 return decodeImpl(data, ISO_8859_16); 134 } 135 136 assert(0); 137 } 138 139 /// Tries to determine the current encoding based on the content. 140 /// Only really helps with the UTF variants. 141 /// Returns null if it can't be reasonably sure. 142 string tryToDetermineEncoding(in ubyte[] rawdata) { 143 import std.utf; 144 try { 145 validate!string(cast(string) rawdata); 146 // the odds of non stuff validating as utf-8 are pretty low 147 return "UTF-8"; 148 } catch(UTFException t) { 149 // it's definitely not UTF-8! 150 // we'll look at the first few characters. If there's a 151 // BOM, it's probably UTF-16 or UTF-32 152 153 if(rawdata.length > 4) { 154 // not checking for utf8 bom; if it was that, we 155 // wouldn't be here. 156 if(rawdata[0] == 0xff && rawdata[1] == 0xfe) 157 return "UTF-16 LE"; 158 else if(rawdata[0] == 0xfe && rawdata[1] == 0xff) 159 return "UTF-16 BE"; 160 else if(rawdata[0] == 0x00 && rawdata[1] == 0x00 161 && rawdata[2] == 0xfe && rawdata[3] == 0xff) 162 return "UTF-32 BE"; 163 else if(rawdata[0] == 0xff && rawdata[1] == 0xfe 164 && rawdata[2] == 0x00 && rawdata[3] == 0x00) 165 return "UTF-32 LE"; 166 else { 167 // this space is intentionally left blank 168 } 169 } 170 } 171 172 // we don't know with enough confidence. The app will have to find another way. 173 return null; 174 } 175 176 // this function actually does the work, using the translation tables 177 // below. 178 string decodeImpl(in ubyte[] data, in dchar[] chars160to255, in dchar[] chars128to159 = null, in dchar[] chars0to127 = null) 179 in { 180 assert(chars160to255.length == 256 - 160); 181 assert(chars128to159 is null || chars128to159.length == 160 - 128); 182 assert(chars0to127 is null || chars0to127.length == 128 - 0); 183 } 184 out(ret) { 185 import std.utf; 186 validate(ret); 187 } 188 body { 189 string utf8; 190 191 /// I'm sure this could be a lot more efficient, but whatever, it 192 /// works. 193 foreach(octet; data) { 194 if(octet < 128) { 195 if(chars0to127 !is null) 196 utf8 ~= chars0to127[octet]; 197 else 198 utf8 ~= cast(char) octet; // ascii is the same 199 } else if(octet < 160) { 200 if(chars128to159 !is null) 201 utf8 ~= chars128to159[octet - 128]; 202 else 203 utf8 ~= " "; 204 } else { 205 utf8 ~= chars160to255[octet - 160]; 206 } 207 } 208 209 return utf8; 210 } 211 212 213 // Here come the translation tables. 214 215 // this table gives characters for decimal 128 through 159. 216 // the < 128 characters are the same as ascii, and > 159 the same as 217 // iso 8859 1, seen below. 218 immutable dchar[] Windows_1252 = [ 219 '€', ' ', '‚', 'ƒ', '„', '…', '†', '‡', 220 'ˆ', '‰', 'Š', '‹', 'Œ', ' ', 'Ž', ' ', 221 ' ', '‘', '’', '“', '”', '•', '–', '—', 222 '˜', '™', 'š', '›', 'œ', ' ', 'ž', 'Ÿ']; 223 224 // the following tables give the characters from decimal 160 up to 255 225 // in the given encodings. 226 227 immutable dchar[] ISO_8859_1 = [ 228 ' ', '¡', '¢', '£', '¤', '¥', '¦', '§', 229 '¨', '©', 'ª', '«', '¬', '', '®', '¯', 230 '°', '±', '²', '³', '´', 'µ', '¶', '·', 231 '¸', '¹', 'º', '»', '¼', '½', '¾', '¿', 232 'À', 'Á', 'Â', 'Ã', 'Ä', 'Å', 'Æ', 'Ç', 233 'È', 'É', 'Ê', 'Ë', 'Ì', 'Í', 'Î', 'Ï', 234 'Ð', 'Ñ', 'Ò', 'Ó', 'Ô', 'Õ', 'Ö', '×', 235 'Ø', 'Ù', 'Ú', 'Û', 'Ü', 'Ý', 'Þ', 'ß', 236 'à', 'á', 'â', 'ã', 'ä', 'å', 'æ', 'ç', 237 'è', 'é', 'ê', 'ë', 'ì', 'í', 'î', 'ï', 238 'ð', 'ñ', 'ò', 'ó', 'ô', 'õ', 'ö', '÷', 239 'ø', 'ù', 'ú', 'û', 'ü', 'ý', 'þ', 'ÿ']; 240 241 immutable dchar[] ISO_8859_2 = [ 242 ' ', 'Ą', '˘', 'Ł', '¤', 'Ľ', 'Ś', '§', 243 '¨', 'Š', 'Ş', 'Ť', 'Ź', '', 'Ž', 'Ż', 244 '°', 'ą', '˛', 'ł', '´', 'ľ', 'ś', 'ˇ', 245 '¸', 'š', 'ş', 'ť', 'ź', '˝', 'ž', 'ż', 246 'Ŕ', 'Á', 'Â', 'Ă', 'Ä', 'Ĺ', 'Ć', 'Ç', 247 'Č', 'É', 'Ę', 'Ë', 'Ě', 'Í', 'Î', 'Ď', 248 'Đ', 'Ń', 'Ň', 'Ó', 'Ô', 'Ő', 'Ö', '×', 249 'Ř', 'Ů', 'Ú', 'Ű', 'Ü', 'Ý', 'Ţ', 'ß', 250 'ŕ', 'á', 'â', 'ă', 'ä', 'ĺ', 'ć', 'ç', 251 'č', 'é', 'ę', 'ë', 'ě', 'í', 'î', 'ď', 252 'đ', 'ń', 'ň', 'ó', 'ô', 'ő', 'ö', '÷', 253 'ř', 'ů', 'ú', 'ű', 'ü', 'ý', 'ţ', '˙']; 254 255 immutable dchar[] ISO_8859_3 = [ 256 ' ', 'Ħ', '˘', '£', '¤', ' ', 'Ĥ', '§', 257 '¨', 'İ', 'Ş', 'Ğ', 'Ĵ', '', ' ', 'Ż', 258 '°', 'ħ', '²', '³', '´', 'µ', 'ĥ', '·', 259 '¸', 'ı', 'ş', 'ğ', 'ĵ', '½', ' ', 'ż', 260 'À', 'Á', 'Â', ' ', 'Ä', 'Ċ', 'Ĉ', 'Ç', 261 'È', 'É', 'Ê', 'Ë', 'Ì', 'Í', 'Î', 'Ï', 262 ' ', 'Ñ', 'Ò', 'Ó', 'Ô', 'Ġ', 'Ö', '×', 263 'Ĝ', 'Ù', 'Ú', 'Û', 'Ü', 'Ŭ', 'Ŝ', 'ß', 264 'à', 'á', 'â', ' ', 'ä', 'ċ', 'ĉ', 'ç', 265 'è', 'é', 'ê', 'ë', 'ì', 'í', 'î', 'ï', 266 ' ', 'ñ', 'ò', 'ó', 'ô', 'ġ', 'ö', '÷', 267 'ĝ', 'ù', 'ú', 'û', 'ü', 'ŭ', 'ŝ', '˙']; 268 269 immutable dchar[] ISO_8859_4 = [ 270 ' ', 'Ą', 'ĸ', 'Ŗ', '¤', 'Ĩ', 'Ļ', '§', 271 '¨', 'Š', 'Ē', 'Ģ', 'Ŧ', '', 'Ž', '¯', 272 '°', 'ą', '˛', 'ŗ', '´', 'ĩ', 'ļ', 'ˇ', 273 '¸', 'š', 'ē', 'ģ', 'ŧ', 'Ŋ', 'ž', 'ŋ', 274 'Ā', 'Á', 'Â', 'Ã', 'Ä', 'Å', 'Æ', 'Į', 275 'Č', 'É', 'Ę', 'Ë', 'Ė', 'Í', 'Î', 'Ī', 276 'Đ', 'Ņ', 'Ō', 'Ķ', 'Ô', 'Õ', 'Ö', '×', 277 'Ø', 'Ų', 'Ú', 'Û', 'Ü', 'Ũ', 'Ū', 'ß', 278 'ā', 'á', 'â', 'ã', 'ä', 'å', 'æ', 'į', 279 'č', 'é', 'ę', 'ë', 'ė', 'í', 'î', 'ī', 280 'đ', 'ņ', 'ō', 'ķ', 'ô', 'õ', 'ö', '÷', 281 'ø', 'ų', 'ú', 'û', 'ü', 'ũ', 'ū', '˙']; 282 283 immutable dchar[] ISO_8859_5 = [ 284 ' ', 'Ё', 'Ђ', 'Ѓ', 'Є', 'Ѕ', 'І', 'Ї', 285 'Ј', 'Љ', 'Њ', 'Ћ', 'Ќ', '', 'Ў', 'Џ', 286 'А', 'Б', 'В', 'Г', 'Д', 'Е', 'Ж', 'З', 287 'И', 'Й', 'К', 'Л', 'М', 'Н', 'О', 'П', 288 'Р', 'С', 'Т', 'У', 'Ф', 'Х', 'Ц', 'Ч', 289 'Ш', 'Щ', 'Ъ', 'Ы', 'Ь', 'Э', 'Ю', 'Я', 290 'а', 'б', 'в', 'г', 'д', 'е', 'ж', 'з', 291 'и', 'й', 'к', 'л', 'м', 'н', 'о', 'п', 292 'р', 'с', 'т', 'у', 'ф', 'х', 'ц', 'ч', 293 'ш', 'щ', 'ъ', 'ы', 'ь', 'э', 'ю', 'я', 294 '№', 'ё', 'ђ', 'ѓ', 'є', 'ѕ', 'і', 'ї', 295 'ј', 'љ', 'њ', 'ћ', 'ќ', '§', 'ў', 'џ']; 296 297 immutable dchar[] ISO_8859_6 = [ 298 ' ', ' ', ' ', ' ', '¤', ' ', ' ', ' ', 299 ' ', ' ', ' ', ' ', '،', '', ' ', ' ', 300 ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', 301 ' ', ' ', ' ', '؛', ' ', ' ', ' ', '؟', 302 ' ', 'ء', 'آ', 'أ', 'ؤ', 'إ', 'ئ', 'ا', 303 'ب', 'ة', 'ت', 'ث', 'ج', 'ح', 'خ', 'د', 304 'ذ', 'ر', 'ز', 'س', 'ش', 'ص', 'ض', 'ط', 305 'ظ', 'ع', 'غ', ' ', ' ', ' ', ' ', ' ', 306 'ـ', 'ف', 'ق', 'ك', 'ل', 'م', 'ن', 'ه', 307 'و', 'ى', 'ي', 'ً', 'ٌ', 'ٍ', 'َ', 'ُ', 308 'ِ', 'ّ', 'ْ', ' ', ' ', ' ', ' ', ' ', 309 ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ']; 310 311 immutable dchar[] ISO_8859_7 = [ 312 ' ', '‘', '’', '£', '€', '₯', '¦', '§', 313 '¨', '©', 'ͺ', '«', '¬', '', ' ', '―', 314 '°', '±', '²', '³', '΄', '΅', 'Ά', '·', 315 'Έ', 'Ή', 'Ί', '»', 'Ό', '½', 'Ύ', 'Ώ', 316 'ΐ', 'Α', 'Β', 'Γ', 'Δ', 'Ε', 'Ζ', 'Η', 317 'Θ', 'Ι', 'Κ', 'Λ', 'Μ', 'Ν', 'Ξ', 'Ο', 318 'Π', 'Ρ', ' ', 'Σ', 'Τ', 'Υ', 'Φ', 'Χ', 319 'Ψ', 'Ω', 'Ϊ', 'Ϋ', 'ά', 'έ', 'ή', 'ί', 320 'ΰ', 'α', 'β', 'γ', 'δ', 'ε', 'ζ', 'η', 321 'θ', 'ι', 'κ', 'λ', 'μ', 'ν', 'ξ', 'ο', 322 'π', 'ρ', 'ς', 'σ', 'τ', 'υ', 'φ', 'χ', 323 'ψ', 'ω', 'ϊ', 'ϋ', 'ό', 'ύ', 'ώ', ' ']; 324 325 immutable dchar[] ISO_8859_8 = [ 326 ' ', ' ', '¢', '£', '¤', '¥', '¦', '§', 327 '¨', '©', '×', '«', '¬', '', '®', '¯', 328 '°', '±', '²', '³', '´', 'µ', '¶', '·', 329 '¸', '¹', '÷', '»', '¼', '½', '¾', ' ', 330 ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', 331 ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', 332 ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', 333 ' ', ' ', ' ', ' ', ' ', ' ', ' ', '‗', 334 'א', 'ב', 'ג', 'ד', 'ה', 'ו', 'ז', 'ח', 335 'ט', 'י', 'ך', 'כ', 'ל', 'ם', 'מ', 'ן', 336 'נ', 'ס', 'ע', 'ף', 'פ', 'ץ', 'צ', 'ק', 337 // v v those are wrong 338 'ר', 'ש', 'ת', ' ', ' ', ' ', ' ', ' ']; // FIXME: those ones marked wrong are supposed to be left to right and right to left markers, not spaces. lol maybe it isn't wrong 339 340 immutable dchar[] ISO_8859_9 = [ 341 ' ', '¡', '¢', '£', '¤', '¥', '¦', '§', 342 '¨', '©', 'ª', '«', '¬', '', '®', '¯', 343 '°', '±', '²', '³', '´', 'µ', '¶', '·', 344 '¸', '¹', 'º', '»', '¼', '½', '¾', '¿', 345 'À', 'Á', 'Â', 'Ã', 'Ä', 'Å', 'Æ', 'Ç', 346 'È', 'É', 'Ê', 'Ë', 'Ì', 'Í', 'Î', 'Ï', 347 'Ğ', 'Ñ', 'Ò', 'Ó', 'Ô', 'Õ', 'Ö', '×', 348 'Ø', 'Ù', 'Ú', 'Û', 'Ü', 'İ', 'Ş', 'ß', 349 'à', 'á', 'â', 'ã', 'ä', 'å', 'æ', 'ç', 350 'è', 'é', 'ê', 'ë', 'ì', 'í', 'î', 'ï', 351 'ğ', 'ñ', 'ò', 'ó', 'ô', 'õ', 'ö', '÷', 352 'ø', 'ù', 'ú', 'û', 'ü', 'ı', 'ş', 'ÿ']; 353 354 immutable dchar[] ISO_8859_10 = [ 355 ' ', 'Ą', 'Ē', 'Ģ', 'Ī', 'Ĩ', 'Ķ', '§', 356 'Ļ', 'Đ', 'Š', 'Ŧ', 'Ž', '', 'Ū', 'Ŋ', 357 '°', 'ą', 'ē', 'ģ', 'ī', 'ĩ', 'ķ', '·', 358 'ļ', 'đ', 'š', 'ŧ', 'ž', '―', 'ū', 'ŋ', 359 'Ā', 'Á', 'Â', 'Ã', 'Ä', 'Å', 'Æ', 'Į', 360 'Č', 'É', 'Ę', 'Ë', 'Ė', 'Í', 'Î', 'Ï', 361 'Ð', 'Ņ', 'Ō', 'Ó', 'Ô', 'Õ', 'Ö', 'Ũ', 362 'Ø', 'Ų', 'Ú', 'Û', 'Ü', 'Ý', 'Þ', 'ß', 363 'ā', 'á', 'â', 'ã', 'ä', 'å', 'æ', 'į', 364 'č', 'é', 'ę', 'ë', 'ė', 'í', 'î', 'ï', 365 'ð', 'ņ', 'ō', 'ó', 'ô', 'õ', 'ö', 'ũ', 366 'ø', 'ų', 'ú', 'û', 'ü', 'ý', 'þ', 'ĸ']; 367 368 immutable dchar[] ISO_8859_11 = [ 369 ' ', 'ก', 'ข', 'ฃ', 'ค', 'ฅ', 'ฆ', 'ง', 370 'จ', 'ฉ', 'ช', 'ซ', 'ฌ', 'ญ', 'ฎ', 'ฏ', 371 'ฐ', 'ฑ', 'ฒ', 'ณ', 'ด', 'ต', 'ถ', 'ท', 372 'ธ', 'น', 'บ', 'ป', 'ผ', 'ฝ', 'พ', 'ฟ', 373 'ภ', 'ม', 'ย', 'ร', 'ฤ', 'ล', 'ฦ', 'ว', 374 'ศ', 'ษ', 'ส', 'ห', 'ฬ', 'อ', 'ฮ', 'ฯ', 375 'ะ', 'ั', 'า', 'ำ', 'ิ', 'ี', 'ึ', 'ื', 376 'ุ', 'ู', 'ฺ', ' ', ' ', ' ', ' ', '฿', 377 'เ', 'แ', 'โ', 'ใ', 'ไ', 'ๅ', 'ๆ', '็', 378 '่', '้', '๊', '๋', '์', 'ํ', '๎', '๏', 379 '๐', '๑', '๒', '๓', '๔', '๕', '๖', '๗', 380 '๘', '๙', '๚', '๛', ' ', ' ', ' ', ' ']; 381 382 immutable dchar[] ISO_8859_13 = [ 383 ' ', '”', '¢', '£', '¤', '„', '¦', '§', 384 'Ø', '©', 'Ŗ', '«', '¬', '', '®', 'Æ', 385 '°', '±', '²', '³', '“', 'µ', '¶', '·', 386 'ø', '¹', 'ŗ', '»', '¼', '½', '¾', 'æ', 387 'Ą', 'Į', 'Ā', 'Ć', 'Ä', 'Å', 'Ę', 'Ē', 388 'Č', 'É', 'Ź', 'Ė', 'Ģ', 'Ķ', 'Ī', 'Ļ', 389 'Š', 'Ń', 'Ņ', 'Ó', 'Ō', 'Ő', 'Ö', '×', 390 'Ų', 'Ł', 'Ś', 'Ū', 'Ü', 'Ż', 'Ž', 'ß', 391 'ą', 'į', 'ā', 'ć', 'ä', 'å', 'ę', 'ē', 392 'č', 'é', 'ź', 'ė', 'ģ', 'ķ', 'ī', 'ļ', 393 'š', 'ń', 'ņ', 'ó', 'ō', 'ő', 'ö', '÷', 394 'ų', 'ł', 'ś', 'ū', 'ü', 'ż', 'ž', '’']; 395 396 immutable dchar[] ISO_8859_14 = [ 397 ' ', 'Ḃ', 'ḃ', '£', 'Ċ', 'ċ', 'Ḋ', '§', 398 'Ẁ', '©', 'Ẃ', 'ḋ', 'Ỳ', '', '®', 'Ÿ', 399 'Ḟ', 'ḟ', 'Ġ', 'ġ', 'Ṁ', 'ṁ', '¶', 'Ṗ', 400 'ẁ', 'ṗ', 'ẃ', 'Ṡ', 'ỳ', 'Ẅ', 'ẅ', 'ṡ', 401 'À', 'Á', 'Â', 'Ã', 'Ä', 'Å', 'Æ', 'Ç', 402 'È', 'É', 'Ê', 'Ë', 'Ì', 'Í', 'Î', 'Ï', 403 'Ŵ', 'Ñ', 'Ò', 'Ó', 'Ô', 'Ő', 'Ö', 'Ṫ', 404 'Ø', 'Ù', 'Ú', 'Û', 'Ü', 'Ý', 'Ŷ', 'ß', 405 'à', 'á', 'â', 'ã', 'ä', 'å', 'æ', 'ç', 406 'è', 'é', 'ê', 'ë', 'ì', 'í', 'î', 'ï', 407 'ŵ', 'ñ', 'ò', 'ó', 'ô', 'ő', 'ö', 'ṫ', 408 'ø', 'ù', 'ú', 'û', 'ü', 'ý', 'ŷ', 'ÿ']; 409 410 immutable dchar[] ISO_8859_15 = [ 411 ' ', '¡', '¢', '£', '€', '¥', 'Š', '§', 412 'š', '©', 'ª', '«', '¬', '', '®', '¯', 413 '°', '±', '²', '³', 'Ž', 'µ', '¶', '·', 414 'ž', '¹', 'º', '»', 'Œ', 'œ', 'Ÿ', '¿', 415 'À', 'Á', 'Â', 'Ã', 'Ä', 'Å', 'Æ', 'Ç', 416 'È', 'É', 'Ê', 'Ë', 'Ì', 'Í', 'Î', 'Ï', 417 'Ð', 'Ñ', 'Ò', 'Ó', 'Ô', 'Ő', 'Ö', '×', 418 'Ø', 'Ù', 'Ú', 'Û', 'Ü', 'Ý', 'Þ', 'ß', 419 'à', 'á', 'â', 'ã', 'ä', 'å', 'æ', 'ç', 420 'è', 'é', 'ê', 'ë', 'ì', 'í', 'î', 'ï', 421 'ð', 'ñ', 'ò', 'ó', 'ô', 'ő', 'ö', '÷', 422 'ø', 'ù', 'ú', 'û', 'ü', 'ý', 'þ', 'ÿ']; 423 424 immutable dchar[] ISO_8859_16 = [ 425 ' ', 'Ą', 'ą', 'Ł', '€', '„', 'Š', '§', 426 'š', '©', 'Ș', '«', 'Ź', '', 'ź', 'Ż', 427 '°', '±', 'Č', 'ł', 'Ž', '”', '¶', '·', 428 'ž', 'č', 'ș', '»', 'Œ', 'œ', 'Ÿ', 'ż', 429 'À', 'Á', 'Â', 'Ă', 'Ä', 'Ć', 'Æ', 'Ç', 430 'È', 'É', 'Ê', 'Ë', 'Ì', 'Í', 'Î', 'Ï', 431 'Ð', 'Ń', 'Ò', 'Ó', 'Ô', 'Ő', 'Ö', 'Ś', 432 'Ű', 'Ù', 'Ú', 'Û', 'Ü', 'Ę', 'Ț', 'ß', 433 'à', 'á', 'â', 'ă', 'ä', 'ć', 'æ', 'ç', 434 'è', 'é', 'ê', 'ë', 'ì', 'í', 'î', 'ï', 435 'đ', 'ń', 'ò', 'ó', 'ô', 'ő', 'ö', 'ś', 436 'ű', 'ù', 'ú', 'û', 'ü', 'ę', 'ț', 'ÿ']; 437 438 immutable dchar[] KOI8_R_Lower = [ 439 '─', '│', '┌', '┐', '└', '┘', '├', '┤', 440 '┬', '┴', '┼', '▀', '▄', '█', '▌', '▐', 441 '░', '▒', '▓', '⌠', '■', '∙', '√', '≈', 442 '≤', '≥', '\u00a0', '⌡', '°', '²', '·', '÷']; 443 444 immutable dchar[] KOI8_R = [ 445 '═', '║', '╒', 'ё', '╓', '╔', '╕', '╖', 446 '╗', '╘', '╙', '╚', '╛', '╜', '╝', '╞', 447 '╟', '╠', '╡', 'ё', '╢', '╣', '╤', '╥', 448 '╦', '╧', '╨', '╩', '╪', '╫', '╬', '©', 449 'ю', 'а', 'б', 'ц', 'д', 'е', 'ф', 'г', 450 'х', 'и', 'й', 'к', 'л', 'м', 'н', 'о', 451 'п', 'я', 'р', 'с', 'т', 'у', 'ж', 'в', 452 'ь', 'ы', 'з', 'ш', 'э', 'щ', 'ч', 'ъ', 453 'ю', 'а', 'б', 'ц', 'д', 'е', 'ф', 'г', 454 'х', 'и', 'й', 'к', 'л', 'м', 'н', 'о', 455 'п', 'я', 'р', 'с', 'т', 'у', 'ж', 'в', 456 'ь', 'ы', 'з', 'ш', 'э', 'щ', 'ч', 'ъ']; 457 458 immutable dchar[] Windows_1251_Lower = [ 459 'Ђ', 'Ѓ', '‚', 'ѓ', '„', '…', '†', '‡', 460 '€', '‰', 'Љ', '‹', 'Њ', 'Ќ', 'Ћ', 'Џ', 461 'ђ', '‘', '’', '“', '”', '•', '–', '—', 462 ' ', '™', 'љ', '›', 'њ', 'ќ', 'ћ', 'џ']; 463 464 immutable dchar[] Windows_1251 = [ 465 ' ', 'Ў', 'ў', 'Ј', '¤', 'Ґ', '¦', '§', 466 'Ё', '©', 'Є', '«', '¬', '', '®', 'Ї', 467 '°', '±', 'І', 'і', 'ґ', 'µ', '¶', '·', 468 'ё', '№', 'є', '»', 'ј', 'Ѕ', 'ѕ', 'ї', 469 'А', 'Б', 'В', 'Г', 'Д', 'Е', 'Ж', 'З', 470 'И', 'Й', 'К', 'Л', 'М', 'Н', 'О', 'П', 471 'Р', 'С', 'Т', 'У', 'Ф', 'Х', 'Ц', 'Ч', 472 'Ш', 'Щ', 'Ъ', 'Ы', 'Ь', 'Э', 'Ю', 'Я', 473 'а', 'б', 'в', 'г', 'д', 'е', 'ж', 'з', 474 'и', 'й', 'к', 'л', 'м', 'н', 'о', 'п', 475 'р', 'с', 'т', 'у', 'ф', 'х', 'ц', 'ч', 476 'ш', 'щ', 'ъ', 'ы', 'ь', 'э', 'ю', 'я']; 477