Mercurial > repos > shellac > sam_consensus_v3

comparison env/lib/python3.9/site-packages/chardet/sbcsgroupprober.py @ 0:4f3585e2f14b draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit 60cee0fc7c0cda8592644e1aad72851dec82c959"
author shellac
date Mon, 22 Mar 2021 18:12:50 +0000
parents
children
comparison
equal deleted inserted replaced
-1:000000000000 0:4f3585e2f14b
1 ######################## BEGIN LICENSE BLOCK ########################
2 # The Original Code is Mozilla Universal charset detector code.
3 #
4 # The Initial Developer of the Original Code is
5 # Netscape Communications Corporation.
6 # Portions created by the Initial Developer are Copyright (C) 2001
7 # the Initial Developer. All Rights Reserved.
8 #
9 # Contributor(s):
10 # Mark Pilgrim - port to Python
11 # Shy Shalom - original C code
12 #
13 # This library is free software; you can redistribute it and/or
14 # modify it under the terms of the GNU Lesser General Public
15 # License as published by the Free Software Foundation; either
16 # version 2.1 of the License, or (at your option) any later version.
17 #
18 # This library is distributed in the hope that it will be useful,
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
21 # Lesser General Public License for more details.
22 #
23 # You should have received a copy of the GNU Lesser General Public
24 # License along with this library; if not, write to the Free Software
25 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
26 # 02110-1301 USA
27 ######################### END LICENSE BLOCK #########################
28
29 from .charsetgroupprober import CharSetGroupProber
30 from .hebrewprober import HebrewProber
31 from .langbulgarianmodel import (ISO_8859_5_BULGARIAN_MODEL,
32 WINDOWS_1251_BULGARIAN_MODEL)
33 from .langgreekmodel import ISO_8859_7_GREEK_MODEL, WINDOWS_1253_GREEK_MODEL
34 from .langhebrewmodel import WINDOWS_1255_HEBREW_MODEL
35 # from .langhungarianmodel import (ISO_8859_2_HUNGARIAN_MODEL,
36 # WINDOWS_1250_HUNGARIAN_MODEL)
37 from .langrussianmodel import (IBM855_RUSSIAN_MODEL, IBM866_RUSSIAN_MODEL,
38 ISO_8859_5_RUSSIAN_MODEL, KOI8_R_RUSSIAN_MODEL,
39 MACCYRILLIC_RUSSIAN_MODEL,
40 WINDOWS_1251_RUSSIAN_MODEL)
41 from .langthaimodel import TIS_620_THAI_MODEL
42 from .langturkishmodel import ISO_8859_9_TURKISH_MODEL
43 from .sbcharsetprober import SingleByteCharSetProber
44
45
46 class SBCSGroupProber(CharSetGroupProber):
47 def __init__(self):
48 super(SBCSGroupProber, self).__init__()
49 hebrew_prober = HebrewProber()
50 logical_hebrew_prober = SingleByteCharSetProber(WINDOWS_1255_HEBREW_MODEL,
51 False, hebrew_prober)
52 # TODO: See if using ISO-8859-8 Hebrew model works better here, since
53 # it's actually the visual one
54 visual_hebrew_prober = SingleByteCharSetProber(WINDOWS_1255_HEBREW_MODEL,
55 True, hebrew_prober)
56 hebrew_prober.set_model_probers(logical_hebrew_prober,
57 visual_hebrew_prober)
58 # TODO: ORDER MATTERS HERE. I changed the order vs what was in master
59 # and several tests failed that did not before. Some thought
60 # should be put into the ordering, and we should consider making
61 # order not matter here, because that is very counter-intuitive.
62 self.probers = [
63 SingleByteCharSetProber(WINDOWS_1251_RUSSIAN_MODEL),
64 SingleByteCharSetProber(KOI8_R_RUSSIAN_MODEL),
65 SingleByteCharSetProber(ISO_8859_5_RUSSIAN_MODEL),
66 SingleByteCharSetProber(MACCYRILLIC_RUSSIAN_MODEL),
67 SingleByteCharSetProber(IBM866_RUSSIAN_MODEL),
68 SingleByteCharSetProber(IBM855_RUSSIAN_MODEL),
69 SingleByteCharSetProber(ISO_8859_7_GREEK_MODEL),
70 SingleByteCharSetProber(WINDOWS_1253_GREEK_MODEL),
71 SingleByteCharSetProber(ISO_8859_5_BULGARIAN_MODEL),
72 SingleByteCharSetProber(WINDOWS_1251_BULGARIAN_MODEL),
73 # TODO: Restore Hungarian encodings (iso-8859-2 and windows-1250)
74 # after we retrain model.
75 # SingleByteCharSetProber(ISO_8859_2_HUNGARIAN_MODEL),
76 # SingleByteCharSetProber(WINDOWS_1250_HUNGARIAN_MODEL),
77 SingleByteCharSetProber(TIS_620_THAI_MODEL),
78 SingleByteCharSetProber(ISO_8859_9_TURKISH_MODEL),
79 hebrew_prober,
80 logical_hebrew_prober,
81 visual_hebrew_prober,
82 ]
83 self.reset()