Mercurial > repos > guerler > hhblits

annotate lib/python3.8/site-packages/pip/_vendor/chardet/sbcsgroupprober.py @ 0:9e54283cc701 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
author guerler
date Mon, 27 Jul 2020 03:47:31 -0400
parents
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
1 ######################## BEGIN LICENSE BLOCK ########################
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
2 # The Original Code is Mozilla Universal charset detector code.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
3 #
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
4 # The Initial Developer of the Original Code is
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
5 # Netscape Communications Corporation.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
6 # Portions created by the Initial Developer are Copyright (C) 2001
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
7 # the Initial Developer. All Rights Reserved.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
8 #
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
9 # Contributor(s):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
10 # Mark Pilgrim - port to Python
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
11 # Shy Shalom - original C code
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
12 #
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
13 # This library is free software; you can redistribute it and/or
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
14 # modify it under the terms of the GNU Lesser General Public
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
15 # License as published by the Free Software Foundation; either
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
16 # version 2.1 of the License, or (at your option) any later version.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
17 #
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
18 # This library is distributed in the hope that it will be useful,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
19 # but WITHOUT ANY WARRANTY; without even the implied warranty of
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
20 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
21 # Lesser General Public License for more details.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
22 #
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
23 # You should have received a copy of the GNU Lesser General Public
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
24 # License along with this library; if not, write to the Free Software
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
25 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
26 # 02110-1301 USA
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
27 ######################### END LICENSE BLOCK #########################
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
28
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
29 from .charsetgroupprober import CharSetGroupProber
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
30 from .sbcharsetprober import SingleByteCharSetProber
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
31 from .langcyrillicmodel import (Win1251CyrillicModel, Koi8rModel,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
32 Latin5CyrillicModel, MacCyrillicModel,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
33 Ibm866Model, Ibm855Model)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
34 from .langgreekmodel import Latin7GreekModel, Win1253GreekModel
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
35 from .langbulgarianmodel import Latin5BulgarianModel, Win1251BulgarianModel
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
36 # from .langhungarianmodel import Latin2HungarianModel, Win1250HungarianModel
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
37 from .langthaimodel import TIS620ThaiModel
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
38 from .langhebrewmodel import Win1255HebrewModel
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
39 from .hebrewprober import HebrewProber
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
40 from .langturkishmodel import Latin5TurkishModel
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
41
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
42
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
43 class SBCSGroupProber(CharSetGroupProber):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
44 def __init__(self):
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
45 super(SBCSGroupProber, self).__init__()
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
46 self.probers = [
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
47 SingleByteCharSetProber(Win1251CyrillicModel),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
48 SingleByteCharSetProber(Koi8rModel),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
49 SingleByteCharSetProber(Latin5CyrillicModel),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
50 SingleByteCharSetProber(MacCyrillicModel),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
51 SingleByteCharSetProber(Ibm866Model),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
52 SingleByteCharSetProber(Ibm855Model),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
53 SingleByteCharSetProber(Latin7GreekModel),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
54 SingleByteCharSetProber(Win1253GreekModel),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
55 SingleByteCharSetProber(Latin5BulgarianModel),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
56 SingleByteCharSetProber(Win1251BulgarianModel),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
57 # TODO: Restore Hungarian encodings (iso-8859-2 and windows-1250)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
58 # after we retrain model.
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
59 # SingleByteCharSetProber(Latin2HungarianModel),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
60 # SingleByteCharSetProber(Win1250HungarianModel),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
61 SingleByteCharSetProber(TIS620ThaiModel),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
62 SingleByteCharSetProber(Latin5TurkishModel),
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
63 ]
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
64 hebrew_prober = HebrewProber()
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
65 logical_hebrew_prober = SingleByteCharSetProber(Win1255HebrewModel,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
66 False, hebrew_prober)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
67 visual_hebrew_prober = SingleByteCharSetProber(Win1255HebrewModel, True,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
68 hebrew_prober)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
69 hebrew_prober.set_model_probers(logical_hebrew_prober, visual_hebrew_prober)
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
70 self.probers.extend([hebrew_prober, logical_hebrew_prober,
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
71 visual_hebrew_prober])
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
72
9e54283cc701 "planemo upload commit d12c32a45bcd441307e632fca6d9af7d60289d44"
guerler
parents:
diff changeset
73 self.reset()