Mercurial > repos > shellac > guppy_basecaller

comparison env/lib/python3.7/site-packages/chardet/sjisprober.py @ 5:9b1c78e6ba9c draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit 6c0a8142489327ece472c84e558c47da711a9142"
author shellac
date Mon, 01 Jun 2020 08:59:25 -0400
parents 79f47841a781
children
comparison
equal deleted inserted replaced
4:79f47841a781 5:9b1c78e6ba9c
1 ######################## BEGIN LICENSE BLOCK ########################
2 # The Original Code is mozilla.org code.
3 #
4 # The Initial Developer of the Original Code is
5 # Netscape Communications Corporation.
6 # Portions created by the Initial Developer are Copyright (C) 1998
7 # the Initial Developer. All Rights Reserved.
8 #
9 # Contributor(s):
10 # Mark Pilgrim - port to Python
11 #
12 # This library is free software; you can redistribute it and/or
13 # modify it under the terms of the GNU Lesser General Public
14 # License as published by the Free Software Foundation; either
15 # version 2.1 of the License, or (at your option) any later version.
16 #
17 # This library is distributed in the hope that it will be useful,
18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20 # Lesser General Public License for more details.
21 #
22 # You should have received a copy of the GNU Lesser General Public
23 # License along with this library; if not, write to the Free Software
24 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
25 # 02110-1301 USA
26 ######################### END LICENSE BLOCK #########################
27
28 from .mbcharsetprober import MultiByteCharSetProber
29 from .codingstatemachine import CodingStateMachine
30 from .chardistribution import SJISDistributionAnalysis
31 from .jpcntx import SJISContextAnalysis
32 from .mbcssm import SJIS_SM_MODEL
33 from .enums import ProbingState, MachineState
34
35
36 class SJISProber(MultiByteCharSetProber):
37 def __init__(self):
38 super(SJISProber, self).__init__()
39 self.coding_sm = CodingStateMachine(SJIS_SM_MODEL)
40 self.distribution_analyzer = SJISDistributionAnalysis()
41 self.context_analyzer = SJISContextAnalysis()
42 self.reset()
43
44 def reset(self):
45 super(SJISProber, self).reset()
46 self.context_analyzer.reset()
47
48 @property
49 def charset_name(self):
50 return self.context_analyzer.charset_name
51
52 @property
53 def language(self):
54 return "Japanese"
55
56 def feed(self, byte_str):
57 for i in range(len(byte_str)):
58 coding_state = self.coding_sm.next_state(byte_str[i])
59 if coding_state == MachineState.ERROR:
60 self.logger.debug('%s %s prober hit error at byte %s',
61 self.charset_name, self.language, i)
62 self._state = ProbingState.NOT_ME
63 break
64 elif coding_state == MachineState.ITS_ME:
65 self._state = ProbingState.FOUND_IT
66 break
67 elif coding_state == MachineState.START:
68 char_len = self.coding_sm.get_current_charlen()
69 if i == 0:
70 self._last_char[1] = byte_str[0]
71 self.context_analyzer.feed(self._last_char[2 - char_len:],
72 char_len)
73 self.distribution_analyzer.feed(self._last_char, char_len)
74 else:
75 self.context_analyzer.feed(byte_str[i + 1 - char_len:i + 3
76 - char_len], char_len)
77 self.distribution_analyzer.feed(byte_str[i - 1:i + 1],
78 char_len)
79
80 self._last_char[0] = byte_str[-1]
81
82 if self.state == ProbingState.DETECTING:
83 if (self.context_analyzer.got_enough_data() and
84 (self.get_confidence() > self.SHORTCUT_THRESHOLD)):
85 self._state = ProbingState.FOUND_IT
86
87 return self.state
88
89 def get_confidence(self):
90 context_conf = self.context_analyzer.get_confidence()
91 distrib_conf = self.distribution_analyzer.get_confidence()
92 return max(context_conf, distrib_conf)