mirror of
https://github.com/sqlmapproject/sqlmap.git
synced 2026-06-11 02:31:51 +00:00
Update of chardet to 4.0.0
This commit is contained in:
Miroslav Štampar
parent
af0e46316a
commit
d47aacbfba
17 changed files with 26950 additions and 1636 deletions
|
|
@ -188,7 +188,7 @@ c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6 lib/core/data.
|
|||
48797d6c34dd9bb8a53f7f3794c85f4288d82a9a1d6be7fcf317d388cb20d4b3 lib/core/replication.py
|
||||
0b8c38a01bb01f843d94a6c5f2075ee47520d0c4aa799cecea9c3e2c5a4a23a6 lib/core/revision.py
|
||||
888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c lib/core/session.py
|
||||
b035d4789fe95557807a3e0b3e16822a31dba2f46cb952cd46c9bce270fa7368 lib/core/settings.py
|
||||
6e5a916581e1e7032cdd451737be69af0bbc42cfc7e46d44a43df0ee272c361d lib/core/settings.py
|
||||
cd5a66deee8963ba8e7e9af3dd36eb5e8127d4d68698811c29e789655f507f82 lib/core/shell.py
|
||||
bcb5d8090d5e3e0ef2a586ba09ba80eef0c6d51feb0f611ed25299fbb254f725 lib/core/subprocessng.py
|
||||
70ea3768f1b3062b22d20644df41c86238157ec80dd43da40545c620714273c6 lib/core/target.py
|
||||
|
|
@ -208,7 +208,7 @@ c5b258be7485089fac9d9cd179960e774fbd85e62836dc67cce76cc028bb6aeb lib/parse/hand
|
|||
d2e771cdacef25ee3fdc0e0355b92e7cd1b68f5edc2756ffc19f75d183ba2c73 lib/parse/payloads.py
|
||||
455ab0ec63e55cd56ce4a884b85bdc089223155008cab0f3696da5a33118f95b lib/parse/sitemap.py
|
||||
1be3da334411657461421b8a26a0f2ff28e1af1e28f1e963c6c92768f9b0847c lib/request/basicauthhandler.py
|
||||
132abf563aeaaf0108b7e3932cfcc9680c8f445e992de4ee71ceed1ddf60bc29 lib/request/basic.py
|
||||
08e31008a089f8ff058afaccc06f42d6ca103408f9167cfb1831428e608374a4 lib/request/basic.py
|
||||
bc61bc944b81a7670884f82231033a6ac703324b34b071c9834886a92e249d0e lib/request/chunkedhandler.py
|
||||
09c2d8786fb5280f5f14a7b4345ecb2e7c2ca836ee06a6cf9b51770df923d94c lib/request/comparison.py
|
||||
9236db2abad1b1d368a3c5a5beb655055fd2445faba57a4172db264b06105bd4 lib/request/connect.py
|
||||
|
|
@ -573,10 +573,10 @@ f862301288d2ba2f913860bb901cd5197e72c0461e3330164f90375f713b8199 thirdparty/bot
|
|||
0ffccae46cb3a15b117acd0790b2738a5b45417d1b2822ceac57bdff10ef3bff thirdparty/chardet/big5freq.py
|
||||
901c476dd7ad0693deef1ae56fe7bdf748a8b7ae20fde1922dddf6941eff8773 thirdparty/chardet/big5prober.py
|
||||
df0a164bad8aac6a282b2ab3e334129e315b2696ba57b834d9d68089b4f0725f thirdparty/chardet/chardistribution.py
|
||||
e9b0eef1822246e49c5f871af4881bd14ebd4c0d8f1975c37a3e82738ffd90ee thirdparty/chardet/charsetgroupprober.py
|
||||
1992d17873fa151467e3786f48ea060b161a984acacf2a7a460390c55782de48 thirdparty/chardet/charsetgroupprober.py
|
||||
2929b0244ae3ca9ca3d1b459982e45e5e33b73c61080b6088d95e29ed64db2d8 thirdparty/chardet/charsetprober.py
|
||||
558a7fe9ccb2922e6c1e05c34999d75b8ab5a1e94773772ef40c904d7eeeba0f thirdparty/chardet/codingstatemachine.py
|
||||
3ca4f31e449bb5b1c3a92f4fcae8cc6d7ef8ab56bc98ca5e4130d5b10859311c thirdparty/chardet/compat.py
|
||||
e34cebeb0202670927c72b8b18670838fcaf7bc0d379b0426dbbedb6f9e6a794 thirdparty/chardet/compat.py
|
||||
4d9e37e105fccf306c9d4bcbffcc26e004154d9d9992a10440bfe5370f5ff68c thirdparty/chardet/cp949prober.py
|
||||
0229b075bf5ab357492996853541f63a158854155de9990927f58ae6c358f1c5 thirdparty/chardet/enums.py
|
||||
924caa560d58c370c8380309d9b765c9081415086e1c05bc7541ac913a0d5927 thirdparty/chardet/escprober.py
|
||||
|
|
@ -589,26 +589,25 @@ d77a7a10fe3245ac6a9cfe221edc47389e91db3c47ab5fe6f214d18f3559f797 thirdparty/cha
|
|||
257f25b3078a2e69c2c2693c507110b0b824affacffe411bbe2bc2e2a3ceae57 thirdparty/chardet/gb2312freq.py
|
||||
806bc85a2f568438c4fb14171ef348cab9cbbc46cc01883251267ae4751fca5c thirdparty/chardet/gb2312prober.py
|
||||
737499f8aee1bf2cc663a251019c4983027fb144bd93459892f318d34601605a thirdparty/chardet/hebrewprober.py
|
||||
62c3f9c1096c1c9d9ab85d516497f2a624ab080eff6d08919b7112fcd23bebe6 thirdparty/chardet/__init__.py
|
||||
99665a5a6bd9921c1f044013f4ed58ea74537cace14fb1478504d302e8dba940 thirdparty/chardet/__init__.py
|
||||
be9989bf606ed09f209cc5513c730579f4d1be8fe16b59abc8b8a0f0207080e8 thirdparty/chardet/jisfreq.py
|
||||
3d894da915104fc2ccddc4f91661c63f48a2b1c1654d6103f763002ef06e9e0a thirdparty/chardet/jpcntx.py
|
||||
d47a904bd3dbb678f5c508318ad24cbf0f17ea42abe4ea1c90d09959f110acf1 thirdparty/chardet/langbulgarianmodel.py
|
||||
2ce0da8efb1eb47f3bc980c340a0360942d7507f3bb48db6ddd85f8e1f59c7d7 thirdparty/chardet/langcyrillicmodel.py
|
||||
f18016edb53c6304896a9d2420949b3ccc35044ab31a35b3a9ca9fd168142800 thirdparty/chardet/langgreekmodel.py
|
||||
2529ea984e44eb6b432d33d3bcba50b20e6038c3b83db75646f57b02f91cd070 thirdparty/chardet/langhebrewmodel.py
|
||||
4616a96121b997465a3be555e056a7e6c5b4591190aa1c0133ad72c77cb1c8e0 thirdparty/chardet/langhungarianmodel.py
|
||||
f25d35ef71aefd6e86f26c6640e4c417896cd98744ec5c567f74244b11065c94 thirdparty/chardet/langthaimodel.py
|
||||
5b6d9e44d26ca88eae5807f05d22955969c27ab62aac8f1d6504e6fccd254459 thirdparty/chardet/langturkishmodel.py
|
||||
afab6f3ad3bc16a8676d6041e55e1ccdc9757d6338a41f651a259053ef20becc thirdparty/chardet/langbulgarianmodel.py
|
||||
d5c32edb05203c1f1b43645b5634782cdc020844e043e0f0a34120dbfb81d75b thirdparty/chardet/langgreekmodel.py
|
||||
bab3262471c85ed0b069602acb5cc463fe129b0c0dcedef7d1b0ceb635f3463b thirdparty/chardet/langhebrewmodel.py
|
||||
383022b2fa827deb3c07815ec8cfcf83d1d8dd90e7132682893e01c72ce873ac thirdparty/chardet/langhungarianmodel.py
|
||||
a69a0a3862fd38f763f40e025321bc478f336e75edf4c37559778261ea5aeac7 thirdparty/chardet/langthaimodel.py
|
||||
1f795d89c23fae196fd2bdd5169556b542fa5f7d16cb9f7abfbfd81f3dac11d8 thirdparty/chardet/langturkishmodel.py
|
||||
4b6228391845937f451053a54855ad815c9b4623fa87b0652e574755c94d914f thirdparty/chardet/latin1prober.py
|
||||
011f797851fdbeea927ef2d064df8be628de6b6e4d3810a85eac3cb393bdc4b4 thirdparty/chardet/mbcharsetprober.py
|
||||
87a4d19e762ad8ec46d56743e493b2c5c755a67edd1b4abebc1f275abe666e1e thirdparty/chardet/mbcsgroupprober.py
|
||||
498df6c15205dc7cdc8d8dc1684b29cbd99eb5b3522b120807444a3e7eed8e92 thirdparty/chardet/mbcssm.py
|
||||
2c34a90a5743085958c149069300f6a05c4b94f5885974f4f5a907ff63e263be thirdparty/chardet/sbcharsetprober.py
|
||||
d48a6b70207f935a9f9a7c460ba3016f110b94aa83dec716e92f1823075ec970 thirdparty/chardet/sbcsgroupprober.py
|
||||
9e6c8ccaec731bcec337a2b7464d8c53324b30b47af4cad6a5d9c7ccec155304 thirdparty/chardet/sbcharsetprober.py
|
||||
86a79f42e5e6885c83040ace8ee8c7ea177a5855e5383d64582b310e18f1e557 thirdparty/chardet/sbcsgroupprober.py
|
||||
208b7e9598f4589a8ae2b9946732993f8189944f0a504b45615b98f7a7a4e4c4 thirdparty/chardet/sjisprober.py
|
||||
a8bd35ef8952644e38d9e076d679e4b53f7f55c0327b4ee5685594794ae3b6d6 thirdparty/chardet/universaldetector.py
|
||||
0e96535c25f49d41d7c6443db2be06671181fe1bde67a856b77b8cf7872058ab thirdparty/chardet/universaldetector.py
|
||||
21d0fcbf7cd63ac07c38b8b23e2fb2fdfab08a9445c55f4d73578a04b4ae204c thirdparty/chardet/utf8prober.py
|
||||
b29dc1d3c9ab0d707ea5fdcaf5fa89ff37831ce08b0bc46b9e04320c56a9ffb8 thirdparty/chardet/version.py
|
||||
0380882c501df0c4551b51e85cfa78e622bd44b956c95ef76b512dc04f13be7f thirdparty/chardet/version.py
|
||||
1c1ee8a91eb20f8038ace6611610673243d0f71e2b7566111698462182c7efdd thirdparty/clientform/clientform.py
|
||||
e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 thirdparty/clientform/__init__.py
|
||||
4e8a7811e12e69074159db5e28c11c18e4de29e175f50f96a3febf0a3e643b34 thirdparty/colorama/ansi.py
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from lib.core.enums import OS
|
|||
from thirdparty import six
|
||||
|
||||
# sqlmap version (<major>.<minor>.<month>.<monthly commit>)
|
||||
VERSION = "1.10.6.56"
|
||||
VERSION = "1.10.6.57"
|
||||
TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
|
||||
TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
|
||||
VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)
|
||||
|
|
|
|||
|
|
@ -258,10 +258,11 @@ def getHeuristicCharEncoding(page):
|
|||
|
||||
>>> getHeuristicCharEncoding(b"<html></html>")
|
||||
'ascii'
|
||||
>>> getHeuristicCharEncoding(b'<!DOCTYPE html><html><head><meta charset="windows-1251"><title>\\xd2\\xe5\\xf1\\xf2</title></head><body>\\xc2 \\xf1\\xee\\xee\\xf2\\xe2\\xe5\\xf2\\xf1\\xf2\\xe2\\xe8\\xe8 \\xf1 \\xef\\xf0\\xe8\\xed\\xf6\\xe8\\xef\\xe0\\xec\\xe8 \\xf0\\xe0\\xe1\\xee\\xf2\\xfb \\xf3\\xf2\\xe8\\xeb\\xe8\\xf2\\xfb \\xe0\\xe2\\xf2\\xee\\xec\\xe0\\xf2\\xe8\\xf7\\xe5\\xf1\\xea\\xee\\xe3\\xee \\xee\\xef\\xf0\\xe5\\xe4\\xe5\\xeb\\xe5\\xed\\xe8\\xff \\xea\\xee\\xe4\\xe8\\xf0\\xee\\xe2\\xea\\xe8, \\xed\\xe0\\xec \\xf2\\xf0\\xe5\\xe1\\xf3\\xe5\\xf2\\xf1\\xff \\xef\\xf0\\xe5\\xe4\\xee\\xf1\\xf2\\xe0\\xe2\\xe8\\xf2\\xfc \\xe7\\xed\\xe0\\xf7\\xe8\\xf2\\xe5\\xeb\\xfc\\xed\\xee \\xe1\\xee\\xeb\\xe5\\xe5 \\xe4\\xeb\\xe8\\xed\\xed\\xfb\\xe9 \\xf4\\xf0\\xe0\\xe3\\xec\\xe5\\xed\\xf2 \\xf2\\xe5\\xea\\xf1\\xf2\\xe0 \\xed\\xe0 \\xf0\\xf3\\xf1\\xf1\\xea\\xee\\xec \\xff\\xe7\\xfb\\xea\\xe5. \\xdd\\xf2\\xee \\xed\\xe5\\xee\\xb1\\xf5\\xee\\xe4\\xe8\\xec\\xee \\xe4\\xeb\\xff \\xf2\\xee\\xe3\\xee, \\xf7\\xf2\\xee\\xf1\\xfb \\xf1\\xf2\\xe0\\xf2\\xe8\\xf1\\xf2\\xe8\\xf7\\xe5\\xf1\\xea\\xe8\\xe9 \\xe0\\xed\\xe0\\xeb\\xe8\\xe7\\xe0\\xf2\\xee\\xf0 \\xf7\\xe0\\xf1\\xf2\\xee\\xf2\\xed\\xee\\xf1\\xf2\\xe8 \\xf1\\xe8\\xec\\xe2\\xee\\xeb\\xee\\xe2 \\xe8 \\xe4\\xe2\\xf3\\xf5\\xe1\\xf3\\xea\\xe2\\xe5\\xed\\xed\\xfb\\xf5 \\xf1\\xee\\xf7\\xe5\\xf2\\xe0\\xed\\xe8\\xb9 \\xf1\\xec\\xee\\xe3 \\xf1 \\xe2\\xfb\\xf1\\xee\\xea\\xee\\xb9 \\xf1\\xf2\\xe5\\xef\\xe5\\xed\\xfc\\xf2 \\xf3\\xe2\\xe5\\xf0\\xe5\\xed\\xed\\xee\\xf1\\xf2\\xe8 \\xe7\\xe0\\xf4\\xe8\\xea\\xf1\\xe8\\xf0\\xee\\xe2\\xe0\\xf2\\xfc \\xe8\\xec\\xe5\\xed\\xed\\xee \\xf1\\xf2\\xe0\\xed\\xe4\\xe0\\xf0\\xf2 Windows-1251, \\xe0 \\xed\\xe5 MacCyrillic \\xe8\\xeb\\xe8 ISO-8859-5. \\xd0\\xf3\\xf1\\xf1\\xea\\xe8\\xb9 \\xff\\xe7\\xfb\\xea \\xee\\xe1\\xbb\\xe0\\xe4\\xe0\\xe5\\xf2 \\xf3\\xed\\xe8\\xea\\xe0\\xeb\\xfc\\xed\\xfb\\xec \\xf0\\xe0\\xf1\\xef\\xf0\\xe5\\xe4\\xe5\\xeb\\xe5\\xed\\xe8\\xe5\\xec \\xe3\\xeb\\xe0\\xf1\\xed\\xfb\\xf5 \\xe8 \\xf1\\xee\\xe3\\xeb\\xe0\\xf1\\xed\\xfb\\xf5 \\xe1\\xf3\\xea\\xe2, \\xf2\\xe0\\xea\\xe8\\xf5 \\xea\\xe0\\xea \\xee, \\xe5, \\xe0, \\xe8, \\xed, \\xf2, \\xea\\xee\\xf2\\xee\\xf0\\xfb\\xe5 \\xe2 \\xf0\\xe0\\xe7\\xed\\xfb\\xf5 \\xea\\xee\\xe4\\xee\\xe2\\xfb\\xf5 \\xf1\\xf2\\xf0\\xe0\\xed\\xe8\\xf6\\xe0\\xf5 \\xe7\\xe0\\xed\\xe8\\xec\\xe0\\xf3\\xf2 \\xf1\\xee\\xe2\\xe5\\xf0\\xf8\\xe5\\xed\\xed\\xee \\xf0\\xe0\\xe7\\xed\\xfb\\xe5 \\xef\\xee\\xf7\\xe8\\xf6\\xe8\\xe8 \\xe2 \\xf2\\xe0\\xe1\\xeb\\xe8\\xf6\\xe5 \\xe1\\xe0\\xb9\\xf2\\xee\\xe2. \\xca\\xee\\xe3\\xe4\\xe0 \\xf2\\xe5\\xea\\xf1\\xf2\\xe0 \\xf1\\xf2\\xe0\\xed\\xee\\xe2\\xe8\\xf2\\xf1\\xff \\xe4\\xee\\xf1\\xf2\\xe0\\xf2\\xee\\xf7\\xed\\xee \\xec\\xed\\xee\\xe3\\xee, \\xe2\\xe5\\xf0\\xee\\xff\\xf2\\xed\\xee\\xf1\\xf2\\xfc \\xee\\xf8\\xe8\\xe1\\xea\\xe8 \\xf1\\xed\\xe8\\xe6\\xe0\\xe5\\xf2\\xf1\\xff \\xef\\xf0\\xe0\\xea\\xf2\\xe8\\xf7\\xe5\\xf1\\xea\\xe8 \\xe4\\xee \\xed\\xf3\\xeb\\xff. \\xcc\\xfb \\xe4\\xee\\xe1\\xe0\\xe2\\xeb\\xff\\xe5\\xec \\xe5\\xf9\\xe5 \\xed\\xe5\\xf1\\xea\\xee\\xeb\\xfc\\xea\\xee \\xef\\xf0\\xe5\\xe4\\xeb\\xee\\xe6\\xe5\\xed\\xe8\\xb9, \\xf7\\xf2\\xee\\xf1\\xfb \\xf0\\xe0\\xf1\\xf8\\xe8\\xf0\\xe8\\xf2\\xfc \\xe2\\xfb\\xe1\\xee\\xf0\\xea\\xf3 \\xe4\\xe0\\xed\\xed\\xfb\\xf5 \\xe4\\xeb\\xff \\xea\\xee\\xf0\\xf0\\xe5\\xea\\xf2\\xed\\xee\\xe3\\xee \\xf2\\xe5\\xf1\\xf2\\xe8\\xf0\\xee\\xe2\\xe0\\xed\\xe8\\xff \\xe2\\xe0\\xf8\\xe5\\xb9 \\xe1\\xe8\\xe1\\xeb\\xe8\\xee\\xf2\\xe5\\xea\\xe8 \\xe2 \\xf1\\xf0\\xe5\\xe4\\xe5 Python.</body></html>')
|
||||
'windows-1251'
|
||||
"""
|
||||
|
||||
key = (len(page), hash(page))
|
||||
|
||||
retVal = kb.cache.encoding.get(key)
|
||||
if retVal is None:
|
||||
retVal = detect(page[:HEURISTIC_PAGE_SIZE_THRESHOLD])["encoding"]
|
||||
|
|
|
|||
48
thirdparty/chardet/__init__.py
vendored
48
thirdparty/chardet/__init__.py
vendored
|
|
@ -16,11 +16,14 @@
|
|||
######################### END LICENSE BLOCK #########################
|
||||
|
||||
|
||||
from .compat import PY2, PY3
|
||||
from .universaldetector import UniversalDetector
|
||||
from .enums import InputState
|
||||
from .version import __version__, VERSION
|
||||
|
||||
|
||||
__all__ = ['UniversalDetector', 'detect', 'detect_all', '__version__', 'VERSION']
|
||||
|
||||
|
||||
def detect(byte_str):
|
||||
"""
|
||||
Detect the encoding of the given byte string.
|
||||
|
|
@ -31,9 +34,50 @@ def detect(byte_str):
|
|||
if not isinstance(byte_str, bytearray):
|
||||
if not isinstance(byte_str, bytes):
|
||||
raise TypeError('Expected object of type bytes or bytearray, got: '
|
||||
'{0}'.format(type(byte_str)))
|
||||
'{}'.format(type(byte_str)))
|
||||
else:
|
||||
byte_str = bytearray(byte_str)
|
||||
detector = UniversalDetector()
|
||||
detector.feed(byte_str)
|
||||
return detector.close()
|
||||
|
||||
|
||||
def detect_all(byte_str):
|
||||
"""
|
||||
Detect all the possible encodings of the given byte string.
|
||||
|
||||
:param byte_str: The byte sequence to examine.
|
||||
:type byte_str: ``bytes`` or ``bytearray``
|
||||
"""
|
||||
if not isinstance(byte_str, bytearray):
|
||||
if not isinstance(byte_str, bytes):
|
||||
raise TypeError('Expected object of type bytes or bytearray, got: '
|
||||
'{}'.format(type(byte_str)))
|
||||
else:
|
||||
byte_str = bytearray(byte_str)
|
||||
|
||||
detector = UniversalDetector()
|
||||
detector.feed(byte_str)
|
||||
detector.close()
|
||||
|
||||
if detector._input_state == InputState.HIGH_BYTE:
|
||||
results = []
|
||||
for prober in detector._charset_probers:
|
||||
if prober.get_confidence() > detector.MINIMUM_THRESHOLD:
|
||||
charset_name = prober.charset_name
|
||||
lower_charset_name = prober.charset_name.lower()
|
||||
# Use Windows encoding name instead of ISO-8859 if we saw any
|
||||
# extra Windows-specific bytes
|
||||
if lower_charset_name.startswith('iso-8859'):
|
||||
if detector._has_win_bytes:
|
||||
charset_name = detector.ISO_WIN_MAP.get(lower_charset_name,
|
||||
charset_name)
|
||||
results.append({
|
||||
'encoding': charset_name,
|
||||
'confidence': prober.get_confidence(),
|
||||
'language': prober.language,
|
||||
})
|
||||
if len(results) > 0:
|
||||
return sorted(results, key=lambda result: -result['confidence'])
|
||||
|
||||
return [detector.result]
|
||||
|
|
|
|||
1
thirdparty/chardet/charsetgroupprober.py
vendored
1
thirdparty/chardet/charsetgroupprober.py
vendored
|
|
@ -73,6 +73,7 @@ class CharSetGroupProber(CharSetProber):
|
|||
continue
|
||||
if state == ProbingState.FOUND_IT:
|
||||
self._best_guess_prober = prober
|
||||
self._state = ProbingState.FOUND_IT
|
||||
return self.state
|
||||
elif state == ProbingState.NOT_ME:
|
||||
prober.active = False
|
||||
|
|
|
|||
6
thirdparty/chardet/compat.py
vendored
6
thirdparty/chardet/compat.py
vendored
|
|
@ -25,10 +25,12 @@ import sys
|
|||
if sys.version_info < (3, 0):
|
||||
PY2 = True
|
||||
PY3 = False
|
||||
base_str = (str, unicode)
|
||||
string_types = (str, unicode)
|
||||
text_type = unicode
|
||||
iteritems = dict.iteritems
|
||||
else:
|
||||
PY2 = False
|
||||
PY3 = True
|
||||
base_str = (bytes, str)
|
||||
string_types = (bytes, str)
|
||||
text_type = str
|
||||
iteritems = dict.items
|
||||
|
|
|
|||
4862
thirdparty/chardet/langbulgarianmodel.py
vendored
4862
thirdparty/chardet/langbulgarianmodel.py
vendored
File diff suppressed because it is too large
Load diff
333
thirdparty/chardet/langcyrillicmodel.py
vendored
333
thirdparty/chardet/langcyrillicmodel.py
vendored
|
|
@ -1,333 +0,0 @@
|
|||
######################## BEGIN LICENSE BLOCK ########################
|
||||
# The Original Code is Mozilla Communicator client code.
|
||||
#
|
||||
# The Initial Developer of the Original Code is
|
||||
# Netscape Communications Corporation.
|
||||
# Portions created by the Initial Developer are Copyright (C) 1998
|
||||
# the Initial Developer. All Rights Reserved.
|
||||
#
|
||||
# Contributor(s):
|
||||
# Mark Pilgrim - port to Python
|
||||
#
|
||||
# This library is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU Lesser General Public
|
||||
# License as published by the Free Software Foundation; either
|
||||
# version 2.1 of the License, or (at your option) any later version.
|
||||
#
|
||||
# This library is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public
|
||||
# License along with this library; if not, write to the Free Software
|
||||
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
|
||||
# 02110-1301 USA
|
||||
######################### END LICENSE BLOCK #########################
|
||||
|
||||
# KOI8-R language model
|
||||
# Character Mapping Table:
|
||||
KOI8R_char_to_order_map = (
|
||||
255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, # 00
|
||||
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, # 10
|
||||
253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, # 20
|
||||
252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253, # 30
|
||||
253,142,143,144,145,146,147,148,149,150,151,152, 74,153, 75,154, # 40
|
||||
155,156,157,158,159,160,161,162,163,164,165,253,253,253,253,253, # 50
|
||||
253, 71,172, 66,173, 65,174, 76,175, 64,176,177, 77, 72,178, 69, # 60
|
||||
67,179, 78, 73,180,181, 79,182,183,184,185,253,253,253,253,253, # 70
|
||||
191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206, # 80
|
||||
207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222, # 90
|
||||
223,224,225, 68,226,227,228,229,230,231,232,233,234,235,236,237, # a0
|
||||
238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253, # b0
|
||||
27, 3, 21, 28, 13, 2, 39, 19, 26, 4, 23, 11, 8, 12, 5, 1, # c0
|
||||
15, 16, 9, 7, 6, 14, 24, 10, 17, 18, 20, 25, 30, 29, 22, 54, # d0
|
||||
59, 37, 44, 58, 41, 48, 53, 46, 55, 42, 60, 36, 49, 38, 31, 34, # e0
|
||||
35, 43, 45, 32, 40, 52, 56, 33, 61, 62, 51, 57, 47, 63, 50, 70, # f0
|
||||
)
|
||||
|
||||
win1251_char_to_order_map = (
|
||||
255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, # 00
|
||||
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, # 10
|
||||
253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, # 20
|
||||
252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253, # 30
|
||||
253,142,143,144,145,146,147,148,149,150,151,152, 74,153, 75,154, # 40
|
||||
155,156,157,158,159,160,161,162,163,164,165,253,253,253,253,253, # 50
|
||||
253, 71,172, 66,173, 65,174, 76,175, 64,176,177, 77, 72,178, 69, # 60
|
||||
67,179, 78, 73,180,181, 79,182,183,184,185,253,253,253,253,253, # 70
|
||||
191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,
|
||||
207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,
|
||||
223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,
|
||||
239,240,241,242,243,244,245,246, 68,247,248,249,250,251,252,253,
|
||||
37, 44, 33, 46, 41, 48, 56, 51, 42, 60, 36, 49, 38, 31, 34, 35,
|
||||
45, 32, 40, 52, 53, 55, 58, 50, 57, 63, 70, 62, 61, 47, 59, 43,
|
||||
3, 21, 10, 19, 13, 2, 24, 20, 4, 23, 11, 8, 12, 5, 1, 15,
|
||||
9, 7, 6, 14, 39, 26, 28, 22, 25, 29, 54, 18, 17, 30, 27, 16,
|
||||
)
|
||||
|
||||
latin5_char_to_order_map = (
|
||||
255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, # 00
|
||||
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, # 10
|
||||
253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, # 20
|
||||
252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253, # 30
|
||||
253,142,143,144,145,146,147,148,149,150,151,152, 74,153, 75,154, # 40
|
||||
155,156,157,158,159,160,161,162,163,164,165,253,253,253,253,253, # 50
|
||||
253, 71,172, 66,173, 65,174, 76,175, 64,176,177, 77, 72,178, 69, # 60
|
||||
67,179, 78, 73,180,181, 79,182,183,184,185,253,253,253,253,253, # 70
|
||||
191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,
|
||||
207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,
|
||||
223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,
|
||||
37, 44, 33, 46, 41, 48, 56, 51, 42, 60, 36, 49, 38, 31, 34, 35,
|
||||
45, 32, 40, 52, 53, 55, 58, 50, 57, 63, 70, 62, 61, 47, 59, 43,
|
||||
3, 21, 10, 19, 13, 2, 24, 20, 4, 23, 11, 8, 12, 5, 1, 15,
|
||||
9, 7, 6, 14, 39, 26, 28, 22, 25, 29, 54, 18, 17, 30, 27, 16,
|
||||
239, 68,240,241,242,243,244,245,246,247,248,249,250,251,252,255,
|
||||
)
|
||||
|
||||
macCyrillic_char_to_order_map = (
|
||||
255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, # 00
|
||||
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, # 10
|
||||
253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, # 20
|
||||
252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253, # 30
|
||||
253,142,143,144,145,146,147,148,149,150,151,152, 74,153, 75,154, # 40
|
||||
155,156,157,158,159,160,161,162,163,164,165,253,253,253,253,253, # 50
|
||||
253, 71,172, 66,173, 65,174, 76,175, 64,176,177, 77, 72,178, 69, # 60
|
||||
67,179, 78, 73,180,181, 79,182,183,184,185,253,253,253,253,253, # 70
|
||||
37, 44, 33, 46, 41, 48, 56, 51, 42, 60, 36, 49, 38, 31, 34, 35,
|
||||
45, 32, 40, 52, 53, 55, 58, 50, 57, 63, 70, 62, 61, 47, 59, 43,
|
||||
191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,
|
||||
207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,
|
||||
223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,
|
||||
239,240,241,242,243,244,245,246,247,248,249,250,251,252, 68, 16,
|
||||
3, 21, 10, 19, 13, 2, 24, 20, 4, 23, 11, 8, 12, 5, 1, 15,
|
||||
9, 7, 6, 14, 39, 26, 28, 22, 25, 29, 54, 18, 17, 30, 27,255,
|
||||
)
|
||||
|
||||
IBM855_char_to_order_map = (
|
||||
255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, # 00
|
||||
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, # 10
|
||||
253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, # 20
|
||||
252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253, # 30
|
||||
253,142,143,144,145,146,147,148,149,150,151,152, 74,153, 75,154, # 40
|
||||
155,156,157,158,159,160,161,162,163,164,165,253,253,253,253,253, # 50
|
||||
253, 71,172, 66,173, 65,174, 76,175, 64,176,177, 77, 72,178, 69, # 60
|
||||
67,179, 78, 73,180,181, 79,182,183,184,185,253,253,253,253,253, # 70
|
||||
191,192,193,194, 68,195,196,197,198,199,200,201,202,203,204,205,
|
||||
206,207,208,209,210,211,212,213,214,215,216,217, 27, 59, 54, 70,
|
||||
3, 37, 21, 44, 28, 58, 13, 41, 2, 48, 39, 53, 19, 46,218,219,
|
||||
220,221,222,223,224, 26, 55, 4, 42,225,226,227,228, 23, 60,229,
|
||||
230,231,232,233,234,235, 11, 36,236,237,238,239,240,241,242,243,
|
||||
8, 49, 12, 38, 5, 31, 1, 34, 15,244,245,246,247, 35, 16,248,
|
||||
43, 9, 45, 7, 32, 6, 40, 14, 52, 24, 56, 10, 33, 17, 61,249,
|
||||
250, 18, 62, 20, 51, 25, 57, 30, 47, 29, 63, 22, 50,251,252,255,
|
||||
)
|
||||
|
||||
IBM866_char_to_order_map = (
|
||||
255,255,255,255,255,255,255,255,255,255,254,255,255,254,255,255, # 00
|
||||
255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255, # 10
|
||||
253,253,253,253,253,253,253,253,253,253,253,253,253,253,253,253, # 20
|
||||
252,252,252,252,252,252,252,252,252,252,253,253,253,253,253,253, # 30
|
||||
253,142,143,144,145,146,147,148,149,150,151,152, 74,153, 75,154, # 40
|
||||
155,156,157,158,159,160,161,162,163,164,165,253,253,253,253,253, # 50
|
||||
253, 71,172, 66,173, 65,174, 76,175, 64,176,177, 77, 72,178, 69, # 60
|
||||
67,179, 78, 73,180,181, 79,182,183,184,185,253,253,253,253,253, # 70
|
||||
37, 44, 33, 46, 41, 48, 56, 51, 42, 60, 36, 49, 38, 31, 34, 35,
|
||||
45, 32, 40, 52, 53, 55, 58, 50, 57, 63, 70, 62, 61, 47, 59, 43,
|
||||
3, 21, 10, 19, 13, 2, 24, 20, 4, 23, 11, 8, 12, 5, 1, 15,
|
||||
191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,
|
||||
207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,
|
||||
223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,
|
||||
9, 7, 6, 14, 39, 26, 28, 22, 25, 29, 54, 18, 17, 30, 27, 16,
|
||||
239, 68,240,241,242,243,244,245,246,247,248,249,250,251,252,255,
|
||||
)
|
||||
|
||||
# Model Table:
|
||||
# total sequences: 100%
|
||||
# first 512 sequences: 97.6601%
|
||||
# first 1024 sequences: 2.3389%
|
||||
# rest sequences: 0.1237%
|
||||
# negative sequences: 0.0009%
|
||||
RussianLangModel = (
|
||||
0,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,1,1,3,3,3,3,1,3,3,3,2,3,2,3,3,
|
||||
3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,0,3,2,2,2,2,2,0,0,2,
|
||||
3,3,3,2,3,3,3,3,3,3,3,3,3,3,2,3,3,0,0,3,3,3,3,3,3,3,3,3,2,3,2,0,
|
||||
0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
3,3,3,2,2,3,3,3,3,3,3,3,3,3,2,3,3,0,0,3,3,3,3,3,3,3,3,2,3,3,1,0,
|
||||
0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
3,2,3,2,3,3,3,3,3,3,3,3,3,3,3,3,3,0,0,3,3,3,3,3,3,3,3,3,3,3,2,1,
|
||||
0,0,0,0,0,0,0,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,3,3,0,0,3,3,3,3,3,3,3,3,3,3,3,2,1,
|
||||
0,0,0,0,0,1,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
3,3,3,3,3,3,3,3,2,2,2,3,1,3,3,1,3,3,3,3,2,2,3,0,2,2,2,3,3,2,1,0,
|
||||
0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,
|
||||
3,3,3,3,3,3,2,3,3,3,3,3,2,2,3,2,3,3,3,2,1,2,2,0,1,2,2,2,2,2,2,0,
|
||||
0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,
|
||||
3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,3,0,2,2,3,3,2,1,2,0,
|
||||
0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,1,0,0,2,0,0,0,0,0,0,0,0,0,
|
||||
3,3,3,3,3,3,2,3,3,1,2,3,2,2,3,2,3,3,3,3,2,2,3,0,3,2,2,3,1,1,1,0,
|
||||
0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
3,3,3,3,3,3,3,3,2,2,3,3,3,3,3,2,3,3,3,3,2,2,2,0,3,3,3,2,2,2,2,0,
|
||||
0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
3,3,3,3,3,3,3,3,3,3,2,3,2,3,3,3,3,3,3,2,3,2,2,0,1,3,2,1,2,2,1,0,
|
||||
0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,
|
||||
3,3,3,3,3,3,3,3,3,3,3,2,1,1,3,0,1,1,1,1,2,1,1,0,2,2,2,1,2,0,1,0,
|
||||
0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
3,3,3,3,3,3,2,3,3,2,2,2,2,1,3,2,3,2,3,2,1,2,2,0,1,1,2,1,2,1,2,0,
|
||||
0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
3,3,3,3,3,3,3,3,3,3,3,3,2,2,3,2,3,3,3,2,2,2,2,0,2,2,2,2,3,1,1,0,
|
||||
0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,
|
||||
3,2,3,2,2,3,3,3,3,3,3,3,3,3,1,3,2,0,0,3,3,3,3,2,3,3,3,3,2,3,2,0,
|
||||
0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
2,3,3,3,3,3,2,2,3,3,0,2,1,0,3,2,3,2,3,0,0,1,2,0,0,1,0,1,2,1,1,0,
|
||||
0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
3,0,3,0,2,3,3,3,3,2,3,3,3,3,1,2,2,0,0,2,3,2,2,2,3,2,3,2,2,3,0,0,
|
||||
0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
3,2,3,0,2,3,2,3,0,1,2,3,3,2,0,2,3,0,0,2,3,2,2,0,1,3,1,3,2,2,1,0,
|
||||
0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
3,1,3,0,2,3,3,3,3,3,3,3,3,2,1,3,2,0,0,2,2,3,3,3,2,3,3,0,2,2,0,0,
|
||||
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
3,3,3,3,3,3,2,2,3,3,2,2,2,3,3,0,0,1,1,1,1,1,2,0,0,1,1,1,1,0,1,0,
|
||||
0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
3,3,3,3,3,3,2,2,3,3,3,3,3,3,3,0,3,2,3,3,2,3,2,0,2,1,0,1,1,0,1,0,
|
||||
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,
|
||||
3,3,3,3,3,3,2,3,3,3,2,2,2,2,3,1,3,2,3,1,1,2,1,0,2,2,2,2,1,3,1,0,
|
||||
0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,
|
||||
2,2,3,3,3,3,3,1,2,2,1,3,1,0,3,0,0,3,0,0,0,1,1,0,1,2,1,0,0,0,0,0,
|
||||
0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
3,2,2,1,1,3,3,3,2,2,1,2,2,3,1,1,2,0,0,2,2,1,3,0,0,2,1,1,2,1,1,0,
|
||||
0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
3,2,3,3,3,3,1,2,2,2,1,2,1,3,3,1,1,2,1,2,1,2,2,0,2,0,0,1,1,0,1,0,
|
||||
0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
2,3,3,3,3,3,2,1,3,2,2,3,2,0,3,2,0,3,0,1,0,1,1,0,0,1,1,1,1,0,1,0,
|
||||
0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
3,3,2,3,3,3,2,2,2,3,3,1,2,1,2,1,0,1,0,1,1,0,1,0,0,2,1,1,1,0,1,0,
|
||||
0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,
|
||||
3,1,1,2,1,2,3,3,2,2,1,2,2,3,0,2,1,0,0,2,2,3,2,1,2,2,2,2,2,3,1,0,
|
||||
0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
3,3,3,3,3,1,1,0,1,1,2,2,1,1,3,0,0,1,3,1,1,1,0,0,0,1,0,1,1,0,0,0,
|
||||
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
2,1,3,3,3,2,0,0,0,2,1,0,1,0,2,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
2,0,1,0,0,2,3,2,2,2,1,2,2,2,1,2,1,0,0,1,1,1,0,2,0,1,1,1,0,0,1,1,
|
||||
1,0,0,0,0,0,1,2,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,
|
||||
2,3,3,3,3,0,0,0,0,1,0,0,0,0,3,0,1,2,1,0,0,0,0,0,0,0,1,1,0,0,1,1,
|
||||
1,0,1,0,1,2,0,0,1,1,2,1,0,1,1,1,1,0,1,1,1,1,0,1,0,0,1,0,0,1,1,0,
|
||||
2,2,3,2,2,2,3,1,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,0,1,0,1,1,1,0,2,1,
|
||||
1,1,1,1,1,1,1,1,2,1,1,1,1,1,1,1,1,1,1,0,1,0,1,1,0,1,1,1,0,1,1,0,
|
||||
3,3,3,2,2,2,2,3,2,2,1,1,2,2,2,2,1,1,3,1,2,1,2,0,0,1,1,0,1,0,2,1,
|
||||
1,1,1,1,1,2,1,0,1,1,1,1,0,1,0,0,1,1,0,0,1,0,1,0,0,1,0,0,0,1,1,0,
|
||||
2,0,0,1,0,3,2,2,2,2,1,2,1,2,1,2,0,0,0,2,1,2,2,1,1,2,2,0,1,1,0,2,
|
||||
1,1,1,1,1,0,1,1,1,2,1,1,1,2,1,0,1,2,1,1,1,1,0,1,1,1,0,0,1,0,0,1,
|
||||
1,3,2,2,2,1,1,1,2,3,0,0,0,0,2,0,2,2,1,0,0,0,0,0,0,1,0,0,0,0,1,1,
|
||||
1,0,1,1,0,1,0,1,1,0,1,1,0,2,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,1,1,0,
|
||||
2,3,2,3,2,1,2,2,2,2,1,0,0,0,2,0,0,1,1,0,0,0,0,0,0,0,1,1,0,0,2,1,
|
||||
1,1,2,1,0,2,0,0,1,0,1,0,0,1,0,0,1,1,0,1,1,0,0,0,0,0,1,0,0,0,0,0,
|
||||
3,0,0,1,0,2,2,2,3,2,2,2,2,2,2,2,0,0,0,2,1,2,1,1,1,2,2,0,0,0,1,2,
|
||||
1,1,1,1,1,0,1,2,1,1,1,1,1,1,1,0,1,1,1,1,1,1,0,1,1,1,1,1,1,0,0,1,
|
||||
2,3,2,3,3,2,0,1,1,1,0,0,1,0,2,0,1,1,3,1,0,0,0,0,0,0,0,1,0,0,2,1,
|
||||
1,1,1,1,1,1,1,0,1,0,1,1,1,1,0,1,1,1,0,0,1,1,0,1,0,0,0,0,0,0,1,0,
|
||||
2,3,3,3,3,1,2,2,2,2,0,1,1,0,2,1,1,1,2,1,0,1,1,0,0,1,0,1,0,0,2,0,
|
||||
0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
2,3,3,3,2,0,0,1,1,2,2,1,0,0,2,0,1,1,3,0,0,1,0,0,0,0,0,1,0,1,2,1,
|
||||
1,1,2,0,1,1,1,0,1,0,1,1,0,1,0,1,1,1,1,0,1,0,0,0,0,0,0,1,0,1,1,0,
|
||||
1,3,2,3,2,1,0,0,2,2,2,0,1,0,2,0,1,1,1,0,1,0,0,0,3,0,1,1,0,0,2,1,
|
||||
1,1,1,0,1,1,0,0,0,0,1,1,0,1,0,0,2,1,1,0,1,0,0,0,1,0,1,0,0,1,1,0,
|
||||
3,1,2,1,1,2,2,2,2,2,2,1,2,2,1,1,0,0,0,2,2,2,0,0,0,1,2,1,0,1,0,1,
|
||||
2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,2,1,1,1,0,1,0,1,1,0,1,1,1,0,0,1,
|
||||
3,0,0,0,0,2,0,1,1,1,1,1,1,1,0,1,0,0,0,1,1,1,0,1,0,1,1,0,0,1,0,1,
|
||||
1,1,0,0,1,0,0,0,1,0,1,1,0,0,1,0,1,0,1,0,0,0,0,1,0,0,0,1,0,0,0,1,
|
||||
1,3,3,2,2,0,0,0,2,2,0,0,0,1,2,0,1,1,2,0,0,0,0,0,0,0,0,1,0,0,2,1,
|
||||
0,1,1,0,0,1,1,0,0,0,1,1,0,1,1,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,
|
||||
2,3,2,3,2,0,0,0,0,1,1,0,0,0,2,0,2,0,2,0,0,0,0,0,1,0,0,1,0,0,1,1,
|
||||
1,1,2,0,1,2,1,0,1,1,2,1,1,1,1,1,2,1,1,0,1,0,0,1,1,1,1,1,0,1,1,0,
|
||||
1,3,2,2,2,1,0,0,2,2,1,0,1,2,2,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,1,1,
|
||||
0,0,1,1,0,1,1,0,0,1,1,0,1,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,
|
||||
1,0,0,1,0,2,3,1,2,2,2,2,2,2,1,1,0,0,0,1,0,1,0,2,1,1,1,0,0,0,0,1,
|
||||
1,1,0,1,1,0,1,1,1,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,
|
||||
2,0,2,0,0,1,0,3,2,1,2,1,2,2,0,1,0,0,0,2,1,0,0,2,1,1,1,1,0,2,0,2,
|
||||
2,1,1,1,1,1,1,1,1,1,1,1,1,2,1,0,1,1,1,1,0,0,0,1,1,1,1,0,1,0,0,1,
|
||||
1,2,2,2,2,1,0,0,1,0,0,0,0,0,2,0,1,1,1,1,0,0,0,0,1,0,1,2,0,0,2,0,
|
||||
1,0,1,1,1,2,1,0,1,0,1,1,0,0,1,0,1,1,1,0,1,0,0,0,1,0,0,1,0,1,1,0,
|
||||
2,1,2,2,2,0,3,0,1,1,0,0,0,0,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,
|
||||
0,0,0,1,1,1,0,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,
|
||||
1,2,2,3,2,2,0,0,1,1,2,0,1,2,1,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,
|
||||
0,1,1,0,0,1,1,0,0,1,1,0,0,1,1,0,1,1,0,0,1,0,0,0,0,0,0,0,0,1,1,0,
|
||||
2,2,1,1,2,1,2,2,2,2,2,1,2,2,0,1,0,0,0,1,2,2,2,1,2,1,1,1,1,1,2,1,
|
||||
1,1,1,1,1,1,1,1,1,1,0,0,1,1,1,0,1,1,1,0,0,0,0,1,1,1,0,1,1,0,0,1,
|
||||
1,2,2,2,2,0,1,0,2,2,0,0,0,0,2,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,2,0,
|
||||
0,0,1,0,0,1,0,0,0,0,1,0,1,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,
|
||||
0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,
|
||||
0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
1,2,2,2,2,0,0,0,2,2,2,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,
|
||||
0,1,1,0,0,1,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
1,2,2,2,2,0,0,0,0,1,0,0,1,1,2,0,0,0,0,1,0,1,0,0,1,0,0,2,0,0,0,1,
|
||||
0,0,1,0,0,1,0,0,0,1,1,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,
|
||||
1,2,2,2,1,1,2,0,2,1,1,1,1,0,2,2,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,1,
|
||||
0,0,1,0,1,1,0,0,0,0,1,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,
|
||||
1,0,2,1,2,0,0,0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,
|
||||
0,0,1,0,1,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,
|
||||
1,0,0,0,0,2,0,1,2,1,0,1,1,1,0,1,0,0,0,1,0,1,0,0,1,0,1,0,0,0,0,1,
|
||||
0,0,0,0,0,1,0,0,1,1,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,
|
||||
2,2,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,
|
||||
1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,
|
||||
2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,
|
||||
1,1,1,0,1,0,1,0,0,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,
|
||||
1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,
|
||||
1,1,0,1,1,0,1,0,1,0,0,0,0,1,1,0,1,1,0,0,0,0,0,1,0,1,1,0,1,0,0,0,
|
||||
0,1,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
|
||||
0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,
|
||||
)
|
||||
|
||||
Koi8rModel = {
|
||||
'char_to_order_map': KOI8R_char_to_order_map,
|
||||
'precedence_matrix': RussianLangModel,
|
||||
'typical_positive_ratio': 0.976601,
|
||||
'keep_english_letter': False,
|
||||
'charset_name': "KOI8-R",
|
||||
'language': 'Russian',
|
||||
}
|
||||
|
||||
Win1251CyrillicModel = {
|
||||
'char_to_order_map': win1251_char_to_order_map,
|
||||
'precedence_matrix': RussianLangModel,
|
||||
'typical_positive_ratio': 0.976601,
|
||||
'keep_english_letter': False,
|
||||
'charset_name': "windows-1251",
|
||||
'language': 'Russian',
|
||||
}
|
||||
|
||||
Latin5CyrillicModel = {
|
||||
'char_to_order_map': latin5_char_to_order_map,
|
||||
'precedence_matrix': RussianLangModel,
|
||||
'typical_positive_ratio': 0.976601,
|
||||
'keep_english_letter': False,
|
||||
'charset_name': "ISO-8859-5",
|
||||
'language': 'Russian',
|
||||
}
|
||||
|
||||
MacCyrillicModel = {
|
||||
'char_to_order_map': macCyrillic_char_to_order_map,
|
||||
'precedence_matrix': RussianLangModel,
|
||||
'typical_positive_ratio': 0.976601,
|
||||
'keep_english_letter': False,
|
||||
'charset_name': "MacCyrillic",
|
||||
'language': 'Russian',
|
||||
}
|
||||
|
||||
Ibm866Model = {
|
||||
'char_to_order_map': IBM866_char_to_order_map,
|
||||
'precedence_matrix': RussianLangModel,
|
||||
'typical_positive_ratio': 0.976601,
|
||||
'keep_english_letter': False,
|
||||
'charset_name': "IBM866",
|
||||
'language': 'Russian',
|
||||
}
|
||||
|
||||
Ibm855Model = {
|
||||
'char_to_order_map': IBM855_char_to_order_map,
|
||||
'precedence_matrix': RussianLangModel,
|
||||
'typical_positive_ratio': 0.976601,
|
||||
'keep_english_letter': False,
|
||||
'charset_name': "IBM855",
|
||||
'language': 'Russian',
|
||||
}
|
||||
4607
thirdparty/chardet/langgreekmodel.py
vendored
4607
thirdparty/chardet/langgreekmodel.py
vendored
File diff suppressed because it is too large
Load diff
4571
thirdparty/chardet/langhebrewmodel.py
vendored
4571
thirdparty/chardet/langhebrewmodel.py
vendored
File diff suppressed because it is too large
Load diff
4859
thirdparty/chardet/langhungarianmodel.py
vendored
4859
thirdparty/chardet/langhungarianmodel.py
vendored
File diff suppressed because it is too large
Load diff
4570
thirdparty/chardet/langthaimodel.py
vendored
4570
thirdparty/chardet/langthaimodel.py
vendored
File diff suppressed because it is too large
Load diff
4562
thirdparty/chardet/langturkishmodel.py
vendored
4562
thirdparty/chardet/langturkishmodel.py
vendored
File diff suppressed because it is too large
Load diff
45
thirdparty/chardet/sbcharsetprober.py
vendored
45
thirdparty/chardet/sbcharsetprober.py
vendored
|
|
@ -26,10 +26,22 @@
|
|||
# 02110-1301 USA
|
||||
######################### END LICENSE BLOCK #########################
|
||||
|
||||
from collections import namedtuple
|
||||
|
||||
from .charsetprober import CharSetProber
|
||||
from .enums import CharacterCategory, ProbingState, SequenceLikelihood
|
||||
|
||||
|
||||
SingleByteCharSetModel = namedtuple('SingleByteCharSetModel',
|
||||
['charset_name',
|
||||
'language',
|
||||
'char_to_order_map',
|
||||
'language_model',
|
||||
'typical_positive_ratio',
|
||||
'keep_ascii_letters',
|
||||
'alphabet'])
|
||||
|
||||
|
||||
class SingleByteCharSetProber(CharSetProber):
|
||||
SAMPLE_SIZE = 64
|
||||
SB_ENOUGH_REL_THRESHOLD = 1024 # 0.25 * SAMPLE_SIZE^2
|
||||
|
|
@ -65,25 +77,25 @@ class SingleByteCharSetProber(CharSetProber):
|
|||
if self._name_prober:
|
||||
return self._name_prober.charset_name
|
||||
else:
|
||||
return self._model['charset_name']
|
||||
return self._model.charset_name
|
||||
|
||||
@property
|
||||
def language(self):
|
||||
if self._name_prober:
|
||||
return self._name_prober.language
|
||||
else:
|
||||
return self._model.get('language')
|
||||
return self._model.language
|
||||
|
||||
def feed(self, byte_str):
|
||||
if not self._model['keep_english_letter']:
|
||||
# TODO: Make filter_international_words keep things in self.alphabet
|
||||
if not self._model.keep_ascii_letters:
|
||||
byte_str = self.filter_international_words(byte_str)
|
||||
if not byte_str:
|
||||
return self.state
|
||||
char_to_order_map = self._model['char_to_order_map']
|
||||
for i, c in enumerate(byte_str):
|
||||
# XXX: Order is in range 1-64, so one would think we want 0-63 here,
|
||||
# but that leads to 27 more test failures than before.
|
||||
order = char_to_order_map[c]
|
||||
char_to_order_map = self._model.char_to_order_map
|
||||
language_model = self._model.language_model
|
||||
for char in byte_str:
|
||||
order = char_to_order_map.get(char, CharacterCategory.UNDEFINED)
|
||||
# XXX: This was SYMBOL_CAT_ORDER before, with a value of 250, but
|
||||
# CharacterCategory.SYMBOL is actually 253, so we use CONTROL
|
||||
# to make it closer to the original intent. The only difference
|
||||
|
|
@ -91,20 +103,21 @@ class SingleByteCharSetProber(CharSetProber):
|
|||
# _total_char purposes.
|
||||
if order < CharacterCategory.CONTROL:
|
||||
self._total_char += 1
|
||||
# TODO: Follow uchardet's lead and discount confidence for frequent
|
||||
# control characters.
|
||||
# See https://github.com/BYVoid/uchardet/commit/55b4f23971db61
|
||||
if order < self.SAMPLE_SIZE:
|
||||
self._freq_char += 1
|
||||
if self._last_order < self.SAMPLE_SIZE:
|
||||
self._total_seqs += 1
|
||||
if not self._reversed:
|
||||
i = (self._last_order * self.SAMPLE_SIZE) + order
|
||||
model = self._model['precedence_matrix'][i]
|
||||
else: # reverse the order of the letters in the lookup
|
||||
i = (order * self.SAMPLE_SIZE) + self._last_order
|
||||
model = self._model['precedence_matrix'][i]
|
||||
self._seq_counters[model] += 1
|
||||
lm_cat = language_model[self._last_order][order]
|
||||
else:
|
||||
lm_cat = language_model[order][self._last_order]
|
||||
self._seq_counters[lm_cat] += 1
|
||||
self._last_order = order
|
||||
|
||||
charset_name = self._model['charset_name']
|
||||
charset_name = self._model.charset_name
|
||||
if self.state == ProbingState.DETECTING:
|
||||
if self._total_seqs > self.SB_ENOUGH_REL_THRESHOLD:
|
||||
confidence = self.get_confidence()
|
||||
|
|
@ -125,7 +138,7 @@ class SingleByteCharSetProber(CharSetProber):
|
|||
r = 0.01
|
||||
if self._total_seqs > 0:
|
||||
r = ((1.0 * self._seq_counters[SequenceLikelihood.POSITIVE]) /
|
||||
self._total_seqs / self._model['typical_positive_ratio'])
|
||||
self._total_seqs / self._model.typical_positive_ratio)
|
||||
r = r * self._freq_char / self._total_char
|
||||
if r >= 1.0:
|
||||
r = 0.99
|
||||
|
|
|
|||
76
thirdparty/chardet/sbcsgroupprober.py
vendored
76
thirdparty/chardet/sbcsgroupprober.py
vendored
|
|
@ -27,47 +27,57 @@
|
|||
######################### END LICENSE BLOCK #########################
|
||||
|
||||
from .charsetgroupprober import CharSetGroupProber
|
||||
from .sbcharsetprober import SingleByteCharSetProber
|
||||
from .langcyrillicmodel import (Win1251CyrillicModel, Koi8rModel,
|
||||
Latin5CyrillicModel, MacCyrillicModel,
|
||||
Ibm866Model, Ibm855Model)
|
||||
from .langgreekmodel import Latin7GreekModel, Win1253GreekModel
|
||||
from .langbulgarianmodel import Latin5BulgarianModel, Win1251BulgarianModel
|
||||
# from .langhungarianmodel import Latin2HungarianModel, Win1250HungarianModel
|
||||
from .langthaimodel import TIS620ThaiModel
|
||||
from .langhebrewmodel import Win1255HebrewModel
|
||||
from .hebrewprober import HebrewProber
|
||||
from .langturkishmodel import Latin5TurkishModel
|
||||
from .langbulgarianmodel import (ISO_8859_5_BULGARIAN_MODEL,
|
||||
WINDOWS_1251_BULGARIAN_MODEL)
|
||||
from .langgreekmodel import ISO_8859_7_GREEK_MODEL, WINDOWS_1253_GREEK_MODEL
|
||||
from .langhebrewmodel import WINDOWS_1255_HEBREW_MODEL
|
||||
# from .langhungarianmodel import (ISO_8859_2_HUNGARIAN_MODEL,
|
||||
# WINDOWS_1250_HUNGARIAN_MODEL)
|
||||
from .langrussianmodel import (IBM855_RUSSIAN_MODEL, IBM866_RUSSIAN_MODEL,
|
||||
ISO_8859_5_RUSSIAN_MODEL, KOI8_R_RUSSIAN_MODEL,
|
||||
MACCYRILLIC_RUSSIAN_MODEL,
|
||||
WINDOWS_1251_RUSSIAN_MODEL)
|
||||
from .langthaimodel import TIS_620_THAI_MODEL
|
||||
from .langturkishmodel import ISO_8859_9_TURKISH_MODEL
|
||||
from .sbcharsetprober import SingleByteCharSetProber
|
||||
|
||||
|
||||
class SBCSGroupProber(CharSetGroupProber):
|
||||
def __init__(self):
|
||||
super(SBCSGroupProber, self).__init__()
|
||||
hebrew_prober = HebrewProber()
|
||||
logical_hebrew_prober = SingleByteCharSetProber(WINDOWS_1255_HEBREW_MODEL,
|
||||
False, hebrew_prober)
|
||||
# TODO: See if using ISO-8859-8 Hebrew model works better here, since
|
||||
# it's actually the visual one
|
||||
visual_hebrew_prober = SingleByteCharSetProber(WINDOWS_1255_HEBREW_MODEL,
|
||||
True, hebrew_prober)
|
||||
hebrew_prober.set_model_probers(logical_hebrew_prober,
|
||||
visual_hebrew_prober)
|
||||
# TODO: ORDER MATTERS HERE. I changed the order vs what was in master
|
||||
# and several tests failed that did not before. Some thought
|
||||
# should be put into the ordering, and we should consider making
|
||||
# order not matter here, because that is very counter-intuitive.
|
||||
self.probers = [
|
||||
SingleByteCharSetProber(Win1251CyrillicModel),
|
||||
SingleByteCharSetProber(Koi8rModel),
|
||||
SingleByteCharSetProber(Latin5CyrillicModel),
|
||||
SingleByteCharSetProber(MacCyrillicModel),
|
||||
SingleByteCharSetProber(Ibm866Model),
|
||||
SingleByteCharSetProber(Ibm855Model),
|
||||
SingleByteCharSetProber(Latin7GreekModel),
|
||||
SingleByteCharSetProber(Win1253GreekModel),
|
||||
SingleByteCharSetProber(Latin5BulgarianModel),
|
||||
SingleByteCharSetProber(Win1251BulgarianModel),
|
||||
SingleByteCharSetProber(WINDOWS_1251_RUSSIAN_MODEL),
|
||||
SingleByteCharSetProber(KOI8_R_RUSSIAN_MODEL),
|
||||
SingleByteCharSetProber(ISO_8859_5_RUSSIAN_MODEL),
|
||||
SingleByteCharSetProber(MACCYRILLIC_RUSSIAN_MODEL),
|
||||
SingleByteCharSetProber(IBM866_RUSSIAN_MODEL),
|
||||
SingleByteCharSetProber(IBM855_RUSSIAN_MODEL),
|
||||
SingleByteCharSetProber(ISO_8859_7_GREEK_MODEL),
|
||||
SingleByteCharSetProber(WINDOWS_1253_GREEK_MODEL),
|
||||
SingleByteCharSetProber(ISO_8859_5_BULGARIAN_MODEL),
|
||||
SingleByteCharSetProber(WINDOWS_1251_BULGARIAN_MODEL),
|
||||
# TODO: Restore Hungarian encodings (iso-8859-2 and windows-1250)
|
||||
# after we retrain model.
|
||||
# SingleByteCharSetProber(Latin2HungarianModel),
|
||||
# SingleByteCharSetProber(Win1250HungarianModel),
|
||||
SingleByteCharSetProber(TIS620ThaiModel),
|
||||
SingleByteCharSetProber(Latin5TurkishModel),
|
||||
# SingleByteCharSetProber(ISO_8859_2_HUNGARIAN_MODEL),
|
||||
# SingleByteCharSetProber(WINDOWS_1250_HUNGARIAN_MODEL),
|
||||
SingleByteCharSetProber(TIS_620_THAI_MODEL),
|
||||
SingleByteCharSetProber(ISO_8859_9_TURKISH_MODEL),
|
||||
hebrew_prober,
|
||||
logical_hebrew_prober,
|
||||
visual_hebrew_prober,
|
||||
]
|
||||
hebrew_prober = HebrewProber()
|
||||
logical_hebrew_prober = SingleByteCharSetProber(Win1255HebrewModel,
|
||||
False, hebrew_prober)
|
||||
visual_hebrew_prober = SingleByteCharSetProber(Win1255HebrewModel, True,
|
||||
hebrew_prober)
|
||||
hebrew_prober.set_model_probers(logical_hebrew_prober, visual_hebrew_prober)
|
||||
self.probers.extend([hebrew_prober, logical_hebrew_prober,
|
||||
visual_hebrew_prober])
|
||||
|
||||
self.reset()
|
||||
|
|
|
|||
8
thirdparty/chardet/universaldetector.py
vendored
8
thirdparty/chardet/universaldetector.py
vendored
|
|
@ -266,7 +266,7 @@ class UniversalDetector(object):
|
|||
'language': max_prober.language}
|
||||
|
||||
# Log all prober confidences if none met MINIMUM_THRESHOLD
|
||||
if self.logger.getEffectiveLevel() == logging.DEBUG:
|
||||
if self.logger.getEffectiveLevel() <= logging.DEBUG:
|
||||
if self.result['encoding'] is None:
|
||||
self.logger.debug('no probers hit minimum threshold')
|
||||
for group_prober in self._charset_probers:
|
||||
|
|
@ -280,7 +280,7 @@ class UniversalDetector(object):
|
|||
prober.get_confidence())
|
||||
else:
|
||||
self.logger.debug('%s %s confidence = %s',
|
||||
prober.charset_name,
|
||||
prober.language,
|
||||
prober.get_confidence())
|
||||
group_prober.charset_name,
|
||||
group_prober.language,
|
||||
group_prober.get_confidence())
|
||||
return self.result
|
||||
|
|
|
|||
2
thirdparty/chardet/version.py
vendored
2
thirdparty/chardet/version.py
vendored
|
|
@ -5,5 +5,5 @@ from within setup.py and from chardet subpackages.
|
|||
:author: Dan Blanchard (dan.blanchard@gmail.com)
|
||||
"""
|
||||
|
||||
__version__ = "3.0.4"
|
||||
__version__ = "4.0.0"
|
||||
VERSION = __version__.split('.')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue