diff options
author | 2017-03-29 13:35:45 +0300 | |
---|---|---|
committer | 2017-03-29 13:35:45 +0300 | |
commit | e2cc2530fc6d54ebc975c01a4ff887ce12f0a736 (patch) | |
tree | 38385867295c8a09fb0d7f8eaf5fa78179e5b13a /jython-tosca-parser/src/main/resources/Lib/site-packages/pip/_vendor/requests/packages/chardet/utf8prober.py | |
parent | bccebaa9888906f8ff78172f62ec592956066d82 (diff) |
[SDC-6] sdc-distribution-client 1707 rebasing
Change-Id: I322a05fd79beb6ba4fee4d32afffecf531b86e98
Signed-off-by: Pavel Aharoni <pa0916@att.com>
Diffstat (limited to 'jython-tosca-parser/src/main/resources/Lib/site-packages/pip/_vendor/requests/packages/chardet/utf8prober.py')
-rw-r--r-- | jython-tosca-parser/src/main/resources/Lib/site-packages/pip/_vendor/requests/packages/chardet/utf8prober.py | 76 |
1 files changed, 76 insertions, 0 deletions
diff --git a/jython-tosca-parser/src/main/resources/Lib/site-packages/pip/_vendor/requests/packages/chardet/utf8prober.py b/jython-tosca-parser/src/main/resources/Lib/site-packages/pip/_vendor/requests/packages/chardet/utf8prober.py new file mode 100644 index 0000000..1c0bb5d --- /dev/null +++ b/jython-tosca-parser/src/main/resources/Lib/site-packages/pip/_vendor/requests/packages/chardet/utf8prober.py @@ -0,0 +1,76 @@ +######################## BEGIN LICENSE BLOCK ######################## +# The Original Code is mozilla.org code. +# +# The Initial Developer of the Original Code is +# Netscape Communications Corporation. +# Portions created by the Initial Developer are Copyright (C) 1998 +# the Initial Developer. All Rights Reserved. +# +# Contributor(s): +# Mark Pilgrim - port to Python +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA +# 02110-1301 USA +######################### END LICENSE BLOCK ######################### + +from . import constants +from .charsetprober import CharSetProber +from .codingstatemachine import CodingStateMachine +from .mbcssm import UTF8SMModel + +ONE_CHAR_PROB = 0.5 + + +class UTF8Prober(CharSetProber): + def __init__(self): + CharSetProber.__init__(self) + self._mCodingSM = CodingStateMachine(UTF8SMModel) + self.reset() + + def reset(self): + CharSetProber.reset(self) + self._mCodingSM.reset() + self._mNumOfMBChar = 0 + + def get_charset_name(self): + return "utf-8" + + def feed(self, aBuf): + for c in aBuf: + codingState = self._mCodingSM.next_state(c) + if codingState == constants.eError: + self._mState = constants.eNotMe + break + elif codingState == constants.eItsMe: + self._mState = constants.eFoundIt + break + elif codingState == constants.eStart: + if self._mCodingSM.get_current_charlen() >= 2: + self._mNumOfMBChar += 1 + + if self.get_state() == constants.eDetecting: + if self.get_confidence() > constants.SHORTCUT_THRESHOLD: + self._mState = constants.eFoundIt + + return self.get_state() + + def get_confidence(self): + unlike = 0.99 + if self._mNumOfMBChar < 6: + for i in range(0, self._mNumOfMBChar): + unlike = unlike * ONE_CHAR_PROB + return 1.0 - unlike + else: + return unlike |