git.reasoningtechnology.com Git - SubU/blob

   1 ######################## BEGIN LICENSE BLOCK ########################
   2 # The Original Code is Mozilla Communicator client code.
   3 #
   4 # The Initial Developer of the Original Code is
   5 # Netscape Communications Corporation.
   6 # Portions created by the Initial Developer are Copyright (C) 1998
   7 # the Initial Developer. All Rights Reserved.
   8 #
   9 # Contributor(s):
  10 #   Mark Pilgrim - port to Python
  11 #
  12 # This library is free software; you can redistribute it and/or
  13 # modify it under the terms of the GNU Lesser General Public
  14 # License as published by the Free Software Foundation; either
  15 # version 2.1 of the License, or (at your option) any later version.
  16 #
  17 # This library is distributed in the hope that it will be useful,
  18 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  20 # Lesser General Public License for more details.
  21 #
  22 # You should have received a copy of the GNU Lesser General Public
  23 # License along with this library; if not, write to the Free Software
  24 # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
  25 # 02110-1301  USA
  26 ######################### END LICENSE BLOCK #########################
  27
  28 from typing import List, Optional, Union
  29
  30 from .charsetprober import CharSetProber
  31 from .enums import LanguageFilter, ProbingState
  32
  33
  34 class CharSetGroupProber(CharSetProber):
  35     def __init__(self, lang_filter: LanguageFilter = LanguageFilter.NONE) -> None:
  36         super().__init__(lang_filter=lang_filter)
  37         self._active_num = 0
  38         self.probers: List[CharSetProber] = []
  39         self._best_guess_prober: Optional[CharSetProber] = None
  40
  41     def reset(self) -> None:
  42         super().reset()
  43         self._active_num = 0
  44         for prober in self.probers:
  45             prober.reset()
  46             prober.active = True
  47             self._active_num += 1
  48         self._best_guess_prober = None
  49
  50     @property
  51     def charset_name(self) -> Optional[str]:
  52         if not self._best_guess_prober:
  53             self.get_confidence()
  54             if not self._best_guess_prober:
  55                 return None
  56         return self._best_guess_prober.charset_name
  57
  58     @property
  59     def language(self) -> Optional[str]:
  60         if not self._best_guess_prober:
  61             self.get_confidence()
  62             if not self._best_guess_prober:
  63                 return None
  64         return self._best_guess_prober.language
  65
  66     def feed(self, byte_str: Union[bytes, bytearray]) -> ProbingState:
  67         for prober in self.probers:
  68             if not prober.active:
  69                 continue
  70             state = prober.feed(byte_str)
  71             if not state:
  72                 continue
  73             if state == ProbingState.FOUND_IT:
  74                 self._best_guess_prober = prober
  75                 self._state = ProbingState.FOUND_IT
  76                 return self.state
  77             if state == ProbingState.NOT_ME:
  78                 prober.active = False
  79                 self._active_num -= 1
  80                 if self._active_num <= 0:
  81                     self._state = ProbingState.NOT_ME
  82                     return self.state
  83         return self.state
  84
  85     def get_confidence(self) -> float:
  86         state = self.state
  87         if state == ProbingState.FOUND_IT:
  88             return 0.99
  89         if state == ProbingState.NOT_ME:
  90             return 0.01
  91         best_conf = 0.0
  92         self._best_guess_prober = None
  93         for prober in self.probers:
  94             if not prober.active:
  95                 self.logger.debug("%s not active", prober.charset_name)
  96                 continue
  97             conf = prober.get_confidence()
  98             self.logger.debug(
  99                 "%s %s confidence = %s", prober.charset_name, prober.language, conf
 100             )
 101             if best_conf < conf:
 102                 best_conf = conf
 103                 self._best_guess_prober = prober
 104         if not self._best_guess_prober:
 105             return 0.0
 106         return best_conf