Re-run 3to2, and fix where something had gone uncommitted in Python3 before.

Author: gsnedders

html5lib/tokenizer.py

@@ -19,10 +19,9 @@
 
 from .inputstream import HTMLInputStream
 
-# Group entities by their first character, for faster lookups
-entitiesByFirstChar = {}
-for e in entities:
-    entitiesByFirstChar.setdefault(e[0], []).append(e)
+from .trie import Trie
+
+entitiesTrie = Trie(entities)
 
 class HTMLTokenizer(object):
     u""" This class takes care of tokenizing HTML.
@@ -183,29 +182,20 @@ def consumeEntity(self, allowedChar=None, fromAttribute=False):
             #
             # Consume characters and compare to these to a substring of the
             # entity names in the list until the substring no longer matches.
-            filteredEntityList = entitiesByFirstChar.get(charStack[0], [])
-
-            def entitiesStartingWith(name):
-                return [e for e in filteredEntityList if e.startswith(name)]
-            entitiesStartingWith.func_annotations = {}
-
             while (charStack[-1] is not EOF):
-                filteredEntityList = entitiesStartingWith(u"".join(charStack))
-                if not filteredEntityList:
+                if not entitiesTrie.has_keys_with_prefix(u"".join(charStack)):
                     break
                 charStack.append(self.stream.char())
 
             # At this point we have a string that starts with some characters
             # that may match an entity
-            entityName = None
-
             # Try to find the longest entity the string will match to take care
             # of &noti for instance.
-            for entityLength in xrange(len(charStack)-1, 1, -1):
-                possibleEntityName = u"".join(charStack[:entityLength])
-                if possibleEntityName in entities:
-                    entityName = possibleEntityName
-                    break
+            try:
+                entityName = entitiesTrie.longest_prefix(u"".join(charStack[:-1]))
+                entityLength = len(entityName)
+            except KeyError:
+                entityName = None
 
             if entityName is not None:
                 if entityName[-1] != u";":

html5lib/trie/__init__.py

@@ -0,0 +1,11 @@
+from __future__ import absolute_import
+from .py import Trie as PyTrie
+
+Trie = PyTrie
+
+try:
+    from .datrie import Trie as DATrie
+except ImportError:
+    pass
+else:
+    Trie = DATrie

html5lib/trie/_base.py

@@ -0,0 +1,38 @@
+from __future__ import absolute_import
+from collections import Mapping
+
+class Trie(Mapping):
+    u"""Abstract base class for tries"""
+
+    def keys(self, prefix=None):
+        keys = super(Trie, self).keys()
+
+        if prefix is None:
+            return set(keys)
+
+        return set(x for x in keys if x.startswith(prefix))
+    keys.func_annotations = {}
+
+    def has_keys_with_prefix(self, prefix):
+        for key in self.keys():
+            if key.startswith(prefix):
+                return True
+
+        return False
+    has_keys_with_prefix.func_annotations = {}
+
+    def longest_prefix(self, prefix):
+        if prefix in self:
+            return prefix
+
+        for i in xrange(1, len(prefix) + 1):
+            if prefix[:-i] in self:
+                return prefix[:-i]
+
+        raise KeyError(prefix)
+    longest_prefix.func_annotations = {}
+
+    def longest_prefix_item(self, prefix):
+        lprefix = self.longest_prefix(prefix)
+        return (lprefix, self[lprefix])
+    longest_prefix_item.func_annotations = {}

html5lib/trie/datrie.py

@@ -0,0 +1,52 @@
+from __future__ import absolute_import
+from itertools import chain
+
+from datrie import Trie as DATrie
+
+from ._base import Trie as ABCTrie
+
+class Trie(ABCTrie):
+    def __init__(self, data):
+        chars = set()
+        for key in data.keys():
+            if not isinstance(key, unicode):
+                raise TypeError(u"All keys must be strings")
+            for char in key:
+                chars.add(char)
+
+        self._data = DATrie(u"".join(chars))
+        for key, value in data.items():
+            self._data[key] = value
+    __init__.func_annotations = {}
+
+    def __contains__(self, key):
+        return key in self._data
+    __contains__.func_annotations = {}
+
+    def __len__(self):
+        return len(self._data)
+    __len__.func_annotations = {}
+
+    def __iter__(self):
+        raise NotImplementedError()
+    __iter__.func_annotations = {}
+
+    def __getitem__(self, key):
+        return self._data[key]
+    __getitem__.func_annotations = {}
+
+    def keys(self, prefix=None):
+        return self._data.keys(prefix)
+    keys.func_annotations = {}
+
+    def has_keys_with_prefix(self, prefix):
+        return self._data.has_keys_with_prefix(prefix)
+    has_keys_with_prefix.func_annotations = {}
+
+    def longest_prefix(self, prefix):
+        return self._data.longest_prefix(prefix)
+    longest_prefix.func_annotations = {}
+
+    def longest_prefix_item(self, prefix):
+        return self._data.longest_prefix_item(prefix)
+    longest_prefix_item.func_annotations = {}

html5lib/trie/py.py

@@ -0,0 +1,71 @@
+from __future__ import absolute_import
+from bisect import bisect_left
+
+from ._base import Trie as ABCTrie
+
+class Trie(ABCTrie):
+    def __init__(self, data):
+        if not all(isinstance(x, unicode) for x in data.keys()):
+            raise TypeError(u"All keys must be strings")
+
+        self._data = data
+        self._keys = sorted(data.keys())
+        self._cachestr = u""
+        self._cachepoints = (0, len(data))
+    __init__.func_annotations = {}
+
+    def __contains__(self, key):
+        return key in self._data
+    __contains__.func_annotations = {}
+
+    def __len__(self):
+        return len(self._data)
+    __len__.func_annotations = {}
+
+    def __iter__(self):
+        return iter(self._data)
+    __iter__.func_annotations = {}
+
+    def __getitem__(self, key):
+        return self._data[key]
+    __getitem__.func_annotations = {}
+
+    def keys(self, prefix=None):
+        if prefix is None or prefix == u"" or not self._keys:
+            return set(self._keys)
+        
+        if prefix.startswith(self._cachestr):
+            lo, hi = self._cachepoints
+            start = i = bisect_left(self._keys, prefix, lo, hi)
+        else:
+            start = i = bisect_left(self._keys, prefix)
+
+        keys = set()
+        if start == len(self._keys):
+            return keys
+
+        while self._keys[i].startswith(prefix):
+            keys.add(self._keys[i])
+            i += 1
+
+        self._cachestr = prefix
+        self._cachepoints = (start, i)
+
+        return keys
+    keys.func_annotations = {}
+
+    def has_keys_with_prefix(self, prefix):
+        if prefix in self._data:
+            return True
+
+        if prefix.startswith(self._cachestr):
+            lo, hi = self._cachepoints
+            i = bisect_left(self._keys, prefix, lo, hi)
+        else:
+            i = bisect_left(self._keys, prefix)
+
+        if i == len(self._keys):
+            return False
+        
+        return self._keys[i].startswith(prefix)
+    has_keys_with_prefix.func_annotations = {}