Fix issue #156.

Fix some Unicode mix-up in the serializer, too, making sure Unicode strings are unicode strings.

Author: gsnedders

html5lib/filters/inject_meta_charset.py

@@ -13,44 +13,44 @@ def __iter__(self):
         for token in _base.Filter.__iter__(self):
             type = token["type"]
             if type == "StartTag":
-                if token["name"].lower() == "head":
+                if token["name"].lower() == u"head":
                     state = "in_head"
 
             elif type == "EmptyTag":
-                if token["name"].lower() == "meta":
+                if token["name"].lower() == u"meta":
                    # replace charset with actual encoding
                    has_http_equiv_content_type = False
                    for (namespace,name),value in token["data"].iteritems():
                        if namespace != None:
                            continue
-                       elif name.lower() == 'charset':
+                       elif name.lower() == u'charset':
                           token["data"][(namespace,name)] = self.encoding
                           meta_found = True
                           break
-                       elif name == 'http-equiv' and value.lower() == 'content-type':
+                       elif name == u'http-equiv' and value.lower() == u'content-type':
                            has_http_equiv_content_type = True
                    else:
-                       if has_http_equiv_content_type and (None, "content") in token["data"]:
-                           token["data"][(None, "content")] = u'text/html; charset=%s' % self.encoding
+                       if has_http_equiv_content_type and (None, u"content") in token["data"]:
+                           token["data"][(None, u"content")] = u'text/html; charset=%s' % self.encoding
                            meta_found = True
 
-                elif token["name"].lower() == "head" and not meta_found:
+                elif token["name"].lower() == u"head" and not meta_found:
                     # insert meta into empty head
-                    yield {"type": "StartTag", "name": "head",
+                    yield {"type": "StartTag", "name": u"head",
                            "data": token["data"]}
-                    yield {"type": "EmptyTag", "name": "meta",
-                           "data": {(None, "charset"): self.encoding}}
-                    yield {"type": "EndTag", "name": "head"}
+                    yield {"type": "EmptyTag", "name": u"meta",
+                           "data": {(None, u"charset"): self.encoding}}
+                    yield {"type": "EndTag", "name": u"head"}
                     meta_found = True
                     continue
 
             elif type == "EndTag":
-                if token["name"].lower() == "head" and pending:
+                if token["name"].lower() == u"head" and pending:
                     # insert meta into head (if necessary) and flush pending queue
                     yield pending.pop(0)
                     if not meta_found:
-                        yield {"type": "EmptyTag", "name": "meta",
-                               "data": {(None, "charset"): self.encoding}}
+                        yield {"type": "EmptyTag", "name": u"meta",
+                               "data": {(None, u"charset"): self.encoding}}
                     while pending:
                         yield pending.pop(0)
                     meta_found = True

html5lib/serializer/htmlserializer.py

@@ -76,14 +76,12 @@ def htmlentityreplace_errors(exc):
 
     del register_error
 
-def encode(text, encoding):
-    return text.encode(encoding, unicode_encode_errors)
 
 class HTMLSerializer(object):
 
     # attribute quoting options
     quote_attr_values = False
-    quote_char = '"'
+    quote_char = u'"'
     use_best_quote_char = True
 
     # tag syntax options
@@ -159,7 +157,22 @@ def __init__(self, **kwargs):
         self.errors = []
         self.strict = False
 
+    def encode(self, string):
+        assert(isinstance(string, unicode))
+        if self.encoding:
+            return string.encode(self.encoding, unicode_encode_errors)
+        else:
+            return string
+
+    def encodeStrict(self, string):
+        assert(isinstance(string, unicode))
+        if self.encoding:
+            return string.encode(self.encoding, "strict")
+        else:
+            return string
+
     def serialize(self, treewalker, encoding=None):
+        self.encoding = encoding
         in_cdata = False
         self.errors = []
         if encoding and self.inject_meta_charset:
@@ -195,27 +208,19 @@ def serialize(self, treewalker, encoding=None):
                     doctype += u" %s%s%s" % (quote_char, token["systemId"], quote_char)
 
                 doctype += u">"
-                
-                if encoding:
-                    yield doctype.encode(encoding)
-                else:
-                    yield doctype
+                yield self.encodeStrict(doctype)
 
             elif type in ("Characters", "SpaceCharacters"):
                 if type == "SpaceCharacters" or in_cdata:
                     if in_cdata and token["data"].find("</") >= 0:
                         self.serializeError(_("Unexpected </ in CDATA"))
-                    if encoding:
-                        yield token["data"].encode(encoding, "strict")
-                    else:
-                        yield token["data"]
-                elif encoding:
-                    yield encode(escape(token["data"]), encoding)
+                    yield self.encode(token["data"])
                 else:
-                    yield escape(token["data"])
+                    yield self.encode(escape(token["data"]))
 
             elif type in ("StartTag", "EmptyTag"):
                 name = token["name"]
+                yield self.encodeStrict(u"<%s" % name)
                 if name in rcdataElements and not self.escape_rcdata:
                     in_cdata = True
                 elif in_cdata:
@@ -225,69 +230,56 @@ def serialize(self, treewalker, encoding=None):
                     #TODO: Add namespace support here
                     k = attr_name
                     v = attr_value
-                    if encoding:
-                        k = k.encode(encoding, "strict")
-                    attributes.append(' ')
+                    yield self.encodeStrict(u' ')
 
-                    attributes.append(k)
+                    yield self.encodeStrict(k)
                     if not self.minimize_boolean_attributes or \
                       (k not in booleanAttributes.get(name, tuple()) \
                       and k not in booleanAttributes.get("", tuple())):
-                        attributes.append("=")
+                        yield self.encodeStrict(u"=")
                         if self.quote_attr_values or not v:
                             quote_attr = True
                         else:
                             quote_attr = reduce(lambda x,y: x or (y in v),
-                                spaceCharacters + ">\"'=", False)
-                        v = v.replace("&", "&amp;")
-                        if self.escape_lt_in_attrs: v = v.replace("<", "&lt;")
-                        if encoding:
-                            v = encode(v, encoding)
+                                spaceCharacters + u">\"'=", False)
+                        v = v.replace(u"&", u"&amp;")
+                        if self.escape_lt_in_attrs: v = v.replace(u"<", u"&lt;")
                         if quote_attr:
                             quote_char = self.quote_char
                             if self.use_best_quote_char:
-                                if "'" in v and '"' not in v:
-                                    quote_char = '"'
-                                elif '"' in v and "'" not in v:
-                                    quote_char = "'"
-                            if quote_char == "'":
-                                v = v.replace("'", "&#39;")
+                                if u"'" in v and u'"' not in v:
+                                    quote_char = u'"'
+                                elif u'"' in v and u"'" not in v:
+                                    quote_char = u"'"
+                            if quote_char == u"'":
+                                v = v.replace(u"'", u"&#39;")
                             else:
-                                v = v.replace('"', "&quot;")
-                            attributes.append(quote_char)
-                            attributes.append(v)
-                            attributes.append(quote_char)
+                                v = v.replace(u'"', u"&quot;")
+                            yield self.encodeStrict(quote_char)
+                            yield self.encode(v)
+                            yield self.encodeStrict(quote_char)
                         else:
-                            attributes.append(v)
+                            yield self.encode(v)
                 if name in voidElements and self.use_trailing_solidus:
                     if self.space_before_trailing_solidus:
-                        attributes.append(" /")
+                        yield self.encodeStrict(u" /")
                     else:
-                        attributes.append("/")
-                if encoding:
-                    yield "<%s%s>" % (name.encode(encoding, "strict"), "".join(attributes))
-                else:
-                    yield u"<%s%s>" % (name, u"".join(attributes))
+                        yield self.encodeStrict(u"/")
+                yield self.encode(u">")
 
             elif type == "EndTag":
                 name = token["name"]
                 if name in rcdataElements:
                     in_cdata = False
                 elif in_cdata:
                     self.serializeError(_("Unexpected child element of a CDATA element"))
-                end_tag = u"</%s>" % name
-                if encoding:
-                    end_tag = end_tag.encode(encoding, "strict")
-                yield end_tag
+                yield self.encodeStrict(u"</%s>" % name)
 
             elif type == "Comment":
                 data = token["data"]
                 if data.find("--") >= 0:
                     self.serializeError(_("Comment contains --"))
-                comment = u"<!--%s-->" % token["data"]
-                if encoding:
-                    comment = comment.encode(encoding, unicode_encode_errors)
-                yield comment
+                yield self.encodeStrict(u"<!--%s-->" % token["data"])
 
             elif type == "Entity":
                 name = token["name"]
@@ -298,9 +290,7 @@ def serialize(self, treewalker, encoding=None):
                     data = entities[key]
                 else:
                     data = u"&%s;" % name
-                if encoding:
-                    data = data.encode(encoding, unicode_encode_errors)
-                yield data
+                yield self.encodeStrict(data)
 
             else:
                 self.serializeError(token["data"])