Merge remote-tracking branch 'origin/master' into escape-characters-serializer

Author: gsnedders

CHANGES.rst

@@ -1,6 +1,31 @@
 Change Log
 ----------
 
+0.99
+~~~~
+
+Released on September 10, 2013
+
+* No library changes from 1.0b3; released as 0.99 as pip has changed
+  behaviour from 1.4 to avoid installing pre-release versions per
+  PEP 440.
+
+
+1.0b3
+~~~~~
+
+Released on July 24, 2013
+
+* Removed ``RecursiveTreeWalker`` from ``treewalkers._base``. Any
+  implementation using it should be moved to
+  ``NonRecursiveTreeWalker``, as everything bundled with html5lib has
+  for years.
+
+* Fix #67 so that ``BufferedStream`` to correctly returns a bytes
+  object, thereby fixing any case where html5lib is passed a
+  non-seekable RawIOBase-like object.
+
+
 1.0b2
 ~~~~~
 

README.rst

@@ -41,6 +41,29 @@ a treebuilder:
   with open("mydocument.html", "rb") as f:
       lxml_etree_document = html5lib.parse(f, treebuilder="lxml")
 
+When using with ``urllib2`` (Python 2), the charset from HTTP should be
+pass into html5lib as follows:
+
+.. code-block:: python
+
+  from contextlib import closing
+  from urllib2 import urlopen
+  import html5lib
+
+  with closing(urlopen("http://example.com/")) as f:
+      document = html5lib.parse(f, encoding=f.info().getparam("charset"))
+
+When using with ``urllib.request`` (Python 3), the charset from HTTP
+should be pass into html5lib as follows:
+
+.. code-block:: python
+
+  from urllib.request import urlopen
+  import html5lib
+
+  with urlopen("http://example.com/") as f:
+      document = html5lib.parse(f, encoding=f.info().get_content_charset())
+
 To have more control over the parser, create a parser object explicitly.
 For instance, to make the parser raise exceptions on parse errors, use:
 

html5lib/__init__.py

@@ -20,4 +20,4 @@
 
 __all__ = ["HTMLParser", "parse", "parseFragment", "getTreeBuilder",
            "getTreeWalker", "serialize"]
-__version__ = "1.0b2"
+__version__ = "0.999-dev"

html5lib/tests/test_treewalkers.py

@@ -83,16 +83,6 @@ def PullDOMAdapter(node):
          "walker": treewalkers.getTreeWalker("lxml")}
 
 
-# Try whatever etree implementations are available from a list that are
-#"supposed" to work
-try:
-    import pxdom
-    treeTypes['pxdom'] = \
-        {"builder": treebuilders.getTreeBuilder("dom", pxdom),
-         "walker": treewalkers.getTreeWalker("dom")}
-except ImportError:
-    pass
-
 try:
     from genshi.core import QName, Attrs
     from genshi.core import START, END, TEXT, COMMENT, DOCTYPE

html5lib/treewalkers/_base.py

@@ -4,6 +4,16 @@
 import gettext
 _ = gettext.gettext
 
+from xml.dom import Node
+
+DOCUMENT = Node.DOCUMENT_NODE
+DOCTYPE = Node.DOCUMENT_TYPE_NODE
+TEXT = Node.TEXT_NODE
+ELEMENT = Node.ELEMENT_NODE
+COMMENT = Node.COMMENT_NODE
+ENTITY = Node.ENTITY_NODE
+UNKNOWN = "<#UNKNOWN#>"
+
 from ..constants import voidElements, spaceCharacters
 spaceCharacters = "".join(spaceCharacters)
 
@@ -115,32 +125,6 @@ def unknown(self, nodeType):
         return self.error(_("Unknown node type: ") + nodeType)
 
 
-class RecursiveTreeWalker(TreeWalker):
-    def walkChildren(self, node):
-        raise NotImplementedError
-
-    def element(self, node, namespace, name, attrs, hasChildren):
-        if name in voidElements:
-            for token in self.emptyTag(namespace, name, attrs, hasChildren):
-                yield token
-        else:
-            yield self.startTag(name, attrs)
-            if hasChildren:
-                for token in self.walkChildren(node):
-                    yield token
-            yield self.endTag(name)
-
-from xml.dom import Node
-
-DOCUMENT = Node.DOCUMENT_NODE
-DOCTYPE = Node.DOCUMENT_TYPE_NODE
-TEXT = Node.TEXT_NODE
-ELEMENT = Node.ELEMENT_NODE
-COMMENT = Node.COMMENT_NODE
-ENTITY = Node.ENTITY_NODE
-UNKNOWN = "<#UNKNOWN#>"
-
-
 class NonRecursiveTreeWalker(TreeWalker):
     def getNodeDetails(self, node):
         raise NotImplementedError

setup.py

@@ -29,7 +29,7 @@
         long_description = readme_file.read() + '\n' + changes_file.read()
 
 setup(name='html5lib',
-      version='1.0b2',
+      version='0.999-dev',
       url='https://github.com/html5lib/html5lib-python',
       license="MIT License",
       description='HTML parser based on the WHATWG HTML specifcation',