Add support for HTTP/1.1 bodies.

Author: Lukasa

hyper/http11/connection.py

@@ -5,18 +5,21 @@
 
 Objects that build hyper's connection-level HTTP/1.1 abstraction.
 """
-import io
 import logging
-import re
+import os
 import socket
 
 from .response import HTTP11Response
 from ..http20.bufsocket import BufferedSocket
 from ..common.headers import HTTPHeaderMap
 from ..common.util import to_bytestring
+from ..compat import bytes
 
 log = logging.getLogger(__name__)
 
+BODY_CHUNKED = 1
+BODY_FLAT = 2
+
 
 class HTTP11Connection(object):
     """
@@ -91,24 +94,24 @@ def request(self, method, url, body=None, headers={}):
         method = to_bytestring(method)
         url = to_bytestring(url)
 
+        if not isinstance(headers, HTTPHeaderMap):
+            # FIXME: Handle things that aren't dictionaries here.
+            headers = HTTPHeaderMap(headers.items())
+
         if self._sock is None:
             self.connect()
 
-        # In this initial implementation, let's just write straight to the
-        # socket. We'll fix this up as we go.
-        # TODO: Fix fix fix.
-        self._sock.send(b' '.join([method, url, b'HTTP/1.1\r\n']))
-
-        for name, value in headers.items():
-            name, value = to_bytestring(name), to_bytestring(value)
-            header = b''.join([name, b': ', value, b'\r\n'])
-            self._sock.send(header)
+        # We may need extra headers. For now, we only add headers based on
+        # body content.
+        if body:
+            body_type = self._add_body_headers(headers, body)
 
-        self._sock.send(b'\r\n')
+        # Begin by emitting the header block.
+        self._send_headers(method, url, headers)
 
+        # Next, send the request body.
         if body:
-            # TODO: Come back here to support non-string bodies.
-            self._sock.send(body)
+            self._send_body(body, body_type)
 
         return
 
@@ -136,3 +139,107 @@ def get_response(self):
             headers[name] = val
 
         return HTTP11Response(code, reason, headers, self._sock)
+
+    def _send_headers(self, method, url, headers):
+        """
+        Handles the logic of sending the header block.
+        """
+        self._sock.send(b' '.join([method, url, b'HTTP/1.1\r\n']))
+
+        for name, value in headers.iter_raw():
+            name, value = to_bytestring(name), to_bytestring(value)
+            header = b''.join([name, b': ', value, b'\r\n'])
+            self._sock.send(header)
+
+        self._sock.send(b'\r\n')
+
+    def _add_body_headers(self, headers, body):
+        """
+        Adds any headers needed for sending the request body. This will always
+        defer to the user-supplied header content.
+
+        :returns: One of (BODY_CHUNKED, BODY_FLAT), indicating what type of
+            request body should be used.
+        """
+        if b'content-length' in headers:
+            return BODY_FLAT
+
+        if b'chunked' in headers.get(b'transfer-encoding', []):
+            return BODY_CHUNKED
+
+        # For bytestring bodies we upload the content with a fixed length.
+        # For file objects, we use the length of the file object.
+        if isinstance(body, bytes):
+            length = str(len(body)).encode('utf-8')
+        elif hasattr(body, 'fileno'):
+            length = str(os.fstat(body.fileno()).st_size).encode('utf-8')
+        else:
+            length = None
+
+        if length:
+            headers[b'content-length'] = length
+            return BODY_FLAT
+
+        headers[b'transfer-encoding'] = b'chunked'
+        return BODY_CHUNKED
+
+    def _send_body(self, body, body_type):
+        """
+        Handles the HTTP/1.1 logic for sending HTTP bodies. This does magical
+        different things in different cases.
+        """
+        if body_type == BODY_FLAT:
+            # Special case for files and other 'readable' objects.
+            if hasattr(body, 'read'):
+                while True:
+                    block = body.read(16*1024)
+                    if not block:
+                        break
+
+                    try:
+                        self._sock.send(block)
+                    except TypeError:
+                        raise ValueError(
+                            "File objects must return bytestrings"
+                        )
+
+                return
+
+            # Case for bytestrings.
+            elif isinstance(body, bytes):
+                try:
+                    self._sock.send(body)
+                except TypeError:
+                    raise ValueError("Body must be a bytestring")
+
+                return
+
+            # Iterables that set a specific content length.
+            else:
+                for item in body:
+                    try:
+                        self._sock.send(item)
+                    except TypeError:
+                        raise ValueError("Body must be a bytestring")
+
+                return
+
+        # Chunked! For chunked bodies we don't special-case, we just iterate
+        # over what we have and send stuff out.
+        for chunk in body:
+            length = '{0:x}'.format(len(chunk)).encode('ascii')
+
+            # For now write this as four 'send' calls. That's probably
+            # inefficient, let's come back to it.
+            try:
+                self._sock.send(length)
+                self._sock.send(b'\r\n')
+                self._sock.send(chunk)
+                self._sock.send(b'\r\n')
+            except TypeError:
+                raise ValueError(
+                    "Iterable bodies must always iterate in bytestrings"
+                )
+
+        self._sock.send(b'0\r\n\r\n')
+        return

test/test_http11.py

@@ -5,10 +5,13 @@
 
 Unit tests for hyper's HTTP/1.1 implementation.
 """
+from collections import namedtuple
 from io import BytesIO
 
+import hyper
 from hyper.http11.connection import HTTP11Connection
 from hyper.http11.response import HTTP11Response
+from hyper.common.headers import HTTPHeaderMap
 
 
 class TestHTTP11Connection(object):
@@ -52,18 +55,126 @@ def test_request_with_bytestring_body(self):
         c = HTTP11Connection('http2bin.org')
         c._sock = sock = DummySocket()
 
-        c.request('POST', '/post', headers={'User-Agent': 'hyper'}, body=b'hi')
+        c.request(
+            'POST',
+            '/post',
+            headers=HTTPHeaderMap([('User-Agent', 'hyper')]),
+            body=b'hi'
+        )
 
         expected = (
             b"POST /post HTTP/1.1\r\n"
             b"User-Agent: hyper\r\n"
+            b"content-length: 2\r\n"
             b"\r\n"
             b"hi"
         )
         received = b''.join(sock.queue)
 
         assert received == expected
 
+    def test_request_with_file_body(self):
+        # Testing this is tricksy: in practice, we do this by passing a fake
+        # file and monkeypatching out 'os.fstat'. This makes it look like a
+        # real file.
+        FstatRval = namedtuple('FstatRval', ['st_size'])
+        def fake_fstat(*args):
+            return FstatRval(16)
+
+        old_fstat = hyper.http11.connection.os.fstat
+
+        try:
+            hyper.http11.connection.os.fstat = fake_fstat
+            c = HTTP11Connection('http2bin.org')
+            c._sock = sock = DummySocket()
+
+            f = DummyFile(b'some binary data')
+            c.request('POST', '/post',  body=f)
+
+            expected = (
+                b"POST /post HTTP/1.1\r\n"
+                b"content-length: 16\r\n"
+                b"\r\n"
+                b"some binary data"
+            )
+            received = b''.join(sock.queue)
+
+            assert received == expected
+
+        finally:
+            # Put back the monkeypatch.
+            hyper.http11.connection.os.fstat = old_fstat
+
+    def test_request_with_generator_body(self):
+        c = HTTP11Connection('http2bin.org')
+        c._sock = sock = DummySocket()
+        def body():
+            yield b'hi'
+            yield b'there'
+            yield b'sir'
+
+        c.request('POST', '/post', body=body())
+
+        expected = (
+            b"POST /post HTTP/1.1\r\n"
+            b"transfer-encoding: chunked\r\n"
+            b"\r\n"
+            b"2\r\nhi\r\n"
+            b"5\r\nthere\r\n"
+            b"3\r\nsir\r\n"
+            b"0\r\n\r\n"
+        )
+        received = b''.join(sock.queue)
+
+        assert received == expected
+
+    def test_content_length_overrides_generator(self):
+        c = HTTP11Connection('http2bin.org')
+        c._sock = sock = DummySocket()
+        def body():
+            yield b'hi'
+            yield b'there'
+            yield b'sir'
+
+        c.request(
+            'POST', '/post', headers={b'content-length': b'10'}, body=body()
+        )
+
+        expected = (
+            b"POST /post HTTP/1.1\r\n"
+            b"content-length: 10\r\n"
+            b"\r\n"
+            b"hitheresir"
+        )
+        received = b''.join(sock.queue)
+
+        assert received == expected
+
+    def test_chunked_overrides_body(self):
+        c = HTTP11Connection('http2bin.org')
+        c._sock = sock = DummySocket()
+
+        f = DummyFile(b'oneline\nanotherline')
+
+        c.request(
+            'POST',
+            '/post',
+            headers={'transfer-encoding': 'chunked'},
+            body=f
+        )
+
+        expected = (
+            b"POST /post HTTP/1.1\r\n"
+            b"transfer-encoding: chunked\r\n"
+            b"\r\n"
+            b"8\r\noneline\n\r\n"
+            b"b\r\nanotherline\r\n"
+            b"0\r\n\r\n"
+        )
+        received = b''.join(sock.queue)
+
+        assert received == expected
+
     def test_get_response(self):
         c = HTTP11Connection('http2bin.org')
         c._sock = sock = DummySocket()
@@ -132,3 +243,21 @@ def close(self):
 
     def readline(self):
         return memoryview(self.buffer.readline())
+
+
+class DummyFile(object):
+    def __init__(self, data):
+        self.buffer = BytesIO(data)
+
+    def read(self, amt=None):
+        return self.buffer.read(amt)
+
+    def fileno(self):
+        return -1
+
+    def readline(self):
+        self.buffer.readline()
+
+    def __iter__(self):
+        return self.buffer.__iter__()
+