Rewrite HTTPClient chunked transfer-encoding (#3034)

* Rewrite HTTPClient chunked transfer-encoding

Fixes #3029

The chunked decoder seems to have had some race conditions resulting in
lost data.

The HTTPClient::getStreamPtr returned the raw WiFiClient which included
all of the chunked block size markers, requiring the user to manually
parse the chunked encoding.

Remove the existing chunked handling, add a HTTPStream as a WiFiClient
subclass.  The HTTPClient will return this HTTPStream which will always
properly cut out chunk markers when present (and pass things raw for
non-encoded streams).  Use this new HTTPStream class to replace the
existing handling in HTTPClient

Update the fingerprint in the StreamHTTPSClient example because their
cert was updated.

* Astyle

* Add block write passthru

* Add other WiFiClient passthru calls

Author: earlephilhower

libraries/HTTPClient/examples/StreamHttpsClient/StreamHttpsClient.ino

@@ -45,7 +45,7 @@ void loop() {
     Serial.print("[HTTPS] begin...\n");
 
     // configure server and url
-    const char *fp = "41:FA:FD:B6:96:5F:33:09:F4:ED:09:28:BF:66:4D:5B:A2:88:03:65";
+    const char *fp = "e5:e8:05:7d:85:70:b0:db:d9:5a:b6:a6:bb:2b:29:ae:d9:b1:b7:f9";
 
     HTTPClient https;
     https.setFingerprint(fp);

libraries/HTTPClient/src/HTTPClient.cpp

@@ -323,6 +323,7 @@ bool HTTPClient::beginInternal(const String& __url, const char* expectedProtocol
         return false;
     }
     DEBUG_HTTPCLIENT("[HTTP-Client][begin] host: %s port: %d url: %s\n", _host.c_str(), _port, _uri.c_str());
+
     return true;
 }
 
@@ -675,6 +676,7 @@ int HTTPClient::sendRequest(const char * type, const uint8_t * payload, size_t s
         }
     } while (redirect);
 
+
     // handle Server Response (Header)
     return returnError(code);
 }
@@ -744,7 +746,7 @@ const String& HTTPClient::getLocation(void) {
 */
 WiFiClient& HTTPClient::getStream(void) {
     if (connected()) {
-        return *_client();
+        return _stream;
     }
 
     DEBUG_HTTPCLIENT("[HTTP-Client] getStream: not connected\n");
@@ -758,7 +760,7 @@ WiFiClient& HTTPClient::getStream(void) {
 */
 WiFiClient* HTTPClient::getStreamPtr(void) {
     if (connected()) {
-        return _client();
+        return &_stream;
     }
 
     DEBUG_HTTPCLIENT("[HTTP-Client] getStreamPtr: not connected\n");
@@ -808,57 +810,10 @@ int HTTPClient::writeToPrint(Print * print) {
         //            return returnError(StreamReportToHttpClientReport(_client->getLastSendReport()));
         //        }
     } else if (_transferEncoding == HTTPC_TE_CHUNKED) {
-        int size = 0;
-        while (1) {
-            if (!connected()) {
-                return returnError(HTTPC_ERROR_CONNECTION_LOST);
-            }
-            String chunkHeader = _client()->readStringUntil('\n');
-
-            if (chunkHeader.length() <= 0) {
-                return returnError(HTTPC_ERROR_READ_TIMEOUT);
-            }
-
-            chunkHeader.trim(); // remove \r
-            DEBUG_HTTPCLIENT("[HTTP-Client] chunk header: '%s'\n", chunkHeader.c_str());
-
-            // read size of chunk
-            len = (uint32_t) strtol((const char *) chunkHeader.c_str(), nullptr, 16);
-            size += len;
-            DEBUG_HTTPCLIENT("[HTTP-Client] read chunk len: %d\n", len);
-
-            // data left?
-            if (len > 0) {
-                // read len bytes with timeout
-                int r = StreamSendSize(_client(), print, len);
-                if (r != len) {
-                    return HTTPC_ERROR_NO_STREAM;
-                }
-
-                //               if (_client->getLastSendReport() != Stream::Report::Success)
-                //                    // not all data transferred
-                //                    return returnError(StreamReportToHttpClientReport(_client->getLastSendReport()));
-                ret += r;
-            } else {
+        ret = StreamSendSize(&_stream, print, -1);
 
-                // if no length Header use global chunk size
-                if (_size <= 0) {
-                    _size = size;
-                }
-
-                // check if we have write all data out
-                if (ret != _size) {
-                    return returnError(HTTPC_ERROR_STREAM_WRITE);
-                }
-                break;
-            }
-
-            // read trailing \r\n at the end of the chunk
-            char buf[2];
-            auto trailing_seq_len = _client()->readBytes((uint8_t*)buf, 2);
-            if (trailing_seq_len != 2 || buf[0] != '\r' || buf[1] != '\n') {
-                return returnError(HTTPC_ERROR_READ_TIMEOUT);
-            }
+        if (ret == 0) {
+            return HTTPC_ERROR_NO_STREAM;
         }
     } else {
         return returnError(HTTPC_ERROR_ENCODING);
@@ -1112,6 +1067,9 @@ int HTTPClient::handleHeaderResponse() {
 
     _canReuse = _reuse;
 
+    // Hook up the HTTPStream to this WiFiClient already connected and headers decoded
+    _stream.reset(_client(), false);
+
     String transferEncoding;
 
     _transferEncoding = HTTPC_TE_IDENTITY;
@@ -1204,6 +1162,9 @@ int HTTPClient::handleHeaderResponse() {
                     _transferEncoding = HTTPC_TE_IDENTITY;
                 }
 
+                // Hook up the HTTPStream to this WiFiClient already connected and headers decoded
+                _stream.reset(_client(), _transferEncoding == HTTPC_TE_CHUNKED);
+
                 if (_returnCode <= 0) {
                     DEBUG_HTTPCLIENT("[HTTP-Client][handleHeaderResponse] Remote host is not an HTTP Server!");
                     _returnCode = HTTPC_ERROR_NO_HTTP_SERVER;

libraries/HTTPClient/src/HTTPClient.h

@@ -176,6 +176,228 @@ typedef struct  {
 } Cookie;
 typedef std::vector<Cookie> CookieJar;
 
+class HTTPStream : public WiFiClient {
+public:
+    HTTPStream() {
+        _conn = nullptr;
+        _chunked = false;
+        _chunkLen = 0;
+        _state = READ_HEX;
+        _partial = 0;
+        _eof = false;
+    }
+
+    HTTPStream(const HTTPStream&);
+    HTTPStream& operator=(const HTTPStream&);
+
+    void reset(WiFiClient *connection, bool chunked) {
+        DEBUG_HTTPCLIENT("[HTTPStream] reset(%p, %d)\n", connection, chunked);
+        _conn = connection;
+        _partial = 0;
+        _state = READ_HEX;
+        _chunked = chunked;
+        _eof = false;
+    }
+
+    virtual size_t write(uint8_t c) override {
+        return _conn->write(c);
+    }
+
+    virtual size_t write(const uint8_t *buf, size_t size) override {
+        return _conn->write(buf, size);
+    }
+
+    virtual uint8_t connected() override {
+        return _conn->connected();
+    }
+
+    virtual int available() override {
+        if (!_chunked) {
+            return _conn->available();
+        }
+        // We're chunked at this point
+        if (_eof) {
+            return 0;
+        }
+        if (!_chunkLen) {
+            tryReadChunkLen(1);
+        }
+        return std::min(_chunkLen, _conn->available());
+    }
+
+    virtual int availableForWrite() override {
+        return _conn->availableForWrite();
+    }
+
+    virtual void flush() override {
+        _conn->flush();
+    }
+
+    virtual void stop() override {
+        _conn->stop();
+    }
+
+    virtual int read() override {
+        if (!_chunked) {
+            return _conn->read();
+        }
+        // We're chunked at this point
+        if (_eof) {
+            DEBUG_HTTPCLIENT("[HTTPStream] ::read() after EOF\n");
+            return -1;
+        }
+        if (!_chunkLen) {
+            tryReadChunkLen(_timeout);
+        }
+        if (!_chunkLen) {
+            DEBUG_HTTPCLIENT("[HTTPStream] ::read no chunkLen\n");
+            return -1;
+        }
+        uint32_t start = millis();
+        while (((millis() - start) < _timeout) && !_conn->available() && _conn->connected()) {
+            delay(1);
+        }
+        if (_conn->available()) {
+            _chunkLen--;
+            return _conn->read();
+        } else {
+            DEBUG_HTTPCLIENT("[HTTPStream] ::read wrapped stream failure. avail = %d, conn = %d\n", _conn->available(), _conn->connected());
+            return -1;
+        }
+    }
+
+    virtual int read(uint8_t *buf, size_t size) override {
+        if (!_chunked) {
+            return _conn->read(buf, size);
+        }
+        for (int i = 0; i < (int)size; i++) {
+            int c = read();
+            if (c < 0) {
+                return i;
+            }
+            *(buf++) = (uint8_t)c;
+        }
+        return size;
+    }
+
+    virtual int peek() override {
+        if (!_chunked) {
+            return _conn->peek();
+        }
+        // We're chunked at this point
+        if (_eof) {
+            DEBUG_HTTPCLIENT("[HTTPStream] ::peek after EOF\n");
+            return -1;
+        }
+        if (!_chunkLen) {
+            tryReadChunkLen(_timeout);
+        }
+        if (!_chunkLen) {
+            DEBUG_HTTPCLIENT("[HTTPStream] ::peek no chunkLen\n");
+            return -1;
+        }
+        uint32_t start = millis();
+        while (((millis() - start) < _timeout) && !_conn->available() && _conn->connected()) {
+            delay(1);
+        }
+        if (_conn->available()) {
+            return _conn->peek();
+        } else {
+            DEBUG_HTTPCLIENT("[HTTPStream] ::peek wrapped stream failure. avail = %d, conn = %d\n", _conn->available(), _conn->connected());
+            return -1;
+        }
+    }
+
+    void setTimeout(unsigned long timeout) {
+        _timeout = timeout;
+        _conn->setTimeout(timeout);
+    }
+
+private:
+    void tryReadChunkLen(uint32_t to) {
+        if (_state == ERROR) {
+            return;
+        }
+        uint32_t start = millis();
+        while ((millis() - start) <= to) {
+            if (_conn->available()) {
+                int recv = _conn->read();
+                if (recv < 0) {
+                    DEBUG_HTTPCLIENT("[HTTPStream] Read of available data failed\n");
+                    _state = ERROR;
+                    return;
+                }
+                switch (_state) {
+                case READ_HEX:
+                    if (recv == '\r') {
+                        DEBUG_HTTPCLIENT("[HTTPStream] Saw \\r of chunk len\r\n");
+                        _state = READ_LF;
+                        break;
+                    }
+                    if (recv >= '0' && recv <= '9') {
+                        DEBUG_HTTPCLIENT("[HTTPStream] Read %c of chunk size\n", recv);
+                        _partial <<= 4;
+                        _partial |= recv - '0';
+                    } else if (tolower(recv) >= 'a' && tolower(recv) <= 'f') {
+                        DEBUG_HTTPCLIENT("[HTTPStream] Read %c of chunk size\n", recv);
+                        _partial <<= 4;
+                        _partial |= tolower(recv) - 'a' + 10;
+                    } else {
+                        DEBUG_HTTPCLIENT("[HTTPStream] READ_HEX error '%c'\n", recv);
+                        _state = ERROR;
+                        return;
+                    }
+                    break;
+                case READ_LF:
+                    if (recv != '\n') {
+                        _state = ERROR;
+                        DEBUG_HTTPCLIENT("[HTTPStream] READ_LF error '%02x'\n", recv);
+                        return;
+                    }
+                    DEBUG_HTTPCLIENT("[HTTPStream] Chunk len = %d\n", _partial);
+                    _chunkLen = _partial;
+                    _partial = 0;
+                    _state = TAIL_CR;
+                    if (_chunkLen == 0) {
+                        // 0-sized chunk is EOF special case
+                        _eof = true;
+                    }
+                    return;
+                case TAIL_CR:
+                    if (recv == '\r') {
+                        DEBUG_HTTPCLIENT("[HTTPStream] Saw \\r of chunk end\n");
+                        _state = TAIL_LF;
+                        break;
+                    }
+                    DEBUG_HTTPCLIENT("[HTTPStream] TAIL_CR error '%c'\n", recv);
+                    _state = ERROR;
+                    return;
+                case TAIL_LF:
+                    if (recv == '\n') {
+                        DEBUG_HTTPCLIENT("[HTTPStream] Saw \\n of chunk end\n");
+                        _state = READ_HEX;
+                        break;
+                    }
+                    DEBUG_HTTPCLIENT("[HTTPStream] TAIL_LF error '%c'\n", recv);
+                    _state = ERROR;
+                    return;
+                case ERROR:
+                    return;
+                }
+            }
+        }
+        DEBUG_HTTPCLIENT("[HTTPStream] Timeout waiting for chunk\n");
+    }
+
+    WiFiClient *_conn;
+    bool _chunked;
+    int _chunkLen;
+    enum { READ_HEX, READ_LF, TAIL_CR, TAIL_LF, ERROR } _state;
+    int _partial;
+    bool _eof;
+};
+
+
 class HTTPClient {
 public:
     HTTPClient() = default;
@@ -407,4 +629,6 @@ class HTTPClient {
     std::unique_ptr<StreamString> _payload;
     // Cookie jar support
     CookieJar *_cookieJar = nullptr;
+
+    HTTPStream _stream;
 };