From 522cf5d11aa43cf4844cce722bd611bf64868151 Mon Sep 17 00:00:00 2001
From: Sandeep Mistry <s.mistry@arduino.cc>
Date: Thu, 23 Mar 2017 14:41:41 -0400
Subject: [PATCH 1/4] Add support for chunked response bodies
---
keywords.txt | 1 +
src/HttpClient.cpp | 104 +++++++++++++++++++++++++++++++++++++++++++--
src/HttpClient.h | 22 ++++++++--
3 files changed, 120 insertions(+), 7 deletions(-)
diff --git a/keywords.txt b/keywords.txt
index 7c2061c..1d8bb80 100644
--- a/keywords.txt
+++ b/keywords.txt
@@ -30,6 +30,7 @@ endOfHeadersReached KEYWORD2
endOfBodyReached KEYWORD2
completed KEYWORD2
contentLength KEYWORD2
+isChunked KEYWORD2
connectionKeepAlive KEYWORD2
noDefaultRequestHeaders KEYWORD2
headerAvailable KEYWORD2
diff --git a/src/HttpClient.cpp b/src/HttpClient.cpp
index f3accc3..2c6d941 100644
--- a/src/HttpClient.cpp
+++ b/src/HttpClient.cpp
@@ -8,6 +8,7 @@
// Initialize constants
const char* HttpClient::kUserAgent = "Arduino/2.2.0";
const char* HttpClient::kContentLengthPrefix = HTTP_HEADER_CONTENT_LENGTH ": ";
+const char* HttpClient::kTransferEncodingChunked = HTTP_HEADER_TRANSFER_ENCODING ": " HTTP_HEADER_VALUE_CHUNKED;
HttpClient::HttpClient(Client& aClient, const char* aServerName, uint16_t aServerPort)
: iClient(&aClient), iServerName(aServerName), iServerAddress(), iServerPort(aServerPort),
@@ -35,6 +36,9 @@ void HttpClient::resetState()
iContentLength = kNoContentLengthHeader;
iBodyLengthConsumed = 0;
iContentLengthPtr = kContentLengthPrefix;
+ iTransferEncodingChunkedPtr = kTransferEncodingChunked;
+ iIsChunked = false;
+ iChunkLength = 0;
iHttpResponseTimeout = kHttpResponseTimeout;
}
@@ -62,7 +66,7 @@ void HttpClient::beginRequest()
int HttpClient::startRequest(const char* aURLPath, const char* aHttpMethod,
const char* aContentType, int aContentLength, const byte aBody[])
{
- if (iState == eReadingBody)
+ if (iState == eReadingBody || iState == eReadingChunkLength || iState == eReadingBodyChunk)
{
flushClientRx();
@@ -528,6 +532,11 @@ int HttpClient::skipResponseHeaders()
}
}
+bool HttpClient::endOfHeadersReached()
+{
+ return (iState == eReadingBody || iState == eReadingChunkLength || iState == eReadingBodyChunk);
+};
+
int HttpClient::contentLength()
{
// skip the response headers, if they haven't been read already
@@ -590,8 +599,65 @@ bool HttpClient::endOfBodyReached()
return false;
}
+int HttpClient::available()
+{
+ if (iState == eReadingChunkLength)
+ {
+ while (iClient->available())
+ {
+ char c = iClient->read();
+
+ if (c == '\n')
+ {
+ iState = eReadingBodyChunk;
+ break;
+ }
+ else if (c == '\r')
+ {
+ // no-op
+ }
+ else if (isHexadecimalDigit(c))
+ {
+ char digit[2] = {c, '\0'};
+
+ iChunkLength = (iChunkLength * 16) + strtol(digit, NULL, 16);
+ }
+ }
+ }
+
+ if (iState == eReadingBodyChunk && iChunkLength == 0)
+ {
+ iState = eReadingChunkLength;
+ }
+
+ if (iState == eReadingChunkLength)
+ {
+ return 0;
+ }
+
+ int clientAvailable = iClient->available();
+
+ if (iState == eReadingBodyChunk)
+ {
+ return min(clientAvailable, iChunkLength);
+ }
+ else
+ {
+ return clientAvailable;
+ }
+}
+
+
int HttpClient::read()
{
+ if (iState == eReadingBodyChunk)
+ {
+ if (!available())
+ {
+ return -1;
+ }
+ }
+
int ret = iClient->read();
if (ret >= 0)
{
@@ -601,6 +667,16 @@ int HttpClient::read()
// So keep track of how many bytes are left
iBodyLengthConsumed++;
}
+
+ if (iState == eReadingBodyChunk)
+ {
+ iChunkLength--;
+
+ if (iChunkLength == 0)
+ {
+ iState = eReadingChunkLength;
+ }
+ }
}
return ret;
}
@@ -714,7 +790,18 @@ int HttpClient::readHeader()
iBodyLengthConsumed = 0;
}
}
- else if ((iContentLengthPtr == kContentLengthPrefix) && (c == '\r'))
+ else if (*iTransferEncodingChunkedPtr == c)
+ {
+ // This character matches, just move along
+ iTransferEncodingChunkedPtr++;
+ if (*iTransferEncodingChunkedPtr == '\0')
+ {
+ // We've reached the end of the Transfer Encoding: chunked header
+ iIsChunked = true;
+ iState = eSkipToEndOfHeader;
+ }
+ }
+ else if (((iContentLengthPtr == kContentLengthPrefix) && (iTransferEncodingChunkedPtr == kTransferEncodingChunked)) && (c == '\r'))
{
// We've found a '\r' at the start of a line, so this is probably
// the end of the headers
@@ -722,7 +809,7 @@ int HttpClient::readHeader()
}
else
{
- // This isn't the Content-Length header, skip to the end of the line
+ // This isn't the Content-Length or Transfer Encoding chunked header, skip to the end of the line
iState = eSkipToEndOfHeader;
}
break;
@@ -742,7 +829,15 @@ int HttpClient::readHeader()
case eLineStartingCRFound:
if (c == '\n')
{
- iState = eReadingBody;
+ if (iIsChunked)
+ {
+ iState = eReadingChunkLength;
+ iChunkLength = 0;
+ }
+ else
+ {
+ iState = eReadingBody;
+ }
}
break;
default:
@@ -755,6 +850,7 @@ int HttpClient::readHeader()
// We've got to the end of this line, start processing again
iState = eStatusCodeRead;
iContentLengthPtr = kContentLengthPrefix;
+ iTransferEncodingChunkedPtr = kTransferEncodingChunked;
}
// And return the character read to whoever wants it
return c;
diff --git a/src/HttpClient.h b/src/HttpClient.h
index eaab360..7433f67 100644
--- a/src/HttpClient.h
+++ b/src/HttpClient.h
@@ -34,7 +34,9 @@ static const int HTTP_ERROR_INVALID_RESPONSE =-4;
#define HTTP_HEADER_CONTENT_LENGTH "Content-Length"
#define HTTP_HEADER_CONTENT_TYPE "Content-Type"
#define HTTP_HEADER_CONNECTION "Connection"
+#define HTTP_HEADER_TRANSFER_ENCODING "Transfer-Encoding"
#define HTTP_HEADER_USER_AGENT "User-Agent"
+#define HTTP_HEADER_VALUE_CHUNKED "chunked"
class HttpClient : public Client
{
@@ -247,7 +249,7 @@ class HttpClient : public Client
/** Test whether all of the response headers have been consumed.
@return true if we are now processing the response body, else false
*/
- bool endOfHeadersReached() { return (iState == eReadingBody); };
+ bool endOfHeadersReached();
/** Test whether the end of the body has been reached.
Only works if the Content-Length header was returned by the server
@@ -265,6 +267,11 @@ class HttpClient : public Client
*/
int contentLength();
+ /** Returns if the response body is chunked
+ @return true if response body is chunked, false otherwise
+ */
+ int isChunked() { return iIsChunked; }
+
/** Return the response body as a String
Also skips response headers if they have not been read already
MUST be called after responseStatusCode()
@@ -286,7 +293,7 @@ class HttpClient : public Client
virtual size_t write(uint8_t aByte) { if (iState < eRequestSent) { finishHeaders(); }; return iClient-> write(aByte); };
virtual size_t write(const uint8_t *aBuffer, size_t aSize) { if (iState < eRequestSent) { finishHeaders(); }; return iClient->write(aBuffer, aSize); };
// Inherited from Stream
- virtual int available() { return iClient->available(); };
+ virtual int available();
/** Read the next byte from the server.
@return Byte read or -1 if there are no bytes available.
*/
@@ -332,6 +339,7 @@ class HttpClient : public Client
// processing)
static const int kHttpResponseTimeout = 30*1000;
static const char* kContentLengthPrefix;
+ static const char* kTransferEncodingChunked;
typedef enum {
eIdle,
eRequestStarted,
@@ -341,7 +349,9 @@ class HttpClient : public Client
eReadingContentLength,
eSkipToEndOfHeader,
eLineStartingCRFound,
- eReadingBody
+ eReadingBody,
+ eReadingChunkLength,
+ eReadingBodyChunk
} tHttpState;
// Client we're using
Client* iClient;
@@ -360,6 +370,12 @@ class HttpClient : public Client
int iBodyLengthConsumed;
// How far through a Content-Length header prefix we are
const char* iContentLengthPtr;
+ // How far through a Transfer-Encoding chunked header we are
+ const char* iTransferEncodingChunkedPtr;
+ // Stores if the response body is chunked
+ bool iIsChunked;
+ // Stores the value of the current chunk length, if present
+ int iChunkLength;
uint32_t iHttpResponseTimeout;
bool iConnectionClose;
bool iSendDefaultRequestHeaders;
From 2a9c01c21070dd035ed8b9d4374dee9cfe2bcb38 Mon Sep 17 00:00:00 2001
From: Sandeep Mistry <s.mistry@arduino.cc>
Date: Sat, 25 Mar 2017 09:40:52 -0400
Subject: [PATCH 2/4] Change read() to always check available() if response is
chunked
---
src/HttpClient.cpp | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
diff --git a/src/HttpClient.cpp b/src/HttpClient.cpp
index 2c6d941..8a5b5c9 100644
--- a/src/HttpClient.cpp
+++ b/src/HttpClient.cpp
@@ -650,12 +650,9 @@ int HttpClient::available()
int HttpClient::read()
{
- if (iState == eReadingBodyChunk)
+ if (iIsChunked && !available())
{
- if (!available())
- {
- return -1;
- }
+ return -1;
}
int ret = iClient->read();
From fdedff59b8a795c5205d9d90fa4e5cee28f3cc78 Mon Sep 17 00:00:00 2001
From: Sandeep Mistry <s.mistry@arduino.cc>
Date: Thu, 13 Apr 2017 09:33:01 -0400
Subject: [PATCH 3/4] Rename isChunked to isResponseChunked
---
keywords.txt | 2 +-
src/HttpClient.h | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/keywords.txt b/keywords.txt
index 1d8bb80..8d6a7ca 100644
--- a/keywords.txt
+++ b/keywords.txt
@@ -30,7 +30,7 @@ endOfHeadersReached KEYWORD2
endOfBodyReached KEYWORD2
completed KEYWORD2
contentLength KEYWORD2
-isChunked KEYWORD2
+isResponseChunked KEYWORD2
connectionKeepAlive KEYWORD2
noDefaultRequestHeaders KEYWORD2
headerAvailable KEYWORD2
diff --git a/src/HttpClient.h b/src/HttpClient.h
index 7433f67..e120ef7 100644
--- a/src/HttpClient.h
+++ b/src/HttpClient.h
@@ -270,7 +270,7 @@ class HttpClient : public Client
/** Returns if the response body is chunked
@return true if response body is chunked, false otherwise
*/
- int isChunked() { return iIsChunked; }
+ int isResponseChunked() { return iIsChunked; }
/** Return the response body as a String
Also skips response headers if they have not been read already
From 0dd9f4209586734e3980737bc3548699555211aa Mon Sep 17 00:00:00 2001
From: Sandeep Mistry <s.mistry@arduino.cc>
Date: Thu, 13 Apr 2017 09:33:29 -0400
Subject: [PATCH 4/4] Add new /chunked endpoint to Node.js test server
---
examples/node_test_server/getPostPutDelete.js | 45 +++++++++++++++++++
1 file changed, 45 insertions(+)
diff --git a/examples/node_test_server/getPostPutDelete.js b/examples/node_test_server/getPostPutDelete.js
index e055b65..f3dd4d8 100644
--- a/examples/node_test_server/getPostPutDelete.js
+++ b/examples/node_test_server/getPostPutDelete.js
@@ -21,6 +21,51 @@ function serverStart() {
console.log('Server listening on port '+ port);
}
+app.get('/chunked', function(request, response) {
+ response.write('\n');
+ response.write(' `:;;;,` .:;;:. \n');
+ response.write(' .;;;;;;;;;;;` :;;;;;;;;;;: TM \n');
+ response.write(' `;;;;;;;;;;;;;;;` :;;;;;;;;;;;;;;; \n');
+ response.write(' :;;;;;;;;;;;;;;;;;; `;;;;;;;;;;;;;;;;;; \n');
+ response.write(' ;;;;;;;;;;;;;;;;;;;;; .;;;;;;;;;;;;;;;;;;;; \n');
+ response.write(' ;;;;;;;;:` `;;;;;;;;; ,;;;;;;;;.` .;;;;;;;; \n');
+ response.write(' .;;;;;;, :;;;;;;; .;;;;;;; ;;;;;;; \n');
+ response.write(' ;;;;;; ;;;;;;; ;;;;;;, ;;;;;;. \n');
+ response.write(' ,;;;;; ;;;;;;.;;;;;;` ;;;;;; \n');
+ response.write(' ;;;;;. ;;;;;;;;;;;` ``` ;;;;;`\n');
+ response.write(' ;;;;; ;;;;;;;;;, ;;; .;;;;;\n');
+ response.write('`;;;;: `;;;;;;;; ;;; ;;;;;\n');
+ response.write(',;;;;` `,,,,,,,, ;;;;;;; .,,;;;,,, ;;;;;\n');
+ response.write(':;;;;` .;;;;;;;; ;;;;;, :;;;;;;;; ;;;;;\n');
+ response.write(':;;;;` .;;;;;;;; `;;;;;; :;;;;;;;; ;;;;;\n');
+ response.write('.;;;;. ;;;;;;;. ;;; ;;;;;\n');
+ response.write(' ;;;;; ;;;;;;;;; ;;; ;;;;;\n');
+ response.write(' ;;;;; .;;;;;;;;;; ;;; ;;;;;,\n');
+ response.write(' ;;;;;; `;;;;;;;;;;;; ;;;;; \n');
+ response.write(' `;;;;;, .;;;;;; ;;;;;;; ;;;;;; \n');
+ response.write(' ;;;;;;: :;;;;;;. ;;;;;;; ;;;;;; \n');
+ response.write(' ;;;;;;;` .;;;;;;;, ;;;;;;;; ;;;;;;;: \n');
+ response.write(' ;;;;;;;;;:,:;;;;;;;;;: ;;;;;;;;;;:,;;;;;;;;;; \n');
+ response.write(' `;;;;;;;;;;;;;;;;;;;. ;;;;;;;;;;;;;;;;;;;; \n');
+ response.write(' ;;;;;;;;;;;;;;;;; :;;;;;;;;;;;;;;;;: \n');
+ response.write(' ,;;;;;;;;;;;;;, ;;;;;;;;;;;;;; \n');
+ response.write(' .;;;;;;;;;` ,;;;;;;;;: \n');
+ response.write(' \n');
+ response.write(' \n');
+ response.write(' \n');
+ response.write(' \n');
+ response.write(' ;;; ;;;;;` ;;;;: .;; ;; ,;;;;;, ;;. `;, ;;;; \n');
+ response.write(' ;;; ;;:;;; ;;;;;; .;; ;; ,;;;;;: ;;; `;, ;;;:;; \n');
+ response.write(' ,;:; ;; ;; ;; ;; .;; ;; ,;, ;;;,`;, ;; ;; \n');
+ response.write(' ;; ;: ;; ;; ;; ;; .;; ;; ,;, ;;;;`;, ;; ;;. \n');
+ response.write(' ;: ;; ;;;;;: ;; ;; .;; ;; ,;, ;;`;;;, ;; ;;` \n');
+ response.write(' ,;;;;; ;;`;; ;; ;; .;; ;; ,;, ;; ;;;, ;; ;; \n');
+ response.write(' ;; ,;, ;; .;; ;;;;;: ;;;;;: ,;;;;;: ;; ;;, ;;;;;; \n');
+ response.write(' ;; ;; ;; ;;` ;;;;. `;;;: ,;;;;;, ;; ;;, ;;;; \n');
+ response.write('\n');
+ response.end();
+});
+
// this is the POST handler:
app.all('/*', function (request, response) {
console.log('Got a ' + request.method + ' request');