Skip to content

Add support for chunked response bodies #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Apr 13, 2017
Merged

Add support for chunked response bodies #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
522cf5d
Add support for chunked response bodies
sandeepmistry Mar 23, 2017
2a9c01c
Change read() to always check available() if response is chunked
sandeepmistry Mar 25, 2017
fdedff5
Rename isChunked to isResponseChunked
sandeepmistry Apr 13, 2017
0dd9f42
Add new /chunked endpoint to Node.js test server
sandeepmistry Apr 13, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions examples/node_test_server/getPostPutDelete.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,51 @@ function serverStart() {
console.log('Server listening on port '+ port);
}

app.get('/chunked', function(request, response) {
response.write('\n');
response.write(' `:;;;,` .:;;:. \n');
response.write(' .;;;;;;;;;;;` :;;;;;;;;;;: TM \n');
response.write(' `;;;;;;;;;;;;;;;` :;;;;;;;;;;;;;;; \n');
response.write(' :;;;;;;;;;;;;;;;;;; `;;;;;;;;;;;;;;;;;; \n');
response.write(' ;;;;;;;;;;;;;;;;;;;;; .;;;;;;;;;;;;;;;;;;;; \n');
response.write(' ;;;;;;;;:` `;;;;;;;;; ,;;;;;;;;.` .;;;;;;;; \n');
response.write(' .;;;;;;, :;;;;;;; .;;;;;;; ;;;;;;; \n');
response.write(' ;;;;;; ;;;;;;; ;;;;;;, ;;;;;;. \n');
response.write(' ,;;;;; ;;;;;;.;;;;;;` ;;;;;; \n');
response.write(' ;;;;;. ;;;;;;;;;;;` ``` ;;;;;`\n');
response.write(' ;;;;; ;;;;;;;;;, ;;; .;;;;;\n');
response.write('`;;;;: `;;;;;;;; ;;; ;;;;;\n');
response.write(',;;;;` `,,,,,,,, ;;;;;;; .,,;;;,,, ;;;;;\n');
response.write(':;;;;` .;;;;;;;; ;;;;;, :;;;;;;;; ;;;;;\n');
response.write(':;;;;` .;;;;;;;; `;;;;;; :;;;;;;;; ;;;;;\n');
response.write('.;;;;. ;;;;;;;. ;;; ;;;;;\n');
response.write(' ;;;;; ;;;;;;;;; ;;; ;;;;;\n');
response.write(' ;;;;; .;;;;;;;;;; ;;; ;;;;;,\n');
response.write(' ;;;;;; `;;;;;;;;;;;; ;;;;; \n');
response.write(' `;;;;;, .;;;;;; ;;;;;;; ;;;;;; \n');
response.write(' ;;;;;;: :;;;;;;. ;;;;;;; ;;;;;; \n');
response.write(' ;;;;;;;` .;;;;;;;, ;;;;;;;; ;;;;;;;: \n');
response.write(' ;;;;;;;;;:,:;;;;;;;;;: ;;;;;;;;;;:,;;;;;;;;;; \n');
response.write(' `;;;;;;;;;;;;;;;;;;;. ;;;;;;;;;;;;;;;;;;;; \n');
response.write(' ;;;;;;;;;;;;;;;;; :;;;;;;;;;;;;;;;;: \n');
response.write(' ,;;;;;;;;;;;;;, ;;;;;;;;;;;;;; \n');
response.write(' .;;;;;;;;;` ,;;;;;;;;: \n');
response.write(' \n');
response.write(' \n');
response.write(' \n');
response.write(' \n');
response.write(' ;;; ;;;;;` ;;;;: .;; ;; ,;;;;;, ;;. `;, ;;;; \n');
response.write(' ;;; ;;:;;; ;;;;;; .;; ;; ,;;;;;: ;;; `;, ;;;:;; \n');
response.write(' ,;:; ;; ;; ;; ;; .;; ;; ,;, ;;;,`;, ;; ;; \n');
response.write(' ;; ;: ;; ;; ;; ;; .;; ;; ,;, ;;;;`;, ;; ;;. \n');
response.write(' ;: ;; ;;;;;: ;; ;; .;; ;; ,;, ;;`;;;, ;; ;;` \n');
response.write(' ,;;;;; ;;`;; ;; ;; .;; ;; ,;, ;; ;;;, ;; ;; \n');
response.write(' ;; ,;, ;; .;; ;;;;;: ;;;;;: ,;;;;;: ;; ;;, ;;;;;; \n');
response.write(' ;; ;; ;; ;;` ;;;;. `;;;: ,;;;;;, ;; ;;, ;;;; \n');
response.write('\n');
response.end();
});

// this is the POST handler:
app.all('/*', function (request, response) {
console.log('Got a ' + request.method + ' request');
Expand Down
1 change: 1 addition & 0 deletions keywords.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ endOfHeadersReached KEYWORD2
endOfBodyReached KEYWORD2
completed KEYWORD2
contentLength KEYWORD2
isResponseChunked KEYWORD2
connectionKeepAlive KEYWORD2
noDefaultRequestHeaders KEYWORD2
headerAvailable KEYWORD2
Expand Down
101 changes: 97 additions & 4 deletions src/HttpClient.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
// Initialize constants
const char* HttpClient::kUserAgent = "Arduino/2.2.0";
const char* HttpClient::kContentLengthPrefix = HTTP_HEADER_CONTENT_LENGTH ": ";
const char* HttpClient::kTransferEncodingChunked = HTTP_HEADER_TRANSFER_ENCODING ": " HTTP_HEADER_VALUE_CHUNKED;

HttpClient::HttpClient(Client& aClient, const char* aServerName, uint16_t aServerPort)
: iClient(&aClient), iServerName(aServerName), iServerAddress(), iServerPort(aServerPort),
Expand Down Expand Up @@ -35,6 +36,9 @@ void HttpClient::resetState()
iContentLength = kNoContentLengthHeader;
iBodyLengthConsumed = 0;
iContentLengthPtr = kContentLengthPrefix;
iTransferEncodingChunkedPtr = kTransferEncodingChunked;
iIsChunked = false;
iChunkLength = 0;
iHttpResponseTimeout = kHttpResponseTimeout;
}

Expand Down Expand Up @@ -62,7 +66,7 @@ void HttpClient::beginRequest()
int HttpClient::startRequest(const char* aURLPath, const char* aHttpMethod,
const char* aContentType, int aContentLength, const byte aBody[])
{
if (iState == eReadingBody)
if (iState == eReadingBody || iState == eReadingChunkLength || iState == eReadingBodyChunk)
{
flushClientRx();

Expand Down Expand Up @@ -528,6 +532,11 @@ int HttpClient::skipResponseHeaders()
}
}

bool HttpClient::endOfHeadersReached()
{
return (iState == eReadingBody || iState == eReadingChunkLength || iState == eReadingBodyChunk);
};

int HttpClient::contentLength()
{
// skip the response headers, if they haven't been read already
Expand Down Expand Up @@ -590,8 +599,62 @@ bool HttpClient::endOfBodyReached()
return false;
}

int HttpClient::available()
{
if (iState == eReadingChunkLength)
{
while (iClient->available())
{
char c = iClient->read();

if (c == '\n')
{
iState = eReadingBodyChunk;
break;
}
else if (c == '\r')
{
// no-op
}
else if (isHexadecimalDigit(c))
{
char digit[2] = {c, '\0'};

iChunkLength = (iChunkLength * 16) + strtol(digit, NULL, 16);
}
}
}

if (iState == eReadingBodyChunk && iChunkLength == 0)
{
iState = eReadingChunkLength;
}

if (iState == eReadingChunkLength)
{
return 0;
}

int clientAvailable = iClient->available();

if (iState == eReadingBodyChunk)
{
return min(clientAvailable, iChunkLength);
}
else
{
return clientAvailable;
}
}


int HttpClient::read()
{
if (iIsChunked && !available())
{
return -1;
}

int ret = iClient->read();
if (ret >= 0)
{
Expand All @@ -601,6 +664,16 @@ int HttpClient::read()
// So keep track of how many bytes are left
iBodyLengthConsumed++;
}

if (iState == eReadingBodyChunk)
{
iChunkLength--;

if (iChunkLength == 0)
{
iState = eReadingChunkLength;
}
}
}
return ret;
}
Expand Down Expand Up @@ -714,15 +787,26 @@ int HttpClient::readHeader()
iBodyLengthConsumed = 0;
}
}
else if ((iContentLengthPtr == kContentLengthPrefix) && (c == '\r'))
else if (*iTransferEncodingChunkedPtr == c)
{
// This character matches, just move along
iTransferEncodingChunkedPtr++;
if (*iTransferEncodingChunkedPtr == '\0')
{
// We've reached the end of the Transfer Encoding: chunked header
iIsChunked = true;
iState = eSkipToEndOfHeader;
}
}
else if (((iContentLengthPtr == kContentLengthPrefix) && (iTransferEncodingChunkedPtr == kTransferEncodingChunked)) && (c == '\r'))
{
// We've found a '\r' at the start of a line, so this is probably
// the end of the headers
iState = eLineStartingCRFound;
}
else
{
// This isn't the Content-Length header, skip to the end of the line
// This isn't the Content-Length or Transfer Encoding chunked header, skip to the end of the line
iState = eSkipToEndOfHeader;
}
break;
Expand All @@ -742,7 +826,15 @@ int HttpClient::readHeader()
case eLineStartingCRFound:
if (c == '\n')
{
iState = eReadingBody;
if (iIsChunked)
{
iState = eReadingChunkLength;
iChunkLength = 0;
}
else
{
iState = eReadingBody;
}
}
break;
default:
Expand All @@ -755,6 +847,7 @@ int HttpClient::readHeader()
// We've got to the end of this line, start processing again
iState = eStatusCodeRead;
iContentLengthPtr = kContentLengthPrefix;
iTransferEncodingChunkedPtr = kTransferEncodingChunked;
}
// And return the character read to whoever wants it
return c;
Expand Down
22 changes: 19 additions & 3 deletions src/HttpClient.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,9 @@ static const int HTTP_ERROR_INVALID_RESPONSE =-4;
#define HTTP_HEADER_CONTENT_LENGTH "Content-Length"
#define HTTP_HEADER_CONTENT_TYPE "Content-Type"
#define HTTP_HEADER_CONNECTION "Connection"
#define HTTP_HEADER_TRANSFER_ENCODING "Transfer-Encoding"
#define HTTP_HEADER_USER_AGENT "User-Agent"
#define HTTP_HEADER_VALUE_CHUNKED "chunked"

class HttpClient : public Client
{
Expand Down Expand Up @@ -247,7 +249,7 @@ class HttpClient : public Client
/** Test whether all of the response headers have been consumed.
@return true if we are now processing the response body, else false
*/
bool endOfHeadersReached() { return (iState == eReadingBody); };
bool endOfHeadersReached();

/** Test whether the end of the body has been reached.
Only works if the Content-Length header was returned by the server
Expand All @@ -265,6 +267,11 @@ class HttpClient : public Client
*/
int contentLength();

/** Returns if the response body is chunked
@return true if response body is chunked, false otherwise
*/
int isResponseChunked() { return iIsChunked; }

/** Return the response body as a String
Also skips response headers if they have not been read already
MUST be called after responseStatusCode()
Expand All @@ -286,7 +293,7 @@ class HttpClient : public Client
virtual size_t write(uint8_t aByte) { if (iState < eRequestSent) { finishHeaders(); }; return iClient-> write(aByte); };
virtual size_t write(const uint8_t *aBuffer, size_t aSize) { if (iState < eRequestSent) { finishHeaders(); }; return iClient->write(aBuffer, aSize); };
// Inherited from Stream
virtual int available() { return iClient->available(); };
virtual int available();
/** Read the next byte from the server.
@return Byte read or -1 if there are no bytes available.
*/
Expand Down Expand Up @@ -332,6 +339,7 @@ class HttpClient : public Client
// processing)
static const int kHttpResponseTimeout = 30*1000;
static const char* kContentLengthPrefix;
static const char* kTransferEncodingChunked;
typedef enum {
eIdle,
eRequestStarted,
Expand All @@ -341,7 +349,9 @@ class HttpClient : public Client
eReadingContentLength,
eSkipToEndOfHeader,
eLineStartingCRFound,
eReadingBody
eReadingBody,
eReadingChunkLength,
eReadingBodyChunk
} tHttpState;
// Client we're using
Client* iClient;
Expand All @@ -360,6 +370,12 @@ class HttpClient : public Client
int iBodyLengthConsumed;
// How far through a Content-Length header prefix we are
const char* iContentLengthPtr;
// How far through a Transfer-Encoding chunked header we are
const char* iTransferEncodingChunkedPtr;
// Stores if the response body is chunked
bool iIsChunked;
// Stores the value of the current chunk length, if present
int iChunkLength;
uint32_t iHttpResponseTimeout;
bool iConnectionClose;
bool iSendDefaultRequestHeaders;
Expand Down