Handle large octet-stream (master branch) (#9440)

* Handle large octet-stream

* Add exemple Upload Huge File

* Remove unuse function printDirectory

* Fix upload path

* Simplify and generalize the body parsing.

* Create .skip.esp32h2

---------

Co-authored-by: me-no-dev <[email protected]>

Author: nathannau

Diff for: libraries/WebServer/examples/UploadHugeFile/.skip.esp32h2

@@ -0,0 +1,13 @@
+# Upload Huge File To SD Over Http
+
+This project is an example of an HTTP server designed to facilitate the transfer of large files using the PUT method, in accordance with RFC specifications.
+
+### Example cURL Command
+
+```bash
+curl -X PUT -T ./my-file.mp3 http://esp-ip/upload/my-file.mp3
+```
+
+## Resources
+
+- RFC HTTP/1.0 - Additional Request Methods - PUT : [Link](https://datatracker.ietf.org/doc/html/rfc1945#appendix-D.1.1)

Diff for: libraries/WebServer/examples/UploadHugeFile/README.md

@@ -0,0 +1,88 @@
+#include <WiFi.h>
+#include <WiFiClient.h>
+#include <WebServer.h>
+#include <uri/UriRegex.h>
+#include <SD.h>
+
+const char* ssid = "**********";
+const char* password = "**********";
+
+WebServer server(80);
+
+File rawFile;
+void handleCreate() {
+  server.send(200, "text/plain", "");
+}
+void handleCreateProcess() {
+  String path = "/" + server.pathArg(0);
+  HTTPRaw& raw = server.raw();
+  if (raw.status == RAW_START) {
+    if (SD.exists((char *)path.c_str())) {
+      SD.remove((char *)path.c_str());
+    }
+    rawFile = SD.open(path.c_str(), FILE_WRITE);
+    Serial.print("Upload: START, filename: "); 
+    Serial.println(path);
+  } else if (raw.status == RAW_WRITE) {
+    if (rawFile) {
+      rawFile.write(raw.buf, raw.currentSize);
+    }
+    Serial.print("Upload: WRITE, Bytes: "); 
+    Serial.println(raw.currentSize);
+  } else if (raw.status == RAW_END) {
+    if (rawFile) {
+      rawFile.close();
+    }
+    Serial.print("Upload: END, Size: "); 
+    Serial.println(raw.totalSize);
+  }
+}
+
+void returnFail(String msg) {
+  server.send(500, "text/plain", msg + "\r\n");
+}
+
+void handleNotFound() {
+  String message = "File Not Found\n\n";
+  message += "URI: ";
+  message += server.uri();
+  message += "\nMethod: ";
+  message += (server.method() == HTTP_GET) ? "GET" : "POST";
+  message += "\nArguments: ";
+  message += server.args();
+  message += "\n";
+  for (uint8_t i = 0; i < server.args(); i++) {
+    message += " " + server.argName(i) + ": " + server.arg(i) + "\n";
+  }
+  server.send(404, "text/plain", message);
+}
+
+void setup(void) {
+  Serial.begin(115200);
+
+  while (!SD.begin()) delay(1);
+  Serial.println("SD Card initialized.");
+
+  WiFi.mode(WIFI_STA);
+  WiFi.begin(ssid, password);
+
+  while (WiFi.status() != WL_CONNECTED) {
+    delay(500);
+    Serial.print(".");
+  }
+  Serial.print("Connected to ");
+  Serial.println(ssid);
+  Serial.print("IP address: ");
+  Serial.println(WiFi.localIP());
+
+  server.on(UriRegex("/upload/(.*)"), HTTP_PUT, handleCreate, handleCreateProcess);
+  server.onNotFound(handleNotFound);
+  server.begin();
+  Serial.println("HTTP server started");
+
+}
+
+void loop(void) {
+  server.handleClient();
+  delay(2);//allow the cpu to switch to other tasks
+}

Diff for: libraries/WebServer/examples/UploadHugeFile/UploadHugeFile.ino

@@ -173,7 +173,30 @@ bool WebServer::_parseRequest(NetworkClient& client) {
       }
     }
 
-    if (!isForm){
+    if (!isForm && _currentHandler && _currentHandler->canRaw(_currentUri)){
+      log_v("Parse raw");
+      _currentRaw.reset(new HTTPRaw());
+      _currentRaw->status = RAW_START;
+      _currentRaw->totalSize = 0;
+      _currentRaw->currentSize = 0;
+      log_v("Start Raw");
+      _currentHandler->raw(*this, _currentUri, *_currentRaw);
+      _currentRaw->status = RAW_WRITE;
+
+      while (_currentRaw->totalSize < _clientContentLength) {
+        _currentRaw->currentSize = client.readBytes(_currentRaw->buf, HTTP_RAW_BUFLEN);
+        _currentRaw->totalSize += _currentRaw->currentSize;
+        if (_currentRaw->currentSize == 0) {
+          _currentRaw->status = RAW_ABORTED;
+          _currentHandler->raw(*this, _currentUri, *_currentRaw);
+          return false;
+        }
+        _currentHandler->raw(*this, _currentUri, *_currentRaw);
+      }
+      _currentRaw->status = RAW_END;
+      _currentHandler->raw(*this, _currentUri, *_currentRaw);
+      log_v("Finish Raw");
+    } else if (!isForm) {
       size_t plainLength;
       char* plainBuf = readBytesWithTimeout(client, _clientContentLength, plainLength, HTTP_MAX_POST_WAIT);
       if (plainLength < _clientContentLength) {

Diff for: libraries/WebServer/src/Parsing.cpp

@@ -438,6 +438,7 @@ void WebServer::handleClient() {
     _currentClient = NetworkClient();
     _currentStatus = HC_NONE;
     _currentUpload.reset();
+    _currentRaw.reset();
   }
 
   if (callYield) {

Diff for: libraries/WebServer/src/WebServer.cpp

@@ -33,6 +33,7 @@
 
 enum HTTPUploadStatus { UPLOAD_FILE_START, UPLOAD_FILE_WRITE, UPLOAD_FILE_END,
                         UPLOAD_FILE_ABORTED };
+enum HTTPRawStatus { RAW_START, RAW_WRITE, RAW_END, RAW_ABORTED };
 enum HTTPClientStatus { HC_NONE, HC_WAIT_READ, HC_WAIT_CLOSE };
 enum HTTPAuthMethod { BASIC_AUTH, DIGEST_AUTH, OTHER_AUTH };
 
@@ -42,6 +43,10 @@ enum HTTPAuthMethod { BASIC_AUTH, DIGEST_AUTH, OTHER_AUTH };
 #define HTTP_UPLOAD_BUFLEN 1436
 #endif
 
+#ifndef HTTP_RAW_BUFLEN
+#define HTTP_RAW_BUFLEN 1436
+#endif
+
 #define HTTP_MAX_DATA_WAIT 5000 //ms to wait for the client to send the request
 #define HTTP_MAX_POST_WAIT 5000 //ms to wait for POST data to arrive
 #define HTTP_MAX_SEND_WAIT 5000 //ms to wait for data chunk to be ACKed
@@ -63,6 +68,15 @@ typedef struct {
   uint8_t buf[HTTP_UPLOAD_BUFLEN];
 } HTTPUpload;
 
+typedef struct
+{
+  HTTPRawStatus status;
+  size_t  totalSize;   // content size
+  size_t  currentSize; // size of data currently in buf
+  uint8_t buf[HTTP_UPLOAD_BUFLEN];
+  void    *data;       // additional data
+} HTTPRaw;
+
 #include "detail/RequestHandler.h"
 
 namespace fs {
@@ -128,6 +142,7 @@ class WebServer
   HTTPMethod method() { return _currentMethod; }
   virtual NetworkClient & client() { return _currentClient; }
   HTTPUpload& upload() { return *_currentUpload; }
+  HTTPRaw& raw() { return *_currentRaw; }
 
   String pathArg(unsigned int i); // get request path argument by number
   String arg(String name);        // get request argument value by name
@@ -232,6 +247,7 @@ class WebServer
   RequestArgument* _postArgs;
 
   std::unique_ptr<HTTPUpload> _currentUpload;
+  std::unique_ptr<HTTPRaw>    _currentRaw;
 
   int              _headerKeysCount;
   RequestArgument* _currentHeaders;

Diff for: libraries/WebServer/src/WebServer.h

@@ -9,8 +9,10 @@ class RequestHandler {
     virtual ~RequestHandler() { }
     virtual bool canHandle(HTTPMethod method, String uri) { (void) method; (void) uri; return false; }
     virtual bool canUpload(String uri) { (void) uri; return false; }
+    virtual bool canRaw(String uri) { (void) uri; return false; }
     virtual bool handle(WebServer& server, HTTPMethod requestMethod, String requestUri) { (void) server; (void) requestMethod; (void) requestUri; return false; }
     virtual void upload(WebServer& server, String requestUri, HTTPUpload& upload) { (void) server; (void) requestUri; (void) upload; }
+    virtual void raw(WebServer& server, String requestUri, HTTPRaw& raw) { (void) server; (void) requestUri; (void) raw; }
 
     RequestHandler* next() { return _next; }
     void next(RequestHandler* r) { _next = r; }

Diff for: libraries/WebServer/src/detail/RequestHandler.h

@@ -38,6 +38,12 @@ class FunctionRequestHandler : public RequestHandler {
 
         return true;
     }
+    bool canRaw(String requestUri) override {
+        if (!_ufn || _method == HTTP_GET)
+            return false;
+
+        return true;
+    }
 
     bool handle(WebServer& server, HTTPMethod requestMethod, String requestUri) override {
         (void) server;
@@ -55,6 +61,13 @@ class FunctionRequestHandler : public RequestHandler {
             _ufn();
     }
 
+    void raw(WebServer& server, String requestUri, HTTPRaw& raw) override {
+        (void)server;
+        (void)raw;
+        if (canRaw(requestUri))
+            _ufn();
+    }
+
 protected:
     WebServer::THandlerFunction _fn;
     WebServer::THandlerFunction _ufn;