From 180416f4bfeebd700c1a6821ce9035e44c83e949 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 29 Jan 2019 23:26:11 +0100 Subject: [PATCH 001/207] initial testing --- cores/esp8266/HardwareSerial.h | 4 +-- cores/esp8266/Print.h | 4 +++ cores/esp8266/Stream.cpp | 42 ++++++++++++++++++++++++ cores/esp8266/Stream.h | 11 +++++++ cores/esp8266/Udp.h | 2 +- libraries/ESP8266WiFi/src/WiFiClient.cpp | 6 ++-- libraries/ESP8266WiFi/src/WiFiClient.h | 4 +-- libraries/ESP8266WiFi/src/WiFiUdp.h | 2 +- tests/host/common/MockSerial.cpp | 6 ++++ 9 files changed, 72 insertions(+), 9 deletions(-) diff --git a/cores/esp8266/HardwareSerial.h b/cores/esp8266/HardwareSerial.h index 9d50be55e5..b0589c73cf 100644 --- a/cores/esp8266/HardwareSerial.h +++ b/cores/esp8266/HardwareSerial.h @@ -134,7 +134,7 @@ class HardwareSerial: public Stream return uart_read_char(_uart); } // ::read(buffer, size): same as readBytes without timeout - size_t read(char* buffer, size_t size) + size_t read(char* buffer, size_t size) override { return uart_read(_uart, buffer, size); } @@ -143,7 +143,7 @@ class HardwareSerial: public Stream { return readBytes((char*)buffer, size); } - int availableForWrite(void) + int availableForWrite(void) override { return static_cast(uart_tx_free(_uart)); } diff --git a/cores/esp8266/Print.h b/cores/esp8266/Print.h index 73a955b4dd..09afbc9db8 100644 --- a/cores/esp8266/Print.h +++ b/cores/esp8266/Print.h @@ -91,6 +91,10 @@ class Print { size_t println(void); virtual void flush() { /* Empty implementation for backward compatibility */ } + + // availableForWrite() must be overridden when possible ! + // overridden by HardwareSerial and WiFiClient when added in Print:: + virtual int availableForWrite() { return 1; } }; #endif diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index a29388b2bf..2cab8baf80 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -22,6 +22,8 @@ #include #include +#include + #define PARSE_TIMEOUT 1000 // default number of milli-seconds to wait #define NO_SKIP_CHAR 1 // a magic char not found in a valid ASCII numeric field @@ -255,3 +257,43 @@ String Stream::readStringUntil(char terminator) { return ret; } +size_t Stream::read (char* buffer, size_t maxLen) +{ +#ifdef DEBUG_ESP_CORE + static bool once = false; + if (!once) { + once = true; + printf((PGM_P)PSTR("Stream::read(data,len) should be overridden for better efficiency\r\n"); + } +#endif + + size_t written = 0; + while (written < maxLen && available()) + buffer[written++] = read(); + return written; +} + +#define MAXTRANSFERBLOCK 128 // allocated in stack, be nice + +size_t Stream::streamTo (Print& to, size_t maxLen) +{ + size_t afw; + size_t written = 0; + + while ((!maxLen || written < maxLen) && (afw = to.availableForWrite())) + { + size_t afr = available(); + if (afw > afr) + afw = afr; + if (!afw) + return written; + if (afw > MAXTRANSFERBLOCK) + afw = MAXTRANSFERBLOCK; + char temp[afw]; + size_t r = read(temp, afw); + size_t w = write(temp, afw); + assert(r == w); + written += w; + } + return written; +} diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index fa786dddc3..efedd5f221 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -104,6 +104,17 @@ class Stream: public Print { virtual String readString(); String readStringUntil(char terminator); + // read at most maxLen bytes + // returns effectively transfered bytes (can be less than maxLen) + // immediate return when no more data are available (no timeout) + virtual size_t read(char* buffer, size_t maxLen); + + // transfer at most maxlen bytes + // returns effectively transfered bytes (can be less than maxLen) + // maxlen==0 means transfer until nothing more can be read + // immediate return when no more data are available (no timeout) + virtual size_t streamTo (Print& to, size_t maxLen = 0); + protected: long parseInt(char skipChar); // as above but the given skipChar is ignored // as above but the given skipChar is ignored diff --git a/cores/esp8266/Udp.h b/cores/esp8266/Udp.h index ba2d5e3715..2d873154ae 100644 --- a/cores/esp8266/Udp.h +++ b/cores/esp8266/Udp.h @@ -73,7 +73,7 @@ class UDP: public Stream { virtual int read(unsigned char* buffer, size_t len) =0; // Read up to len characters from the current packet and place them into buffer // Returns the number of characters read, or 0 if none are available - virtual int read(char* buffer, size_t len) =0; + virtual size_t read(char* buffer, size_t len) =0; // Return the next byte from the current packet without moving on to the next byte virtual int peek() =0; virtual void flush() =0; // Finish reading the current packet diff --git a/libraries/ESP8266WiFi/src/WiFiClient.cpp b/libraries/ESP8266WiFi/src/WiFiClient.cpp index 4e100a032d..b623fb2f98 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.cpp +++ b/libraries/ESP8266WiFi/src/WiFiClient.cpp @@ -206,9 +206,9 @@ bool WiFiClient::getSync() const return _client->getSync(); } -size_t WiFiClient::availableForWrite () +int WiFiClient::availableForWrite () { - return _client? _client->availableForWrite(): 0; + return _client? (int)_client->availableForWrite(): 0; } size_t WiFiClient::write(uint8_t b) @@ -276,7 +276,7 @@ int WiFiClient::read() int WiFiClient::read(uint8_t* buf, size_t size) { - return (int) _client->read(reinterpret_cast(buf), size); + return (int)_client->read(reinterpret_cast(buf), size); } int WiFiClient::peek() diff --git a/libraries/ESP8266WiFi/src/WiFiClient.h b/libraries/ESP8266WiFi/src/WiFiClient.h index cd83fcbcbd..3860549133 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.h +++ b/libraries/ESP8266WiFi/src/WiFiClient.h @@ -66,7 +66,7 @@ class WiFiClient : public Client, public SList { virtual int available(); virtual int read(); - virtual int read(uint8_t *buf, size_t size); + virtual int read(uint8_t *buf, size_t size) override; virtual int peek(); virtual size_t peekBytes(uint8_t *buffer, size_t length); size_t peekBytes(char *buffer, size_t length) { @@ -84,7 +84,7 @@ class WiFiClient : public Client, public SList { static void setLocalPortStart(uint16_t port) { _localPort = port; } - size_t availableForWrite(); + int availableForWrite() override; friend class WiFiServer; diff --git a/libraries/ESP8266WiFi/src/WiFiUdp.h b/libraries/ESP8266WiFi/src/WiFiUdp.h index fb205513c5..42551c738b 100644 --- a/libraries/ESP8266WiFi/src/WiFiUdp.h +++ b/libraries/ESP8266WiFi/src/WiFiUdp.h @@ -89,7 +89,7 @@ class WiFiUDP : public UDP, public SList { int read(unsigned char* buffer, size_t len) override; // Read up to len characters from the current packet and place them into buffer // Returns the number of characters read, or 0 if none are available - int read(char* buffer, size_t len) override { return read((unsigned char*)buffer, len); }; + size_t read(char* buffer, size_t len) override { return read((unsigned char*)buffer, len); }; // Return the next byte from the current packet without moving on to the next byte int peek() override; void flush() override; // Finish reading the current packet diff --git a/tests/host/common/MockSerial.cpp b/tests/host/common/MockSerial.cpp index eedcaed115..8f2cdb9285 100644 --- a/tests/host/common/MockSerial.cpp +++ b/tests/host/common/MockSerial.cpp @@ -134,4 +134,10 @@ size_t uart_read(uart_t* uart, char* userbuffer, size_t usersize) return read(0, userbuffer, usersize); } +size_t uart_tx_free(uart_t* uart) // availableForWrite() +{ + (void)uart; + return 1; +} + } // extern "C" From 62c8bdd276058d5fc38e6550ef5f3b78499ff840 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 30 Jan 2019 01:14:17 +0100 Subject: [PATCH 002/207] Stream::printTo(Print&,len=0) only tested on emulation --- cores/esp8266/Print.cpp | 15 ++++++--------- cores/esp8266/Print.h | 2 +- cores/esp8266/Stream.cpp | 41 +++++++++++++++++----------------------- cores/esp8266/debug.h | 15 ++++++++++++++- 4 files changed, 38 insertions(+), 35 deletions(-) diff --git a/cores/esp8266/Print.cpp b/cores/esp8266/Print.cpp index 2870f3f87b..2d4e826905 100644 --- a/cores/esp8266/Print.cpp +++ b/cores/esp8266/Print.cpp @@ -33,15 +33,7 @@ /* default implementation: may be overridden */ size_t Print::write(const uint8_t *buffer, size_t size) { - -#ifdef DEBUG_ESP_CORE - static char not_the_best_way [] PROGMEM STORE_ATTR = "Print::write(data,len) should be overridden for better efficiency\r\n"; - static bool once = false; - if (!once) { - once = true; - os_printf_plus(not_the_best_way); - } -#endif + IAMSLOW("Print::write(buffer,len)"); size_t n = 0; while (size--) { @@ -304,3 +296,8 @@ size_t Print::printFloat(double number, uint8_t digits) { return n; } + +int Print::availableForWrite() { + IAMSLOW("Print::availableForWrite()"); + return 1; +} diff --git a/cores/esp8266/Print.h b/cores/esp8266/Print.h index 09afbc9db8..1a0c4e6dde 100644 --- a/cores/esp8266/Print.h +++ b/cores/esp8266/Print.h @@ -94,7 +94,7 @@ class Print { // availableForWrite() must be overridden when possible ! // overridden by HardwareSerial and WiFiClient when added in Print:: - virtual int availableForWrite() { return 1; } + virtual int availableForWrite(); }; #endif diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index 2cab8baf80..d31aa7ebd3 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -259,40 +259,33 @@ String Stream::readStringUntil(char terminator) { size_t Stream::read (char* buffer, size_t maxLen) { -#ifdef DEBUG_ESP_CORE - static bool once = false; - if (!once) { - once = true; - printf((PGM_P)PSTR("Stream::read(data,len) should be overridden for better efficiency\r\n"); - } -#endif + IAMSLOW("Stream::read(buffer,len)"); - size_t written = 0; - while (written < maxLen && available()) - buffer[written++] = read(); - return written; + size_t nbread = 0; + while (nbread < maxLen && available()) + buffer[nbread++] = read(); + return nbread; } #define MAXTRANSFERBLOCK 128 // allocated in stack, be nice size_t Stream::streamTo (Print& to, size_t maxLen) { - size_t afw; + size_t w; size_t written = 0; - - while ((!maxLen || written < maxLen) && (afw = to.availableForWrite())) + while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) { - size_t afr = available(); - if (afw > afr) - afw = afr; - if (!afw) + size_t r = available(); + if (w > r) + w = r; + if (!w) return written; - if (afw > MAXTRANSFERBLOCK) - afw = MAXTRANSFERBLOCK; - char temp[afw]; - size_t r = read(temp, afw); - size_t w = write(temp, afw); - assert(r == w); + if (w > MAXTRANSFERBLOCK) + w = MAXTRANSFERBLOCK; + char temp[w]; + r = read(temp, w); + w = to.write(temp, r); + assert(w == r); written += w; } return written; diff --git a/cores/esp8266/debug.h b/cores/esp8266/debug.h index 0a804e76b2..0a4571771d 100644 --- a/cores/esp8266/debug.h +++ b/cores/esp8266/debug.h @@ -5,7 +5,7 @@ #include #ifdef DEBUG_ESP_CORE -#define DEBUGV(...) ets_printf(__VA_ARGS__) +#define DEBUGV(...) ::printf(__VA_ARGS__) #endif #ifndef DEBUGV @@ -25,6 +25,19 @@ extern "C" { void __panic_func(const char* file, int line, const char* func) __attribute__((noreturn)); #define panic() __panic_func(PSTR(__FILE__), __LINE__, __func__) +#ifdef DEBUG_ESP_CORE +#define IAMSLOW(str) \ + do { \ + static bool once = false;\ + if (!once) { \ + once = true; \ + DEBUGV((PGM_P)PSTR(str " should be overridden for better efficiency\r\n")); \ + } \ + } while (0) +#else +#define IAMSLOW(str) do { (void)0; } while (0) +#endif + #ifdef __cplusplus } #endif From 6d6d57d27ad057c1b8d05af456fe7c44aa814359 Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Thu, 31 Jan 2019 01:47:39 +0100 Subject: [PATCH 003/207] host emulation: make millis() starts at 0 --- tests/host/common/Arduino.cpp | 12 ++++++++++-- tests/host/common/Arduino.h | 1 + tests/host/common/ArduinoMain.cpp | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/host/common/Arduino.cpp b/tests/host/common/Arduino.cpp index 5f173646a2..fe145ffbeb 100644 --- a/tests/host/common/Arduino.cpp +++ b/tests/host/common/Arduino.cpp @@ -18,18 +18,26 @@ #include +unsigned long millis0 = 0, micros0 = 0; + +extern "C" void init_milliscros () +{ + millis0 = millis(); + micros0 = micros(); +} + extern "C" unsigned long millis() { timeval time; gettimeofday(&time, NULL); - return (time.tv_sec * 1000) + (time.tv_usec / 1000); + return (time.tv_sec * 1000) + (time.tv_usec / 1000) - millis0; } extern "C" unsigned long micros() { timeval time; gettimeofday(&time, NULL); - return (time.tv_sec * 1000000) + time.tv_usec; + return (time.tv_sec * 1000000) + time.tv_usec - micros0; } diff --git a/tests/host/common/Arduino.h b/tests/host/common/Arduino.h index a1aba082c8..95153a349b 100644 --- a/tests/host/common/Arduino.h +++ b/tests/host/common/Arduino.h @@ -202,6 +202,7 @@ extern "C" { void analogWriteFreq(uint32_t freq); void analogWriteRange(uint32_t range); + void init_milliscros (); unsigned long millis(void); unsigned long micros(void); void delay(unsigned long); diff --git a/tests/host/common/ArduinoMain.cpp b/tests/host/common/ArduinoMain.cpp index 54420d9e2a..ef85c6bec8 100644 --- a/tests/host/common/ArduinoMain.cpp +++ b/tests/host/common/ArduinoMain.cpp @@ -127,7 +127,7 @@ int main (int argc, char* const argv []) // setup global global_ipv4_netfmt wifi_get_ip_info(0, nullptr); - + init_milliscros(); setup(); while (!user_exit) { From 7b459384cead19d69772fa8fc1556d5d7256cd29 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Thu, 31 Jan 2019 01:50:57 +0100 Subject: [PATCH 004/207] PolledTimeout: add time policy based on CPU cycles --- cores/esp8266/PolledTimeout.h | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/cores/esp8266/PolledTimeout.h b/cores/esp8266/PolledTimeout.h index d13bae200d..d5b333bfe9 100644 --- a/cores/esp8266/PolledTimeout.h +++ b/cores/esp8266/PolledTimeout.h @@ -47,15 +47,31 @@ struct YieldOrSkip } //YieldPolicy +namespace TimePolicy +{ + +struct TimeMillis +{ + static decltype(millis()) time() {return millis();} + static constexpr decltype(millis()) toMillis = 1; +}; + +struct TimeCycle +{ + static decltype(ESP.getCycleCount()) time() {return ESP.getCycleCount();} + static constexpr decltype(ESP.getCycleCount()) toMillis = F_CPU / 1000; +}; + +} //TimePolicy -template +template class timeoutTemplate { public: - using timeType = decltype(millis()); + using timeType = decltype(TimePolicyT::time()); timeoutTemplate(timeType timeout) - : _timeout(timeout), _start(millis()) + : _timeout(timeout * TimePolicyT::toMillis), _start(TimePolicyT::time()) {} bool expired() @@ -79,7 +95,7 @@ class timeoutTemplate void reset() { - _start = millis(); + _start = TimePolicyT::time(); } timeType getTimeout() const @@ -96,7 +112,7 @@ class timeoutTemplate bool expiredRetrigger() { - timeType current = millis(); + timeType current = TimePolicyT::time(); if(checkExpired(current)) { unsigned long n = (current - _start) / _timeout; //how many _timeouts periods have elapsed, will usually be 1 (current - _start >= _timeout) @@ -108,7 +124,7 @@ class timeoutTemplate bool expiredOneShot() const { - return checkExpired(millis()); + return checkExpired(TimePolicyT::time()); } timeType _timeout; @@ -117,6 +133,7 @@ class timeoutTemplate using oneShot = polledTimeout::timeoutTemplate; using periodic = polledTimeout::timeoutTemplate; +using periodicCycle = polledTimeout::timeoutTemplate; } //polledTimeout From ad264b1c042d4f90e8fc4fab6c36cf66c9b8ebd9 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Thu, 31 Jan 2019 02:05:32 +0100 Subject: [PATCH 005/207] yield in streamTo loop --- cores/esp8266/Stream.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index d31aa7ebd3..b2120f2d36 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -18,11 +18,14 @@ Created July 2011 parsing functions based on TextFinder library by Michael Margolis + + streamTo additions 2019/01 - David Gauchard */ #include #include #include +#include #define PARSE_TIMEOUT 1000 // default number of milli-seconds to wait #define NO_SKIP_CHAR 1 // a magic char not found in a valid ASCII numeric field @@ -271,6 +274,7 @@ size_t Stream::read (char* buffer, size_t maxLen) size_t Stream::streamTo (Print& to, size_t maxLen) { + esp8266::polledTimeout::periodicCycle yieldNow(100); size_t w; size_t written = 0; while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) @@ -287,6 +291,9 @@ size_t Stream::streamTo (Print& to, size_t maxLen) w = to.write(temp, r); assert(w == r); written += w; + + if (yieldNow) + yield(); } return written; } From 8365dba5b4e34efd33db59eace45b7f38d9f1589 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Thu, 31 Jan 2019 02:23:12 +0100 Subject: [PATCH 006/207] yet another circular buffer for stream transfers (may not be kept) --- cores/esp8266/CircularBuffer.h | 187 +++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 cores/esp8266/CircularBuffer.h diff --git a/cores/esp8266/CircularBuffer.h b/cores/esp8266/CircularBuffer.h new file mode 100644 index 0000000000..47e80a97f2 --- /dev/null +++ b/cores/esp8266/CircularBuffer.h @@ -0,0 +1,187 @@ +/* + CircularBuffer.h - yet another circular buffer, + compatible with Arduino Stream transfer API + Copyright (c) 2019 david gauchard. All right reserved. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include + +// CONSTst (CONST-for-stream) should be 'const', +// Stream&Print should be updated accordingly (or Ardui-not). +#define CONSTst //const + +template +class CircularBuffer: public Stream +{ +protected: + + idx_t _posr, _posw; + uint8_t _buffer[_len]; + bool _full; + + void inc_posr (idx_t contig = 1) + { + if ((_posr += contig) == _len) + _posr = 0; + _full = false; + } + + void inc_posw (idx_t contig = 1) + { + if ((_posw += contig) == _len) + _posw = 0; + if (_posw == _posr) + _full = true; + } + + idx_t availableForWriteContig () const + { + if (full()) + return 0; + if (_posr > _posw) + return _posr - _posw; + return _len - _posw; + } + + idx_t availableForReadContig () const + { + if (full()) + return _len; + if (_posr > _posw) + return _len - _posr; + return _posw - _posr; + } + +public: + + CircularBuffer (): _posr(0), _posw(0), _full(false) { } + + size_t length () const + { + return _len; + } + + bool full () const + { + return _full; + } + + bool empty () const + { + return _posw == _posr && !_full; + } + + int availableForWrite () CONSTst override + { + if (empty()) + return _len; + if (_posw > _posr) + return (_len - _posw) + (_posr - 0); + return _posr - _posw; + } + + int available () CONSTst override + { + if (full()) + return _len; + if (_posr > _posw) + return (_len - _posr) + (_posw - 0); + return _posw - _posr; + } + + // char/byte oriented + + int peek () CONSTst override + { + return empty()? -1 : _buffer[_posr]; + } + + int read () override + { + if (empty()) + return -1; + int c = _buffer[_posr]; + inc_posr(); + return c; + } + + size_t write (uint8_t c) override + { + if (full()) + return 0; + _buffer[_posw] = c; + inc_posw(); + return 1; + } + + // buffer oriented + + size_t peek (uint8_t* buffer, size_t len) const /*override*/ + { + if (empty()) + return 0; + idx_t chunk1 = availableForReadContig(); + if (chunk1 > len) + chunk1 = len; + memcpy(buffer, &_buffer[_posr], chunk1); + return chunk1; + } + + size_t peekMaxSize () const + { + return availableForReadContig(); + } + + const uint8_t* peekPtr () const + { + return &_buffer[_posr]; + } + + size_t read (char* buffer, size_t maxLen) override + { + idx_t copied = 0; + while (copied < maxLen) + { + idx_t contig = availableForReadContig(); + if (!contig) + break; + if (contig > maxLen - copied) + contig = maxLen - copied; + memcpy(buffer + copied, &_buffer[_posr], contig); + copied += contig; + inc_posr(contig); + } + return copied; + } + + size_t write (const uint8_t* buffer, size_t maxLen) override + { + idx_t copied = 0; + while (copied < maxLen) + { + idx_t contig = availableForWriteContig(); + if (!contig) + break; + if (contig > maxLen - copied) + contig = maxLen - copied; + memcpy(&_buffer[_posw], buffer + copied, contig); + copied += contig; + inc_posw(contig); + } + return copied; + } +}; From 9b934cb9051d2c62c99ad65576aa09621a5dd952 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Thu, 31 Jan 2019 02:37:18 +0100 Subject: [PATCH 007/207] member name decoration --- cores/esp8266/CircularBuffer.h | 86 +++++++++++++++++----------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/cores/esp8266/CircularBuffer.h b/cores/esp8266/CircularBuffer.h index 47e80a97f2..fe94b29380 100644 --- a/cores/esp8266/CircularBuffer.h +++ b/cores/esp8266/CircularBuffer.h @@ -24,112 +24,112 @@ // Stream&Print should be updated accordingly (or Ardui-not). #define CONSTst //const -template +template class CircularBuffer: public Stream { protected: - idx_t _posr, _posw; - uint8_t _buffer[_len]; - bool _full; - + idx_t m_posr, m_posw; + uint8_t m_buffer[m_len]; + bool m_full; + void inc_posr (idx_t contig = 1) { - if ((_posr += contig) == _len) - _posr = 0; - _full = false; + if ((m_posr += contig) == m_len) + m_posr = 0; + m_full = false; } void inc_posw (idx_t contig = 1) { - if ((_posw += contig) == _len) - _posw = 0; - if (_posw == _posr) - _full = true; + if ((m_posw += contig) == m_len) + m_posw = 0; + if (m_posw == m_posr) + m_full = true; } idx_t availableForWriteContig () const { if (full()) return 0; - if (_posr > _posw) - return _posr - _posw; - return _len - _posw; + if (m_posr > m_posw) + return m_posr - m_posw; + return m_len - m_posw; } idx_t availableForReadContig () const { if (full()) - return _len; - if (_posr > _posw) - return _len - _posr; - return _posw - _posr; + return m_len; + if (m_posr > m_posw) + return m_len - m_posr; + return m_posw - m_posr; } public: - CircularBuffer (): _posr(0), _posw(0), _full(false) { } + CircularBuffer (): m_posr(0), m_posw(0), m_full(false) { } size_t length () const { - return _len; + return m_len; } bool full () const { - return _full; + return m_full; } bool empty () const { - return _posw == _posr && !_full; + return m_posw == m_posr && !m_full; } int availableForWrite () CONSTst override { if (empty()) - return _len; - if (_posw > _posr) - return (_len - _posw) + (_posr - 0); - return _posr - _posw; + return m_len; + if (m_posw > m_posr) + return (m_len - m_posw) + (m_posr - 0); + return m_posr - m_posw; } - + int available () CONSTst override { if (full()) - return _len; - if (_posr > _posw) - return (_len - _posr) + (_posw - 0); - return _posw - _posr; + return m_len; + if (m_posr > m_posw) + return (m_len - m_posr) + (m_posw - 0); + return m_posw - m_posr; } // char/byte oriented int peek () CONSTst override { - return empty()? -1 : _buffer[_posr]; + return empty()? -1 : m_buffer[m_posr]; } int read () override { if (empty()) return -1; - int c = _buffer[_posr]; + int c = m_buffer[m_posr]; inc_posr(); return c; } - + size_t write (uint8_t c) override { if (full()) return 0; - _buffer[_posw] = c; + m_buffer[m_posw] = c; inc_posw(); return 1; } // buffer oriented - + size_t peek (uint8_t* buffer, size_t len) const /*override*/ { if (empty()) @@ -137,18 +137,18 @@ class CircularBuffer: public Stream idx_t chunk1 = availableForReadContig(); if (chunk1 > len) chunk1 = len; - memcpy(buffer, &_buffer[_posr], chunk1); + memcpy(buffer, &m_buffer[m_posr], chunk1); return chunk1; } - + size_t peekMaxSize () const { return availableForReadContig(); } - + const uint8_t* peekPtr () const { - return &_buffer[_posr]; + return &m_buffer[m_posr]; } size_t read (char* buffer, size_t maxLen) override @@ -161,7 +161,7 @@ class CircularBuffer: public Stream break; if (contig > maxLen - copied) contig = maxLen - copied; - memcpy(buffer + copied, &_buffer[_posr], contig); + memcpy(buffer + copied, &m_buffer[m_posr], contig); copied += contig; inc_posr(contig); } @@ -178,7 +178,7 @@ class CircularBuffer: public Stream break; if (contig > maxLen - copied) contig = maxLen - copied; - memcpy(&_buffer[_posw], buffer + copied, contig); + memcpy(&m_buffer[m_posw], buffer + copied, contig); copied += contig; inc_posw(contig); } From b82fa0e50133e09e52b91c905b996679faf55536 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Thu, 31 Jan 2019 02:38:01 +0100 Subject: [PATCH 008/207] cpu cycles adaptation for emulation on host --- cores/esp8266/PolledTimeout.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cores/esp8266/PolledTimeout.h b/cores/esp8266/PolledTimeout.h index d5b333bfe9..dfcd3b27bc 100644 --- a/cores/esp8266/PolledTimeout.h +++ b/cores/esp8266/PolledTimeout.h @@ -56,12 +56,19 @@ struct TimeMillis static constexpr decltype(millis()) toMillis = 1; }; +#ifdef CORE_MOCK +struct TimeCycle: public TimeMillis {}; +#else + struct TimeCycle { static decltype(ESP.getCycleCount()) time() {return ESP.getCycleCount();} static constexpr decltype(ESP.getCycleCount()) toMillis = F_CPU / 1000; }; +#endif // cpu cycles + + } //TimePolicy template From 24b5d99d93cb5fbaf85f91d99f321ffff5603417 Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Thu, 31 Jan 2019 15:03:08 +0100 Subject: [PATCH 009/207] emulation script to run sketches: official example FSBrowser (SERVE/DOWNLOAD) and Http Client Poster (UPLOAD) --- tests/host/emutest-FSBrowser+ClientPoster.sh | 80 +++++++++++++++++++ .../HttpClientPost/HttpClientPost.ino | 77 ++++++++++++++++++ 2 files changed, 157 insertions(+) create mode 100755 tests/host/emutest-FSBrowser+ClientPoster.sh create mode 100644 tests/host/otherexamples/HttpClientPost/HttpClientPost.ino diff --git a/tests/host/emutest-FSBrowser+ClientPoster.sh b/tests/host/emutest-FSBrowser+ClientPoster.sh new file mode 100755 index 0000000000..d88c37cc65 --- /dev/null +++ b/tests/host/emutest-FSBrowser+ClientPoster.sh @@ -0,0 +1,80 @@ +#!/bin/sh + +# Purpose: debugging, timing, optimizing the core data transfers (Stream, FS, network) +# +# instructions: simply run this script +# +# what it does: +# +# Run FSbrowser official example +# use it to upload a random (big) file to emulated SPIFFS +# Stop it, copy the emulated SPIFFS file for another HTttpClientPost sketch +# Restart the Server +# Start the Poster, it will upload a copy to the server +# end + +set -e + +sizekb=16000 # SPIFFS SIZE in KBytes (file size will be the third of this value) +DEBUG="D=1" # comment this or not + +sleep=5 +sizeb=$(($sizekb * 1024)) +fileb=$(($sizeb / 3)) + +make $DEBUG ../../libraries/ESP8266WebServer/examples/FSBrowser/FSBrowser +make $DEBUG otherexamples/HttpClientPost/HttpClientPost + +killall -9 FSBrowser HttpClientPost || true + +rm -f FSBrowser-spiffs${sizekb}KB +./bin/FSBrowser/FSBrowser -S ${sizekb} -f & PIDFSBrowser=$! + +echo "------------------------------" +echo " let server start..." +echo "------------------------------" + +# make a big random file +dd if=/dev/urandom of=randfile bs=${fileb} count=1 + +sleep ${sleep} + +echo "------------------------------" +echo " Uploading $file to SPIFFS" +echo "------------------------------" +curl -F "file=@$PWD/randfile" 127.0.0.1:9080/edit +echo "------------------------------" +echo " Uploading to SPIFFS (done)" +echo "------------------------------" + +kill -INT $PIDFSBrowser + +# FSBrowser has generated SPIFFS backup file, copy it for the other sketch +# This sketch will repost this file to the FSBrowser, so we can debug/tune + +# copy SPIFFS to http client sketch +cp ./bin/FSBrowser/FSBrowser-spiffs${sizekb}KB ./bin/HttpClientPost/HttpClientPost-spiffs${sizekb}KB +ls -al ./bin/FSBrowser/FSBrowser-spiffs${sizekb}KB ./bin/HttpClientPost/HttpClientPost-spiffs${sizekb}KB + +echo "------------------------------" +echo " let server start again..." +echo "------------------------------" +./bin/FSBrowser/FSBrowser -S ${sizekb} -f & PIDFSBrowser=$! +sleep ${sleep} + +echo "------------------------------" +echo " start uploader sketch" +echo "------------------------------" +# run http client poster sketch +./bin/HttpClientPost/HttpClientPost -S ${sizekb} -f & PIDClient=$! + +echo "------------------------------" +echo "Let run the sketches, press enter to kill" +echo "------------------------------" +read junk + +echo "killing everybody" +kill -INT $PIDClient $PIDFSBrowser || true +sleep 2 +echo "hardkilling everybody" +kill -9 $PIDClient $PIDFSBrowser diff --git a/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino b/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino new file mode 100644 index 0000000000..5d5b81e27b --- /dev/null +++ b/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino @@ -0,0 +1,77 @@ + +// used by ../../test.sh + +#include +#include +#include + +#define FILE "/randfile" +#define URL "http://127.0.0.1:9080/edit" + +#ifndef STASSID +#define STASSID "your-ssid" +#define STAPSK "your-password" +#endif + +WiFiClient client; + +void setup() { + Serial.begin(115200); + Serial.println("starting..."); + WiFi.begin(STASSID, STAPSK); + while (!WiFi.isConnected()) { + Serial.println("."); + delay(250); + } + + if (!SPIFFS.begin()) + Serial.println("CLIENT: failed to start SPIFFS!"); + + auto f = SPIFFS.open(FILE, "r"); + const auto f_size = f.size(); + Serial.printf("CLIENT: size to upload: %d\n", (int)f_size); + + Serial.println("CLIENT: connecting to server..."); + if (!client.connect("127.0.0.1", 9080)) { + Serial.println("\nfailed to connect\n"); + return; + } + + client.println("POST /edit HTTP/1.1"); + client.println("Host: 127.0.0.1"); + client.println("Connection: close"); + client.println("Content-Type: multipart/form-data; boundary=glark"); + client.printf("Content-Length: %d\r\n", f_size); + client.println(); + client.println("--glark"); + client.printf("Content-Disposition: form-data; name=\"%s\"; filename=\"%s\"\r\n", FILE "copy", FILE "copy"); + client.println("Content-Type: application/octet-stream"); + client.println(""); + + Serial.printf("\r\n\r\n"); + Serial.printf("\r\n\r\n##########################\r\n"); + auto A = millis(); + Serial.printf("\r\n\r\nCLIENT: ----> time-keep: %lums sent: %d\r\n\r\n", (millis() - A), (int)client.write(f)); + client.println(); + client.println("--glark--"); + A = millis(); + Serial.println("CLIENT waiting for server ack..."); + while (!client.available()) + yield(); + A = millis() - A; + Serial.printf("\r\n\r\n##########################\r\n"); + Serial.printf("\r\n\r\nCLIENT: ----> server response: +%lums\r\n", A); + Serial.printf("\r\n\r\n##########################\r\n"); + + Serial.println("CLIENT: server response:"); + while (client.available()) + Serial.write(client.read()); + + Serial.println("CLIENT: end"); + client.stop(); + f.close(); +} + +void loop () +{ +} From e32f37eec4622fd75acf0350e1bd4e54d6d95a56 Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Thu, 31 Jan 2019 15:05:31 +0100 Subject: [PATCH 010/207] mock tcp send: fix for large data --- tests/host/common/ClientContextSocket.cpp | 46 ++++++++++++----------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/tests/host/common/ClientContextSocket.cpp b/tests/host/common/ClientContextSocket.cpp index 2bee9fa1cf..af19122eb1 100644 --- a/tests/host/common/ClientContextSocket.cpp +++ b/tests/host/common/ClientContextSocket.cpp @@ -145,32 +145,36 @@ ssize_t mockRead (int sock, char* dst, size_t size, int timeout_ms, char* ccinbu ssize_t mockWrite (int sock, const uint8_t* data, size_t size, int timeout_ms) { - struct pollfd p; - p.fd = sock; - p.events = POLLOUT; - int ret = poll(&p, 1, timeout_ms); - if (ret == -1) - { - fprintf(stderr, MOCK "ClientContext::write: poll(%d): %s\n", sock, strerror(errno)); - return 0; - } - if (ret) + size_t sent = 0; + while (sent < size) { -#ifndef MSG_NOSIGNAL - ret = ::write(sock, data, size); -#else - ret = ::send(sock, data, size, MSG_NOSIGNAL); -#endif + + struct pollfd p; + p.fd = sock; + p.events = POLLOUT; + int ret = poll(&p, 1, timeout_ms); if (ret == -1) { - fprintf(stderr, MOCK "ClientContext::read: write(%d): %s\n", sock, strerror(errno)); - return -1; + fprintf(stderr, MOCK "ClientContext::write: poll(%d): %s\n", sock, strerror(errno)); + return 0; } - if (ret != (int)size) + if (ret) { - fprintf(stderr, MOCK "ClientContext::write: short write (%d < %zd) (TODO)\n", ret, size); - exit(EXIT_FAILURE); +#ifndef MSG_NOSIGNAL + ret = ::write(sock, data + sent, size - sent); +#else + ret = ::send(sock, data + sent, size - sent, MSG_NOSIGNAL); +#endif + if (ret == -1) + { + fprintf(stderr, MOCK "ClientContext::read: write(%d): %s\n", sock, strerror(errno)); + return -1; + } + sent += ret; + if (sent < size) + fprintf(stderr, MOCK "ClientContext::write: sent %d bytes (%zd / %zd)\n", ret, sent, size); } } - return ret; + fprintf(stderr, MOCK "ClientContext::write: total sent %zd bytes\n", sent); + return sent; } From c717d2a619ea50c468dd5424d6acc461ec7a9e94 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 5 Feb 2019 00:20:02 +0100 Subject: [PATCH 011/207] wip (peekBuffer) --- cores/esp8266/CircularBuffer.h | 85 ++++++++++++++-------------------- cores/esp8266/PolledTimeout.h | 10 ++-- cores/esp8266/Stream.cpp | 27 +++++++++-- cores/esp8266/Stream.h | 7 +++ cores/esp8266/debug.h | 2 +- 5 files changed, 73 insertions(+), 58 deletions(-) diff --git a/cores/esp8266/CircularBuffer.h b/cores/esp8266/CircularBuffer.h index fe94b29380..4f3bc19d0b 100644 --- a/cores/esp8266/CircularBuffer.h +++ b/cores/esp8266/CircularBuffer.h @@ -48,24 +48,6 @@ class CircularBuffer: public Stream m_full = true; } - idx_t availableForWriteContig () const - { - if (full()) - return 0; - if (m_posr > m_posw) - return m_posr - m_posw; - return m_len - m_posw; - } - - idx_t availableForReadContig () const - { - if (full()) - return m_len; - if (m_posr > m_posw) - return m_len - m_posr; - return m_posw - m_posr; - } - public: CircularBuffer (): m_posr(0), m_posw(0), m_full(false) { } @@ -85,24 +67,6 @@ class CircularBuffer: public Stream return m_posw == m_posr && !m_full; } - int availableForWrite () CONSTst override - { - if (empty()) - return m_len; - if (m_posw > m_posr) - return (m_len - m_posw) + (m_posr - 0); - return m_posr - m_posw; - } - - int available () CONSTst override - { - if (full()) - return m_len; - if (m_posr > m_posw) - return (m_len - m_posr) + (m_posw - 0); - return m_posw - m_posr; - } - // char/byte oriented int peek () CONSTst override @@ -128,35 +92,43 @@ class CircularBuffer: public Stream return 1; } + idx_t availableForWrite () CONSTst override + { + if (full()) + return 0; + if (m_posr > m_posw) + return m_posr - m_posw; + return m_len - m_posw; + } + + idx_t available () CONSTst override + { + if (full()) + return m_len; + if (m_posr > m_posw) + return m_len - m_posr; + return m_posw - m_posr; + } + // buffer oriented size_t peek (uint8_t* buffer, size_t len) const /*override*/ { if (empty()) return 0; - idx_t chunk1 = availableForReadContig(); + idx_t chunk1 = availableForRead(); if (chunk1 > len) chunk1 = len; memcpy(buffer, &m_buffer[m_posr], chunk1); return chunk1; } - size_t peekMaxSize () const - { - return availableForReadContig(); - } - - const uint8_t* peekPtr () const - { - return &m_buffer[m_posr]; - } - size_t read (char* buffer, size_t maxLen) override { idx_t copied = 0; while (copied < maxLen) { - idx_t contig = availableForReadContig(); + idx_t contig = availableForRead(); if (!contig) break; if (contig > maxLen - copied) @@ -173,7 +145,7 @@ class CircularBuffer: public Stream idx_t copied = 0; while (copied < maxLen) { - idx_t contig = availableForWriteContig(); + idx_t contig = availableForWrite(); if (!contig) break; if (contig > maxLen - copied) @@ -184,4 +156,19 @@ class CircularBuffer: public Stream } return copied; } + + virtual const char* peekBuffer () override + { + return &m_buffer[m_posr]; + } + + virtual void peekConsume (size_t consume) override + { +#ifdef DEBUG_ESP_CORE + if (consume > available()) + DEBUGV("consume overflow\r\n"); +#endif + inc_posr(consume); + } + }; diff --git a/cores/esp8266/PolledTimeout.h b/cores/esp8266/PolledTimeout.h index dfcd3b27bc..6e238a0686 100644 --- a/cores/esp8266/PolledTimeout.h +++ b/cores/esp8266/PolledTimeout.h @@ -138,9 +138,13 @@ class timeoutTemplate timeType _start; }; -using oneShot = polledTimeout::timeoutTemplate; -using periodic = polledTimeout::timeoutTemplate; -using periodicCycle = polledTimeout::timeoutTemplate; +using oneShotMs = polledTimeout::timeoutTemplate; +using periodicMs = polledTimeout::timeoutTemplate; + +using oneShot = polledTimeout::timeoutTemplate; +using periodic = polledTimeout::timeoutTemplate; +using oneShotYield = polledTimeout::timeoutTemplate; +using periodicYield = polledTimeout::timeoutTemplate; } //polledTimeout diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index b2120f2d36..fd2ee88c2c 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -274,7 +274,7 @@ size_t Stream::read (char* buffer, size_t maxLen) size_t Stream::streamTo (Print& to, size_t maxLen) { - esp8266::polledTimeout::periodicCycle yieldNow(100); + esp8266::polledTimeout::periodic yieldNow(100); size_t w; size_t written = 0; while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) @@ -286,10 +286,21 @@ size_t Stream::streamTo (Print& to, size_t maxLen) return written; if (w > MAXTRANSFERBLOCK) w = MAXTRANSFERBLOCK; - char temp[w]; - r = read(temp, w); - w = to.write(temp, r); - assert(w == r); + + const char* pb = peekBuffer(); + if (pb) + { + w = to.write(pb, w); + peekConsume(w); + } + else + { + char temp[w]; + r = read(temp, w); + w = to.write(temp, r); + assert(w == r); + } + written += w; if (yieldNow) @@ -297,3 +308,9 @@ size_t Stream::streamTo (Print& to, size_t maxLen) } return written; } + +void Stream::peekConsume (size_t consume) +{ + (void)consume; + panic(); // invalid use (peekBuffer() is nullptr) +} diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index efedd5f221..5ccc431535 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -115,6 +115,13 @@ class Stream: public Print { // immediate return when no more data are available (no timeout) virtual size_t streamTo (Print& to, size_t maxLen = 0); + // return a pointer to available data buffer (size = available()) + // semantic forbids any kind of read() before calling peekConsume() + virtual const char* peekBuffer () { return nullptr; } + + // consume bytes after use (see peekBuffer) + virtual void peekConsume (size_t consume); + protected: long parseInt(char skipChar); // as above but the given skipChar is ignored // as above but the given skipChar is ignored diff --git a/cores/esp8266/debug.h b/cores/esp8266/debug.h index 0a4571771d..f84d351d3c 100644 --- a/cores/esp8266/debug.h +++ b/cores/esp8266/debug.h @@ -9,7 +9,7 @@ #endif #ifndef DEBUGV -#define DEBUGV(...) +#define DEBUGV(...) do { (void)0; } while (0) #endif #ifdef __cplusplus From 85b703100848797746832eb7b1277fb4707dc1c6 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 24 Mar 2019 00:39:02 +0100 Subject: [PATCH 012/207] peek in hardwareserial --- cores/esp8266/HardwareSerial.h | 20 ++++++++++++++++++++ cores/esp8266/Stream.cpp | 8 +++----- cores/esp8266/Stream.h | 5 ++++- cores/esp8266/debug.cpp | 2 ++ cores/esp8266/debug.h | 3 ++- cores/esp8266/uart.cpp | 28 ++++++++++++++++++++++++++++ cores/esp8266/uart.h | 9 +++++++++ 7 files changed, 68 insertions(+), 7 deletions(-) diff --git a/cores/esp8266/HardwareSerial.h b/cores/esp8266/HardwareSerial.h index 0032c19ad8..155744207e 100644 --- a/cores/esp8266/HardwareSerial.h +++ b/cores/esp8266/HardwareSerial.h @@ -128,6 +128,26 @@ class HardwareSerial: public Stream // return -1 when data is unvailable (arduino api) return uart_peek_char(_uart); } + + // return number of byte accessible by peekBuffer() + size_t peekAvailable () override + { + return uart_peek_available(_uart); + } + + // return a pointer to available data buffer (size = available()) + // semantic forbids any kind of read() before calling peekConsume() + const char* peekBuffer () override + { + return uart_peek_buffer(_uart); + } + + // consume bytes after use (see peekBuffer) + void peekConsume (size_t consume) override; + { + return uart_peek_consume(_uart, consume); + } + int read(void) override { // return -1 when data is unvailable (arduino api) diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index fd2ee88c2c..377f741f06 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -18,8 +18,6 @@ Created July 2011 parsing functions based on TextFinder library by Michael Margolis - - streamTo additions 2019/01 - David Gauchard */ #include @@ -284,9 +282,6 @@ size_t Stream::streamTo (Print& to, size_t maxLen) w = r; if (!w) return written; - if (w > MAXTRANSFERBLOCK) - w = MAXTRANSFERBLOCK; - const char* pb = peekBuffer(); if (pb) { @@ -295,6 +290,9 @@ size_t Stream::streamTo (Print& to, size_t maxLen) } else { + if (w > MAXTRANSFERBLOCK) + w = MAXTRANSFERBLOCK; + char temp[w]; r = read(temp, w); w = to.write(temp, r); diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 5ccc431535..8fcd745328 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -115,10 +115,13 @@ class Stream: public Print { // immediate return when no more data are available (no timeout) virtual size_t streamTo (Print& to, size_t maxLen = 0); - // return a pointer to available data buffer (size = available()) + // return a pointer to available data buffer (size = peekAvailable()) // semantic forbids any kind of read() before calling peekConsume() virtual const char* peekBuffer () { return nullptr; } + // return number of byte accessible by peekBuffer() + virtual size_t peekAvailable () { return 0; } + // consume bytes after use (see peekBuffer) virtual void peekConsume (size_t consume); diff --git a/cores/esp8266/debug.cpp b/cores/esp8266/debug.cpp index d8bf8bb11d..736ee69335 100644 --- a/cores/esp8266/debug.cpp +++ b/cores/esp8266/debug.cpp @@ -21,6 +21,8 @@ #include "Arduino.h" #include "debug.h" +const char* overrideme PROGMEM = " should be overridden for better efficiency\r\n"; + void ICACHE_RAM_ATTR hexdump(const void *mem, uint32_t len, uint8_t cols) { const uint8_t* src = (const uint8_t*) mem; os_printf("\n[HEXDUMP] Address: 0x%08X len: 0x%X (%d)", (ptrdiff_t)src, len, len); diff --git a/cores/esp8266/debug.h b/cores/esp8266/debug.h index 78bd82025d..02b30a4c61 100644 --- a/cores/esp8266/debug.h +++ b/cores/esp8266/debug.h @@ -26,12 +26,13 @@ void __panic_func(const char* file, int line, const char* func) __attribute__((n #define panic() __panic_func(PSTR(__FILE__), __LINE__, __func__) #ifdef DEBUG_ESP_CORE +extern const char* overrideme PROGMEM; #define IAMSLOW(str) \ do { \ static bool once = false;\ if (!once) { \ once = true; \ - DEBUGV((PGM_P)PSTR(str " should be overridden for better efficiency\r\n")); \ + DEBUGV((PGM_P)PSTR(str "%s", overrideme)); \ } \ } while (0) #else diff --git a/cores/esp8266/uart.cpp b/cores/esp8266/uart.cpp index 3a27ff62f4..d55b280b81 100644 --- a/cores/esp8266/uart.cpp +++ b/cores/esp8266/uart.cpp @@ -229,6 +229,34 @@ uart_peek_char(uart_t* uart) return ret; } +// return number of byte accessible by uart_peek_buffer() +size_t uart_peek_available (uart_t* uart) +{ + ETS_UART_INTR_DISABLE(); + uart_rx_copy_fifo_to_buffer_unsafe(uart); + ETS_UART_INTR_ENABLE(); + if(rx_buffer->wpos < rx_buffer->rpos) + return rx_buffer->size - rx_buffer->rpos; + return rx_buffer->wpos - rx_buffer->rpos; +} + +// return a pointer to available data buffer (size = available()) +// semantic forbids any kind of read() before calling peekConsume() +const char* uart_peek_buffer (uart_t* uart) +{ + return &uart->rx_buffer->buffer[uart->rx_buffer->rpos]; +} + +// consume bytes after use (see uart_peek_buffer) +void uart_peek_consume (uart_t* uart, size_t consume) +{ + ETS_UART_INTR_DISABLE(); + rx_buffer->rpos += consume; + if (rx_buffer->rpos >= rx_buffer->size) + rx_buffer->rpos -= rx_buffer->size; + ETS_UART_INTR_ENABLE(); +} + int uart_read_char(uart_t* uart) { diff --git a/cores/esp8266/uart.h b/cores/esp8266/uart.h index 7f9dce0f0a..823bc59fbd 100644 --- a/cores/esp8266/uart.h +++ b/cores/esp8266/uart.h @@ -147,6 +147,15 @@ int uart_get_debug(); void uart_start_detect_baudrate(int uart_nr); int uart_detect_baudrate(int uart_nr); +// return number of byte accessible by peekBuffer() +size_t uart_peek_available (uart_t* uart); + +// return a pointer to available data buffer (size = available()) +// semantic forbids any kind of read() before calling peekConsume() +const char* uart_peek_buffer (uart_t* uart); + +// consume bytes after use (see peekBuffer) +void uart_peek_consume (uart_t* uart, size_t consume); #if defined (__cplusplus) } // extern "C" From 7e883c8241022922e7b9161daacebc0c7fb7fc8d Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 25 Mar 2019 00:49:52 +0100 Subject: [PATCH 013/207] peek-buffer with WiFiClient --- cores/esp8266/HardwareSerial.h | 12 +++++------ cores/esp8266/Stream.cpp | 6 ------ cores/esp8266/Stream.h | 2 +- libraries/ESP8266WiFi/src/WiFiClient.cpp | 20 +++++++++++++++++++ libraries/ESP8266WiFi/src/WiFiClient.h | 10 ++++++++++ .../ESP8266WiFi/src/WiFiClientSecureAxTLS.h | 10 ++++++++++ .../ESP8266WiFi/src/WiFiClientSecureBearSSL.h | 12 +++++++++++ .../ESP8266WiFi/src/include/ClientContext.h | 19 ++++++++++++++++++ 8 files changed, 78 insertions(+), 13 deletions(-) diff --git a/cores/esp8266/HardwareSerial.h b/cores/esp8266/HardwareSerial.h index 155744207e..cbfbd625e0 100644 --- a/cores/esp8266/HardwareSerial.h +++ b/cores/esp8266/HardwareSerial.h @@ -129,12 +129,6 @@ class HardwareSerial: public Stream return uart_peek_char(_uart); } - // return number of byte accessible by peekBuffer() - size_t peekAvailable () override - { - return uart_peek_available(_uart); - } - // return a pointer to available data buffer (size = available()) // semantic forbids any kind of read() before calling peekConsume() const char* peekBuffer () override @@ -142,6 +136,12 @@ class HardwareSerial: public Stream return uart_peek_buffer(_uart); } + // return number of byte accessible by peekBuffer() + size_t peekAvailable () override + { + return uart_peek_available(_uart); + } + // consume bytes after use (see peekBuffer) void peekConsume (size_t consume) override; { diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index 377f741f06..b42de0e579 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -306,9 +306,3 @@ size_t Stream::streamTo (Print& to, size_t maxLen) } return written; } - -void Stream::peekConsume (size_t consume) -{ - (void)consume; - panic(); // invalid use (peekBuffer() is nullptr) -} diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 8fcd745328..6b44fc9091 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -123,7 +123,7 @@ class Stream: public Print { virtual size_t peekAvailable () { return 0; } // consume bytes after use (see peekBuffer) - virtual void peekConsume (size_t consume); + virtual void peekConsume (size_t consume) { (void)consume; } protected: long parseInt(char skipChar); // as above but the given skipChar is ignored diff --git a/libraries/ESP8266WiFi/src/WiFiClient.cpp b/libraries/ESP8266WiFi/src/WiFiClient.cpp index b623fb2f98..60629ddaea 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.cpp +++ b/libraries/ESP8266WiFi/src/WiFiClient.cpp @@ -423,3 +423,23 @@ uint8_t WiFiClient::getKeepAliveCount () const { return _client->getKeepAliveCount(); } + + +// return a pointer to available data buffer (size = peekAvailable()) +// semantic forbids any kind of read() before calling peekConsume() +virtual const char* WiFiClient::peekBuffer () +{ + return _client->peekBuffer(); +} + +// return number of byte accessible by peekBuffer() +virtual size_t WiFiClient::peekAvailable () +{ + return _client->peekAvailale(); +} + +// consume bytes after use (see peekBuffer) +virtual void WiFiClient::peekConsume (size_t consume) +{ + return _client->peekConsume(consume); +} diff --git a/libraries/ESP8266WiFi/src/WiFiClient.h b/libraries/ESP8266WiFi/src/WiFiClient.h index 3860549133..71a6389f62 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.h +++ b/libraries/ESP8266WiFi/src/WiFiClient.h @@ -118,6 +118,16 @@ class WiFiClient : public Client, public SList { bool getSync() const; void setSync(bool sync); + // return number of byte accessible by peekBuffer() + virtual size_t peekAvailable () override; + + // return a pointer to available data buffer (size = peekAvailable()) + // semantic forbids any kind of read() before calling peekConsume() + virtual const char* peekBuffer () override; + + // consume bytes after use (see peekBuffer) + virtual void peekConsume (size_t consume) override; + protected: static int8_t _s_connected(void* arg, void* tpcb, int8_t err); diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h b/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h index b1ae65d57a..ee2473306b 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h @@ -82,6 +82,16 @@ class WiFiClientSecure : public WiFiClient { return loadCACert(file, file.size()); } + // return a pointer to available data buffer (size = peekAvailable()) + // semantic forbids any kind of read() before calling peekConsume() + virtual const char* peekBuffer () { return nullptr; } + + // return number of byte accessible by peekBuffer() + virtual size_t peekAvailable () { return 0; } + + // consume bytes after use (see peekBuffer) + virtual void peekConsume (size_t consume) {} + friend class WiFiServerSecure; // Needs access to custom constructor below protected: // Only called by WiFiServerSecure diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h index 0ac9003c78..8056033d5c 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h @@ -122,6 +122,18 @@ class WiFiClientSecure : public WiFiClient { static bool probeMaxFragmentLength(const char *hostname, uint16_t port, uint16_t len); static bool probeMaxFragmentLength(const String& host, uint16_t port, uint16_t len); + // peek blocks not implemented yet: + + // return a pointer to available data buffer (size = peekAvailable()) + // semantic forbids any kind of read() before calling peekConsume() + virtual const char* peekBuffer () { return nullptr; } + + // return number of byte accessible by peekBuffer() + virtual size_t peekAvailable () { return 0; } + + // consume bytes after use (see peekBuffer) + virtual void peekConsume (size_t consume) {} + //////////////////////////////////////////////////// // AxTLS API deprecated warnings to help upgrading diff --git a/libraries/ESP8266WiFi/src/include/ClientContext.h b/libraries/ESP8266WiFi/src/include/ClientContext.h index 5be690f247..2fb7b06578 100644 --- a/libraries/ESP8266WiFi/src/include/ClientContext.h +++ b/libraries/ESP8266WiFi/src/include/ClientContext.h @@ -420,6 +420,25 @@ class ClientContext _sync = sync; } + // return a pointer to available data buffer (size = peekAvailable()) + // semantic forbids any kind of read() before calling peekConsume() + const char* peekBuffer () + { + return _rx_buf->payload + _rx_buf_offset + } + + // return number of byte accessible by peekBuffer() + size_t peekAvailable () + { + return _rx_buf->tot_len - _rx_buf_offset; + } + + // consume bytes after use (see peekBuffer) + void peekConsume (size_t consume) + { + _consume(consume); + } + protected: bool _is_timeout() From b520efcfd3c07a8d9b5ded3bba903351c831970e Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 25 Mar 2019 22:24:35 +0100 Subject: [PATCH 014/207] wip --- cores/esp8266/HardwareSerial.h | 2 +- cores/esp8266/Stream.cpp | 16 ++++++++++------ cores/esp8266/uart.cpp | 14 +++++++------- libraries/ESP8266WiFi/src/WiFiClient.cpp | 8 ++++---- .../ESP8266WiFi/src/WiFiClientSecureAxTLS.h | 2 +- .../ESP8266WiFi/src/WiFiClientSecureBearSSL.h | 2 +- .../ESP8266WiFi/src/include/ClientContext.h | 6 +++++- 7 files changed, 29 insertions(+), 21 deletions(-) diff --git a/cores/esp8266/HardwareSerial.h b/cores/esp8266/HardwareSerial.h index cbfbd625e0..ba9fbf9989 100644 --- a/cores/esp8266/HardwareSerial.h +++ b/cores/esp8266/HardwareSerial.h @@ -143,7 +143,7 @@ class HardwareSerial: public Stream } // consume bytes after use (see peekBuffer) - void peekConsume (size_t consume) override; + void peekConsume (size_t consume) override { return uart_peek_consume(_uart, consume); } diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index b42de0e579..87344a8bdc 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -277,22 +277,26 @@ size_t Stream::streamTo (Print& to, size_t maxLen) size_t written = 0; while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) { - size_t r = available(); - if (w > r) - w = r; - if (!w) - return written; const char* pb = peekBuffer(); if (pb) { + size_t r = peekAvailable(); + if (w > r) + w = r; + if (!w) + return written; w = to.write(pb, w); peekConsume(w); } else { + size_t r = available(); + if (w > r) + w = r; + if (!w) + return written; if (w > MAXTRANSFERBLOCK) w = MAXTRANSFERBLOCK; - char temp[w]; r = read(temp, w); w = to.write(temp, r); diff --git a/cores/esp8266/uart.cpp b/cores/esp8266/uart.cpp index d55b280b81..3cd7cab65c 100644 --- a/cores/esp8266/uart.cpp +++ b/cores/esp8266/uart.cpp @@ -235,25 +235,25 @@ size_t uart_peek_available (uart_t* uart) ETS_UART_INTR_DISABLE(); uart_rx_copy_fifo_to_buffer_unsafe(uart); ETS_UART_INTR_ENABLE(); - if(rx_buffer->wpos < rx_buffer->rpos) - return rx_buffer->size - rx_buffer->rpos; - return rx_buffer->wpos - rx_buffer->rpos; + if(uart->rx_buffer->wpos < uart->rx_buffer->rpos) + return uart->rx_buffer->size - uart->rx_buffer->rpos; + return uart->rx_buffer->wpos - uart->rx_buffer->rpos; } // return a pointer to available data buffer (size = available()) // semantic forbids any kind of read() before calling peekConsume() const char* uart_peek_buffer (uart_t* uart) { - return &uart->rx_buffer->buffer[uart->rx_buffer->rpos]; + return (const char*)&uart->rx_buffer->buffer[uart->rx_buffer->rpos]; } // consume bytes after use (see uart_peek_buffer) void uart_peek_consume (uart_t* uart, size_t consume) { ETS_UART_INTR_DISABLE(); - rx_buffer->rpos += consume; - if (rx_buffer->rpos >= rx_buffer->size) - rx_buffer->rpos -= rx_buffer->size; + uart->rx_buffer->rpos += consume; + if (uart->rx_buffer->rpos >= uart->rx_buffer->size) + uart->rx_buffer->rpos -= uart->rx_buffer->size; ETS_UART_INTR_ENABLE(); } diff --git a/libraries/ESP8266WiFi/src/WiFiClient.cpp b/libraries/ESP8266WiFi/src/WiFiClient.cpp index 60629ddaea..d01afed931 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.cpp +++ b/libraries/ESP8266WiFi/src/WiFiClient.cpp @@ -427,19 +427,19 @@ uint8_t WiFiClient::getKeepAliveCount () const // return a pointer to available data buffer (size = peekAvailable()) // semantic forbids any kind of read() before calling peekConsume() -virtual const char* WiFiClient::peekBuffer () +const char* WiFiClient::peekBuffer () { return _client->peekBuffer(); } // return number of byte accessible by peekBuffer() -virtual size_t WiFiClient::peekAvailable () +size_t WiFiClient::peekAvailable () { - return _client->peekAvailale(); + return _client->peekAvailable(); } // consume bytes after use (see peekBuffer) -virtual void WiFiClient::peekConsume (size_t consume) +void WiFiClient::peekConsume (size_t consume) { return _client->peekConsume(consume); } diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h b/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h index ee2473306b..21619255e4 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h @@ -90,7 +90,7 @@ class WiFiClientSecure : public WiFiClient { virtual size_t peekAvailable () { return 0; } // consume bytes after use (see peekBuffer) - virtual void peekConsume (size_t consume) {} + virtual void peekConsume (size_t consume) { (void)consume; } friend class WiFiServerSecure; // Needs access to custom constructor below protected: diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h index 8056033d5c..0ab29fcfd6 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h @@ -132,7 +132,7 @@ class WiFiClientSecure : public WiFiClient { virtual size_t peekAvailable () { return 0; } // consume bytes after use (see peekBuffer) - virtual void peekConsume (size_t consume) {} + virtual void peekConsume (size_t consume) { (void)consume; } //////////////////////////////////////////////////// // AxTLS API deprecated warnings to help upgrading diff --git a/libraries/ESP8266WiFi/src/include/ClientContext.h b/libraries/ESP8266WiFi/src/include/ClientContext.h index 2fb7b06578..36fd1def0d 100644 --- a/libraries/ESP8266WiFi/src/include/ClientContext.h +++ b/libraries/ESP8266WiFi/src/include/ClientContext.h @@ -424,12 +424,16 @@ class ClientContext // semantic forbids any kind of read() before calling peekConsume() const char* peekBuffer () { - return _rx_buf->payload + _rx_buf_offset + if (!_rx_buf) + return nullptr; + return (const char*)_rx_buf->payload + _rx_buf_offset; } // return number of byte accessible by peekBuffer() size_t peekAvailable () { + if (!_rx_buf) + return 0; return _rx_buf->tot_len - _rx_buf_offset; } From bb4b64dcff4041eb8bf4742b894c50c385b70f18 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 26 Mar 2019 11:28:30 +0100 Subject: [PATCH 015/207] +template StreamMove() --- cores/esp8266/HardwareSerial.h | 2 +- cores/esp8266/Stream.cpp | 5 +- cores/esp8266/Stream.h | 71 +++++++++++++++++-- libraries/ESP8266WiFi/src/WiFiClient.cpp | 6 +- libraries/ESP8266WiFi/src/WiFiClient.h | 4 +- .../ESP8266WiFi/src/WiFiClientSecureAxTLS.h | 4 +- .../ESP8266WiFi/src/WiFiClientSecureBearSSL.h | 4 +- .../ESP8266WiFi/src/include/ClientContext.h | 4 +- 8 files changed, 80 insertions(+), 20 deletions(-) diff --git a/cores/esp8266/HardwareSerial.h b/cores/esp8266/HardwareSerial.h index ba9fbf9989..581e90c73e 100644 --- a/cores/esp8266/HardwareSerial.h +++ b/cores/esp8266/HardwareSerial.h @@ -137,7 +137,7 @@ class HardwareSerial: public Stream } // return number of byte accessible by peekBuffer() - size_t peekAvailable () override + size_t availableForPeek () override { return uart_peek_available(_uart); } diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index 87344a8bdc..3a5e080fd5 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -268,6 +268,7 @@ size_t Stream::read (char* buffer, size_t maxLen) return nbread; } +#if 0 #define MAXTRANSFERBLOCK 128 // allocated in stack, be nice size_t Stream::streamTo (Print& to, size_t maxLen) @@ -280,7 +281,7 @@ size_t Stream::streamTo (Print& to, size_t maxLen) const char* pb = peekBuffer(); if (pb) { - size_t r = peekAvailable(); + size_t r = availableForPeek(); if (w > r) w = r; if (!w) @@ -310,3 +311,5 @@ size_t Stream::streamTo (Print& to, size_t maxLen) } return written; } + +#endif diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 6b44fc9091..26c54ec362 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -35,6 +35,8 @@ readBytesBetween( pre_string, terminator, buffer, length) */ +template size_t streamMove (Tfrom& from, Tto& to, size_t maxLen = 0); + class Stream: public Print { protected: unsigned long _timeout; // number of milliseconds to wait for the next char before aborting timed read @@ -109,18 +111,19 @@ class Stream: public Print { // immediate return when no more data are available (no timeout) virtual size_t read(char* buffer, size_t maxLen); - // transfer at most maxlen bytes - // returns effectively transfered bytes (can be less than maxLen) - // maxlen==0 means transfer until nothing more can be read - // immediate return when no more data are available (no timeout) - virtual size_t streamTo (Print& to, size_t maxLen = 0); + // transfer at most maxlen bytes (maxlen==0 means transfer until starvation) + // immediate return number of transfered bytes (no timeout) + virtual size_t streamTo (Print& to, size_t maxLen = 0) + { + return streamMove(*this, to, maxLen); + } - // return a pointer to available data buffer (size = peekAvailable()) + // return a pointer to available data buffer (size = availableForPeek()) // semantic forbids any kind of read() before calling peekConsume() virtual const char* peekBuffer () { return nullptr; } // return number of byte accessible by peekBuffer() - virtual size_t peekAvailable () { return 0; } + virtual size_t availableForPeek () { return 0; } // consume bytes after use (see peekBuffer) virtual void peekConsume (size_t consume) { (void)consume; } @@ -133,4 +136,58 @@ class Stream: public Print { float parseFloat(char skipChar); // as above but the given skipChar is ignored }; +#include +//#include "Esp.h" +//#include "PolledTimeout.h" +template size_t streamMove (Tfrom& from, Tto& to, size_t maxLen = 0) +{ +// static constexpr auto yield_ms = 100; + static constexpr auto maxStreamToOnHeap = 128; +// esp8266::polledTimeout::periodic yieldNow(yield_ms); + size_t written = 0; + size_t w; + + if (from.peekBuffer()) + // direct write API available, avoid one copy + while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) + { + const char* pb = from.peekBuffer(); + size_t r = from.availableForPeek(); + if (w > r) + w = r; + if (!w) + return written; + w = to.write(pb, w); + from.peekConsume(w); + + written += w; + +// if (yieldNow) +// yield(); + } + else + // use Stream blck-read/write API + while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) + { + size_t r = from.available(); + if (w > r) + w = r; + if (!w) + return written; + if (w > maxStreamToOnHeap) + w = maxStreamToOnHeap; + char temp[w]; + r = from.read(temp, w); + w = to.write(temp, r); + assert(w == r); + + written += w; + +// if (yieldNow) +// yield(); + } + + return written; +} + #endif diff --git a/libraries/ESP8266WiFi/src/WiFiClient.cpp b/libraries/ESP8266WiFi/src/WiFiClient.cpp index d01afed931..832b9c69c4 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.cpp +++ b/libraries/ESP8266WiFi/src/WiFiClient.cpp @@ -425,7 +425,7 @@ uint8_t WiFiClient::getKeepAliveCount () const } -// return a pointer to available data buffer (size = peekAvailable()) +// return a pointer to available data buffer (size = availableForPeek()) // semantic forbids any kind of read() before calling peekConsume() const char* WiFiClient::peekBuffer () { @@ -433,9 +433,9 @@ const char* WiFiClient::peekBuffer () } // return number of byte accessible by peekBuffer() -size_t WiFiClient::peekAvailable () +size_t WiFiClient::availableForPeek () { - return _client->peekAvailable(); + return _client->availableForPeek(); } // consume bytes after use (see peekBuffer) diff --git a/libraries/ESP8266WiFi/src/WiFiClient.h b/libraries/ESP8266WiFi/src/WiFiClient.h index 71a6389f62..c3a48b7391 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.h +++ b/libraries/ESP8266WiFi/src/WiFiClient.h @@ -119,9 +119,9 @@ class WiFiClient : public Client, public SList { void setSync(bool sync); // return number of byte accessible by peekBuffer() - virtual size_t peekAvailable () override; + virtual size_t availableForPeek () override; - // return a pointer to available data buffer (size = peekAvailable()) + // return a pointer to available data buffer (size = availableForPeek()) // semantic forbids any kind of read() before calling peekConsume() virtual const char* peekBuffer () override; diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h b/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h index 21619255e4..9b68313b2e 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h @@ -82,12 +82,12 @@ class WiFiClientSecure : public WiFiClient { return loadCACert(file, file.size()); } - // return a pointer to available data buffer (size = peekAvailable()) + // return a pointer to available data buffer (size = availableForPeek()) // semantic forbids any kind of read() before calling peekConsume() virtual const char* peekBuffer () { return nullptr; } // return number of byte accessible by peekBuffer() - virtual size_t peekAvailable () { return 0; } + virtual size_t availableForPeek () { return 0; } // consume bytes after use (see peekBuffer) virtual void peekConsume (size_t consume) { (void)consume; } diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h index 0ab29fcfd6..37b87ad7a8 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h @@ -124,12 +124,12 @@ class WiFiClientSecure : public WiFiClient { // peek blocks not implemented yet: - // return a pointer to available data buffer (size = peekAvailable()) + // return a pointer to available data buffer (size = availableForPeek()) // semantic forbids any kind of read() before calling peekConsume() virtual const char* peekBuffer () { return nullptr; } // return number of byte accessible by peekBuffer() - virtual size_t peekAvailable () { return 0; } + virtual size_t availableForPeek () { return 0; } // consume bytes after use (see peekBuffer) virtual void peekConsume (size_t consume) { (void)consume; } diff --git a/libraries/ESP8266WiFi/src/include/ClientContext.h b/libraries/ESP8266WiFi/src/include/ClientContext.h index 36fd1def0d..2197d7b4f1 100644 --- a/libraries/ESP8266WiFi/src/include/ClientContext.h +++ b/libraries/ESP8266WiFi/src/include/ClientContext.h @@ -420,7 +420,7 @@ class ClientContext _sync = sync; } - // return a pointer to available data buffer (size = peekAvailable()) + // return a pointer to available data buffer (size = availableForPeek()) // semantic forbids any kind of read() before calling peekConsume() const char* peekBuffer () { @@ -430,7 +430,7 @@ class ClientContext } // return number of byte accessible by peekBuffer() - size_t peekAvailable () + size_t availableForPeek () { if (!_rx_buf) return 0; From 44b55d401dada50f55e2fcb073d2092e810ffe55 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 27 Mar 2019 01:01:32 +0100 Subject: [PATCH 016/207] update peekbuffer api, convergence with official arduino api --- cores/esp8266/FS.cpp | 2 +- cores/esp8266/FS.h | 2 +- cores/esp8266/HardwareSerial.h | 9 +++- cores/esp8266/Stream.cpp | 2 +- cores/esp8266/Stream.h | 43 ++++++++++++++----- cores/esp8266/Udp.h | 2 +- cores/esp8266/uart.cpp | 4 +- libraries/ESP8266WiFi/src/WiFiClient.cpp | 4 +- libraries/ESP8266WiFi/src/WiFiClient.h | 8 +++- .../ESP8266WiFi/src/WiFiClientSecureAxTLS.h | 3 ++ .../ESP8266WiFi/src/WiFiClientSecureBearSSL.h | 5 ++- libraries/ESP8266WiFi/src/WiFiUdp.h | 2 +- 12 files changed, 62 insertions(+), 24 deletions(-) diff --git a/cores/esp8266/FS.cpp b/cores/esp8266/FS.cpp index 4951b798b7..9eb7da3730 100644 --- a/cores/esp8266/FS.cpp +++ b/cores/esp8266/FS.cpp @@ -58,7 +58,7 @@ int File::read() { return result; } -size_t File::read(uint8_t* buf, size_t size) { +int File::read(uint8_t* buf, size_t size) { if (!_p) return -1; diff --git a/cores/esp8266/FS.h b/cores/esp8266/FS.h index d9c43ef3b8..4babe5d451 100644 --- a/cores/esp8266/FS.h +++ b/cores/esp8266/FS.h @@ -63,7 +63,7 @@ class File : public Stream size_t readBytes(char *buffer, size_t length) override { return read((uint8_t*)buffer, length); } - size_t read(uint8_t* buf, size_t size); + int read(uint8_t* buf, size_t size); bool seek(uint32_t pos, SeekMode mode); bool seek(uint32_t pos) { return seek(pos, SeekSet); diff --git a/cores/esp8266/HardwareSerial.h b/cores/esp8266/HardwareSerial.h index 581e90c73e..84be5bb470 100644 --- a/cores/esp8266/HardwareSerial.h +++ b/cores/esp8266/HardwareSerial.h @@ -129,6 +129,9 @@ class HardwareSerial: public Stream return uart_peek_char(_uart); } + // allow optimization for streamMove/STREAM_MOVE + static constexpr bool peekBufferAvailableAPI () { return true; } + // return a pointer to available data buffer (size = available()) // semantic forbids any kind of read() before calling peekConsume() const char* peekBuffer () override @@ -154,10 +157,14 @@ class HardwareSerial: public Stream return uart_read_char(_uart); } // ::read(buffer, size): same as readBytes without timeout - size_t read(char* buffer, size_t size) override + int read(char* buffer, size_t size) override { return uart_read(_uart, buffer, size); } + int read(uint8_t* buffer, size_t size) override + { + return uart_read(_uart, (char*)buffer, size); + } size_t readBytes(char* buffer, size_t size) override; size_t readBytes(uint8_t* buffer, size_t size) override { diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index 3a5e080fd5..c48fde85aa 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -258,7 +258,7 @@ String Stream::readStringUntil(char terminator) { return ret; } -size_t Stream::read (char* buffer, size_t maxLen) +int Stream::read (char* buffer, size_t maxLen) { IAMSLOW("Stream::read(buffer,len)"); diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 26c54ec362..6102253a01 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -88,6 +88,7 @@ class Stream: public Print { float parseFloat(); // float version of parseInt + // both arduino's "size_t Stream::readBytes(char*/uint8_t*, size_t): virtual size_t readBytes(char *buffer, size_t length); // read chars from stream into buffer virtual size_t readBytes(uint8_t *buffer, size_t length) { return readBytes((char *) buffer, length); @@ -106,28 +107,44 @@ class Stream: public Print { virtual String readString(); String readStringUntil(char terminator); - // read at most maxLen bytes + // additions: Client:: read() definitions moved here, because HardwareSerial have them too + + // read at most maxLen bytes: // returns effectively transfered bytes (can be less than maxLen) // immediate return when no more data are available (no timeout) - virtual size_t read(char* buffer, size_t maxLen); + virtual int read(char* buffer, size_t maxLen); + // arduino's "int Client::read(uint8_t*,size_t)": + virtual int read(uint8_t* buffer, size_t maxLen) { return read((char*)buffer, maxLen); } - // transfer at most maxlen bytes (maxlen==0 means transfer until starvation) - // immediate return number of transfered bytes (no timeout) - virtual size_t streamTo (Print& to, size_t maxLen = 0) - { - return streamMove(*this, to, maxLen); - } + // addition: directly access to read buffer + // (currently implemented in HardwareSerial:: and WiFiClient::) // return a pointer to available data buffer (size = availableForPeek()) + // api availability when return value is not nullptr // semantic forbids any kind of read() before calling peekConsume() virtual const char* peekBuffer () { return nullptr; } + // should be reimplemented as constexpr where relevant + // for streamMove<> / STREAM_MOVE optimization + bool peekBufferAvailableAPI () { return !!peekBuffer(); } + // return number of byte accessible by peekBuffer() virtual size_t availableForPeek () { return 0; } // consume bytes after use (see peekBuffer) virtual void peekConsume (size_t consume) { (void)consume; } + // additions: streamTo + + // transfer at most maxlen bytes (maxlen==0 means transfer until starvation) + // immediate return number of transfered bytes (no timeout) + // generic implementation using arduino virtual API + // (also available: virtual-less template streamMove(from,to)) + virtual size_t streamTo (Print& to, size_t maxLen = 0) + { + return streamMove(*this, to, maxLen); + } + protected: long parseInt(char skipChar); // as above but the given skipChar is ignored // as above but the given skipChar is ignored @@ -139,7 +156,11 @@ class Stream: public Print { #include //#include "Esp.h" //#include "PolledTimeout.h" -template size_t streamMove (Tfrom& from, Tto& to, size_t maxLen = 0) + +#define STREAM_MOVE(from,to,...) (streamMove(from, to, ## __VA_ARGS__)) + +template +size_t streamMove (Tfrom& from, Tto& to, size_t maxLen = 0) { // static constexpr auto yield_ms = 100; static constexpr auto maxStreamToOnHeap = 128; @@ -147,8 +168,8 @@ template size_t streamMove (Tfrom& from, Tto& to, size_t written = 0; size_t w; - if (from.peekBuffer()) - // direct write API available, avoid one copy + if (from.peekBufferAvailableAPI()) + // direct buffer read API available, avoid one copy while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) { const char* pb = from.peekBuffer(); diff --git a/cores/esp8266/Udp.h b/cores/esp8266/Udp.h index 2d873154ae..ba2d5e3715 100644 --- a/cores/esp8266/Udp.h +++ b/cores/esp8266/Udp.h @@ -73,7 +73,7 @@ class UDP: public Stream { virtual int read(unsigned char* buffer, size_t len) =0; // Read up to len characters from the current packet and place them into buffer // Returns the number of characters read, or 0 if none are available - virtual size_t read(char* buffer, size_t len) =0; + virtual int read(char* buffer, size_t len) =0; // Return the next byte from the current packet without moving on to the next byte virtual int peek() =0; virtual void flush() =0; // Finish reading the current packet diff --git a/cores/esp8266/uart.cpp b/cores/esp8266/uart.cpp index 3cd7cab65c..8b04787e41 100644 --- a/cores/esp8266/uart.cpp +++ b/cores/esp8266/uart.cpp @@ -235,7 +235,7 @@ size_t uart_peek_available (uart_t* uart) ETS_UART_INTR_DISABLE(); uart_rx_copy_fifo_to_buffer_unsafe(uart); ETS_UART_INTR_ENABLE(); - if(uart->rx_buffer->wpos < uart->rx_buffer->rpos) + if(uart->rx_buffer->wpos < uart->rx_buffer->rpos) return uart->rx_buffer->size - uart->rx_buffer->rpos; return uart->rx_buffer->wpos - uart->rx_buffer->rpos; } @@ -253,7 +253,7 @@ void uart_peek_consume (uart_t* uart, size_t consume) ETS_UART_INTR_DISABLE(); uart->rx_buffer->rpos += consume; if (uart->rx_buffer->rpos >= uart->rx_buffer->size) - uart->rx_buffer->rpos -= uart->rx_buffer->size; + uart->rx_buffer->rpos -= uart->rx_buffer->size; ETS_UART_INTR_ENABLE(); } diff --git a/libraries/ESP8266WiFi/src/WiFiClient.cpp b/libraries/ESP8266WiFi/src/WiFiClient.cpp index 832b9c69c4..dd2a623414 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.cpp +++ b/libraries/ESP8266WiFi/src/WiFiClient.cpp @@ -274,9 +274,9 @@ int WiFiClient::read() } -int WiFiClient::read(uint8_t* buf, size_t size) +int WiFiClient::read(char* buf, size_t size) { - return (int)_client->read(reinterpret_cast(buf), size); + return (int)_client->read(buf, size); } int WiFiClient::peek() diff --git a/libraries/ESP8266WiFi/src/WiFiClient.h b/libraries/ESP8266WiFi/src/WiFiClient.h index c3a48b7391..42ebcefcfa 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.h +++ b/libraries/ESP8266WiFi/src/WiFiClient.h @@ -66,9 +66,10 @@ class WiFiClient : public Client, public SList { virtual int available(); virtual int read(); - virtual int read(uint8_t *buf, size_t size) override; + virtual int read(char *buf, size_t size) override; + virtual int read(uint8_t *buf, size_t size) override { return read((char*)buf, size); } virtual int peek(); - virtual size_t peekBytes(uint8_t *buffer, size_t length); + virtual size_t peekBytes(uint8_t *buffer, size_t length); // timeout size_t peekBytes(char *buffer, size_t length) { return peekBytes((uint8_t *) buffer, length); } @@ -118,6 +119,9 @@ class WiFiClient : public Client, public SList { bool getSync() const; void setSync(bool sync); + // allow optimization for streamMove/STREAM_MOVE + static constexpr bool peekBufferAvailableAPI () { return true; } + // return number of byte accessible by peekBuffer() virtual size_t availableForPeek () override; diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h b/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h index 9b68313b2e..75b2e6c582 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h @@ -82,6 +82,9 @@ class WiFiClientSecure : public WiFiClient { return loadCACert(file, file.size()); } + // disallow optimization for streamMove/STREAM_MOVE + static constexpr bool peekBufferAvailableAPI () { return false; } + // return a pointer to available data buffer (size = availableForPeek()) // semantic forbids any kind of read() before calling peekConsume() virtual const char* peekBuffer () { return nullptr; } diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h index 37b87ad7a8..4d3e3b400d 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h @@ -122,7 +122,10 @@ class WiFiClientSecure : public WiFiClient { static bool probeMaxFragmentLength(const char *hostname, uint16_t port, uint16_t len); static bool probeMaxFragmentLength(const String& host, uint16_t port, uint16_t len); - // peek blocks not implemented yet: + // peek buffer not implemented yet: + + // disallow optimization for streamMove/STREAM_MOVE + static constexpr bool peekBufferAvailableAPI () { return false; } // return a pointer to available data buffer (size = availableForPeek()) // semantic forbids any kind of read() before calling peekConsume() diff --git a/libraries/ESP8266WiFi/src/WiFiUdp.h b/libraries/ESP8266WiFi/src/WiFiUdp.h index 42551c738b..fb205513c5 100644 --- a/libraries/ESP8266WiFi/src/WiFiUdp.h +++ b/libraries/ESP8266WiFi/src/WiFiUdp.h @@ -89,7 +89,7 @@ class WiFiUDP : public UDP, public SList { int read(unsigned char* buffer, size_t len) override; // Read up to len characters from the current packet and place them into buffer // Returns the number of characters read, or 0 if none are available - size_t read(char* buffer, size_t len) override { return read((unsigned char*)buffer, len); }; + int read(char* buffer, size_t len) override { return read((unsigned char*)buffer, len); }; // Return the next byte from the current packet without moving on to the next byte int peek() override; void flush() override; // Finish reading the current packet From 51b192e26268ec13cf9b576ef2e10f46bdd512e3 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 27 Mar 2019 09:27:01 +0100 Subject: [PATCH 017/207] fix host emulation warning --- cores/esp8266/Stream.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 6102253a01..fb2f0b19d9 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -35,7 +35,8 @@ readBytesBetween( pre_string, terminator, buffer, length) */ -template size_t streamMove (Tfrom& from, Tto& to, size_t maxLen = 0); +template +size_t streamMove (Tfrom& from, Tto& to, size_t maxLen = 0); class Stream: public Print { protected: @@ -160,7 +161,7 @@ class Stream: public Print { #define STREAM_MOVE(from,to,...) (streamMove(from, to, ## __VA_ARGS__)) template -size_t streamMove (Tfrom& from, Tto& to, size_t maxLen = 0) +size_t streamMove (Tfrom& from, Tto& to, size_t maxLen) { // static constexpr auto yield_ms = 100; static constexpr auto maxStreamToOnHeap = 128; From ad22743d06542443ddf8b1e1482d91d7d046e14c Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 6 Apr 2019 23:22:37 +0200 Subject: [PATCH 018/207] wip --- tests/host/common/include/ClientContext.h | 27 ++++++++++++++++++++ tests/host/emutest-FSBrowser+ClientPoster.sh | 4 +-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/tests/host/common/include/ClientContext.h b/tests/host/common/include/ClientContext.h index f9e4a9e3d5..0c81726ab5 100644 --- a/tests/host/common/include/ClientContext.h +++ b/tests/host/common/include/ClientContext.h @@ -297,6 +297,33 @@ class ClientContext _sync = sync; } + // return a pointer to available data buffer (size = availableForPeek()) + // semantic forbids any kind of read() before calling peekConsume() + const char* peekBuffer () + { + XXX + } + + // return number of byte accessible by peekBuffer() + size_t availableForPeek () + { + XXX + ssize_t ret = mockPeekBytes(_sock, dst, size, 0, _inbuf, _inbufsize); + if (ret < 0) + { + abort(); + return 0; + } + return ret; + } + + // consume bytes after use (see peekBuffer) + void peekConsume (size_t consume) + { + XXX + mockConsume(_sock, consume); + } + private: discard_cb_t _discard_cb = nullptr; diff --git a/tests/host/emutest-FSBrowser+ClientPoster.sh b/tests/host/emutest-FSBrowser+ClientPoster.sh index d88c37cc65..5788086aa2 100755 --- a/tests/host/emutest-FSBrowser+ClientPoster.sh +++ b/tests/host/emutest-FSBrowser+ClientPoster.sh @@ -22,8 +22,8 @@ sleep=5 sizeb=$(($sizekb * 1024)) fileb=$(($sizeb / 3)) -make $DEBUG ../../libraries/ESP8266WebServer/examples/FSBrowser/FSBrowser -make $DEBUG otherexamples/HttpClientPost/HttpClientPost +make -j $DEBUG ../../libraries/ESP8266WebServer/examples/FSBrowser/FSBrowser +make -j $DEBUG otherexamples/HttpClientPost/HttpClientPost killall -9 FSBrowser HttpClientPost || true From 2eabc73649690efdff6f73a101718f6c7b59fd44 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Thu, 11 Apr 2019 01:36:08 +0200 Subject: [PATCH 019/207] bstream: wip --- tests/host/common/Arduino.cpp | 2 ++ tests/host/common/ClientContextSocket.cpp | 6 +++++- tests/host/common/MockUART.cpp | 6 ++++++ tests/host/common/include/ClientContext.h | 15 +++++++++------ tests/host/emutest-FSBrowser+ClientPoster.sh | 15 ++++++++++----- .../HttpClientPost/HttpClientPost.ino | 3 +-- 6 files changed, 33 insertions(+), 14 deletions(-) diff --git a/tests/host/common/Arduino.cpp b/tests/host/common/Arduino.cpp index e01aab8284..d5100bf4d6 100644 --- a/tests/host/common/Arduino.cpp +++ b/tests/host/common/Arduino.cpp @@ -71,3 +71,5 @@ extern "C" void delayMicroseconds(unsigned int us) { usleep(us); } + +const char* overrideme PROGMEM = " should be overridden for better efficiency\r\n"; diff --git a/tests/host/common/ClientContextSocket.cpp b/tests/host/common/ClientContextSocket.cpp index 4db72c9bc0..b2366fa72c 100644 --- a/tests/host/common/ClientContextSocket.cpp +++ b/tests/host/common/ClientContextSocket.cpp @@ -136,7 +136,11 @@ ssize_t mockPeekBytes (int sock, char* dst, size_t usersize, int timeout_ms, cha p.events = POLLIN; } while (poll(&p, 1, timeout_ms) == 1); - memcpy(dst, ccinbuf, retsize); + if (dst) + { + memcpy(dst, ccinbuf, retsize); + } + return retsize; } diff --git a/tests/host/common/MockUART.cpp b/tests/host/common/MockUART.cpp index 152072bca8..de2764a822 100644 --- a/tests/host/common/MockUART.cpp +++ b/tests/host/common/MockUART.cpp @@ -432,3 +432,9 @@ uart_detect_baudrate(int uart_nr) } }; + + +size_t uart_peek_available (uart_t* uart) { return 0; } +const char* uart_peek_buffer (uart_t* uart) { return nullptr; } +void uart_peek_consume (uart_t* uart, size_t consume) { (void)uart; (void)consume; } + diff --git a/tests/host/common/include/ClientContext.h b/tests/host/common/include/ClientContext.h index 0c81726ab5..66ca5cfc52 100644 --- a/tests/host/common/include/ClientContext.h +++ b/tests/host/common/include/ClientContext.h @@ -301,27 +301,30 @@ class ClientContext // semantic forbids any kind of read() before calling peekConsume() const char* peekBuffer () { - XXX + return _inbuf; } // return number of byte accessible by peekBuffer() size_t availableForPeek () { - XXX - ssize_t ret = mockPeekBytes(_sock, dst, size, 0, _inbuf, _inbufsize); + ssize_t ret = mockPeekBytes(_sock, nullptr, sizeof _inbuf, 0, _inbuf, _inbufsize); if (ret < 0) { abort(); return 0; } - return ret; + return _inbufsize; } // consume bytes after use (see peekBuffer) void peekConsume (size_t consume) { - XXX - mockConsume(_sock, consume); + assert(consume <= _inbufsize); + size_t move = consume; + if (consume + move > sizeof _inbuf) + move = sizeof _inbuf - consume; + memmove(_inbuf, _inbuf + consume, move); + _inbufsize -= consume; } private: diff --git a/tests/host/emutest-FSBrowser+ClientPoster.sh b/tests/host/emutest-FSBrowser+ClientPoster.sh index 5788086aa2..12df7a4e05 100755 --- a/tests/host/emutest-FSBrowser+ClientPoster.sh +++ b/tests/host/emutest-FSBrowser+ClientPoster.sh @@ -8,7 +8,7 @@ # # Run FSbrowser official example # use it to upload a random (big) file to emulated SPIFFS -# Stop it, copy the emulated SPIFFS file for another HTttpClientPost sketch +# Stop it, copy the emulated SPIFFS file for another HTTPClientPost sketch # Restart the Server # Start the Poster, it will upload a copy to the server # end @@ -17,6 +17,7 @@ set -e sizekb=16000 # SPIFFS SIZE in KBytes (file size will be the third of this value) DEBUG="D=1" # comment this or not +options="-f -b" sleep=5 sizeb=$(($sizekb * 1024)) @@ -27,8 +28,9 @@ make -j $DEBUG otherexamples/HttpClientPost/HttpClientPost killall -9 FSBrowser HttpClientPost || true +# remove SPIFFS rm -f FSBrowser-spiffs${sizekb}KB -./bin/FSBrowser/FSBrowser -S ${sizekb} -f & PIDFSBrowser=$! +./bin/FSBrowser/FSBrowser $options -S ${sizekb} & PIDFSBrowser=$! echo "------------------------------" echo " let server start..." @@ -45,9 +47,11 @@ echo "------------------------------" curl -F "file=@$PWD/randfile" 127.0.0.1:9080/edit echo "------------------------------" echo " Uploading to SPIFFS (done)" +echo " (wait&kill FSBrowser) " echo "------------------------------" - +sleep ${sleep} kill -INT $PIDFSBrowser +sleep ${sleep} # FSBrowser has generated SPIFFS backup file, copy it for the other sketch # This sketch will repost this file to the FSBrowser, so we can debug/tune @@ -59,18 +63,19 @@ ls -al ./bin/FSBrowser/FSBrowser-spiffs${sizekb}KB ./bin/HttpClientPost/HttpClie echo "------------------------------" echo " let server start again..." echo "------------------------------" -./bin/FSBrowser/FSBrowser -S ${sizekb} -f & PIDFSBrowser=$! +./bin/FSBrowser/FSBrowser $options -S ${sizekb} & PIDFSBrowser=$! sleep ${sleep} echo "------------------------------" echo " start uploader sketch" echo "------------------------------" # run http client poster sketch -./bin/HttpClientPost/HttpClientPost -S ${sizekb} -f & PIDClient=$! +./bin/HttpClientPost/HttpClientPost $options -S ${sizekb} & PIDClient=$! echo "------------------------------" echo "Let run the sketches, press enter to kill" echo "------------------------------" +echo "upload duration:" read junk echo "killing everybody" diff --git a/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino b/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino index 5d5b81e27b..e687c0ff1a 100644 --- a/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino +++ b/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino @@ -54,13 +54,12 @@ void setup() { Serial.printf("\r\n\r\nCLIENT: ----> time-keep: %lums sent: %d\r\n\r\n", (millis() - A), (int)client.write(f)); client.println(); client.println("--glark--"); - A = millis(); Serial.println("CLIENT waiting for server ack..."); while (!client.available()) yield(); A = millis() - A; Serial.printf("\r\n\r\n##########################\r\n"); - Serial.printf("\r\n\r\nCLIENT: ----> server response: +%lums\r\n", A); + Serial.printf("\r\n\r\nCLIENT: ----> upload duration: +%lums\r\n", A); Serial.printf("\r\n\r\n##########################\r\n"); Serial.println("CLIENT: server response:"); From 31170913ad7c12571a88a8a0291617ebb2b35031 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Thu, 11 Apr 2019 02:30:53 +0200 Subject: [PATCH 020/207] bstream: wip --- cores/esp8266/Arduino.h | 11 +++++ cores/esp8266/Esp.h | 16 ++---- cores/esp8266/PolledTimeout.h | 6 +-- cores/esp8266/Stream.cpp | 43 +--------------- cores/esp8266/Stream.h | 28 +++++------ .../src/ESP8266HTTPClient.cpp | 13 +++++ tests/host/common/MockEsp.cpp | 7 ++- tests/host/common/mock.h | 49 ++++++++++--------- 8 files changed, 77 insertions(+), 96 deletions(-) diff --git a/cores/esp8266/Arduino.h b/cores/esp8266/Arduino.h index 34e5fb8fce..b1c7d607b3 100644 --- a/cores/esp8266/Arduino.h +++ b/cores/esp8266/Arduino.h @@ -301,6 +301,17 @@ extern "C" void configTime(long timezone, int daylightOffset_sec, #include "pins_arduino.h" +#ifdef CORE_MOCK +uint32_t espGetCycleCount(); +#else +inline uint32_t espGetCycleCount() __attribute__((always_inline)) +{ + uint32_t ccount; + __asm__ __volatile__("esync; rsr %0,ccount":"=a" (ccount)); + return ccount; +} +#endif + #endif #ifdef DEBUG_ESP_OOM diff --git a/cores/esp8266/Esp.h b/cores/esp8266/Esp.h index f717c15881..f067dca6d9 100644 --- a/cores/esp8266/Esp.h +++ b/cores/esp8266/Esp.h @@ -200,20 +200,12 @@ class EspClass { bool eraseConfig(); -#ifndef CORE_MOCK - inline -#endif +#ifdef CORE_MOCK uint32_t getCycleCount(); -}; - -#ifndef CORE_MOCK -uint32_t EspClass::getCycleCount() -{ - uint32_t ccount; - __asm__ __volatile__("esync; rsr %0,ccount":"=a" (ccount)); - return ccount; -} +#else + inline uint32_t getCycleCount() { return espGetCycleCount(); } #endif +}; extern EspClass ESP; diff --git a/cores/esp8266/PolledTimeout.h b/cores/esp8266/PolledTimeout.h index 91bebaa79b..5a37d5e83c 100644 --- a/cores/esp8266/PolledTimeout.h +++ b/cores/esp8266/PolledTimeout.h @@ -70,12 +70,12 @@ struct TimeSourceMillis struct TimeSourceCycles { - // time policy based on ESP.getCycleCount() + // time policy based on espGetCycleCount() // this particular time measurement is intended to be called very often // (every loop, every yield) - using timeType = decltype(ESP.getCycleCount()); - static timeType time() {return ESP.getCycleCount();} + using timeType = decltype(espGetCycleCount()); + static timeType time() {return espGetCycleCount();} static constexpr timeType ticksPerSecond = F_CPU; // 80'000'000 or 160'000'000 Hz static constexpr timeType ticksPerSecondMax = 160000000; // 160MHz }; diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index c48fde85aa..b0d5ef452a 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -268,48 +268,7 @@ int Stream::read (char* buffer, size_t maxLen) return nbread; } -#if 0 -#define MAXTRANSFERBLOCK 128 // allocated in stack, be nice - size_t Stream::streamTo (Print& to, size_t maxLen) { - esp8266::polledTimeout::periodic yieldNow(100); - size_t w; - size_t written = 0; - while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) - { - const char* pb = peekBuffer(); - if (pb) - { - size_t r = availableForPeek(); - if (w > r) - w = r; - if (!w) - return written; - w = to.write(pb, w); - peekConsume(w); - } - else - { - size_t r = available(); - if (w > r) - w = r; - if (!w) - return written; - if (w > MAXTRANSFERBLOCK) - w = MAXTRANSFERBLOCK; - char temp[w]; - r = read(temp, w); - w = to.write(temp, r); - assert(w == r); - } - - written += w; - - if (yieldNow) - yield(); - } - return written; + return streamMove(*this, to, maxLen); } - -#endif diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index fb2f0b19d9..b756118217 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -35,8 +35,11 @@ readBytesBetween( pre_string, terminator, buffer, length) */ +#include +using OneShotFastMs = esp8266::polledTimeout::oneShotFastMs; +using PeriodicFastMs = esp8266::polledTimeout::periodicFastMs; template -size_t streamMove (Tfrom& from, Tto& to, size_t maxLen = 0); +size_t streamMove (Tfrom& from, Tto& to, size_t maxLen = 0, OneShotFastMs timeout = OneShotFastMs::alwaysExpired); class Stream: public Print { protected: @@ -141,10 +144,7 @@ class Stream: public Print { // immediate return number of transfered bytes (no timeout) // generic implementation using arduino virtual API // (also available: virtual-less template streamMove(from,to)) - virtual size_t streamTo (Print& to, size_t maxLen = 0) - { - return streamMove(*this, to, maxLen); - } + virtual size_t streamTo (Print& to, size_t maxLen = 0); protected: long parseInt(char skipChar); // as above but the given skipChar is ignored @@ -155,20 +155,18 @@ class Stream: public Print { }; #include -//#include "Esp.h" -//#include "PolledTimeout.h" #define STREAM_MOVE(from,to,...) (streamMove(from, to, ## __VA_ARGS__)) template -size_t streamMove (Tfrom& from, Tto& to, size_t maxLen) +size_t streamMove (Tfrom& from, Tto& to, size_t maxLen, OneShotFastMs timeout) { -// static constexpr auto yield_ms = 100; + static constexpr auto yield_ms = 100; static constexpr auto maxStreamToOnHeap = 128; -// esp8266::polledTimeout::periodic yieldNow(yield_ms); + PeriodicFastMs yieldNow(yield_ms); size_t written = 0; size_t w; - +//////////////XXX use timeout if (from.peekBufferAvailableAPI()) // direct buffer read API available, avoid one copy while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) @@ -184,8 +182,8 @@ size_t streamMove (Tfrom& from, Tto& to, size_t maxLen) written += w; -// if (yieldNow) -// yield(); + if (yieldNow) + yield(); } else // use Stream blck-read/write API @@ -205,8 +203,8 @@ size_t streamMove (Tfrom& from, Tto& to, size_t maxLen) written += w; -// if (yieldNow) -// yield(); + if (yieldNow) + yield(); } return written; diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 2c9542cb03..dd32fa5379 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -744,6 +744,17 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) return returnError(HTTPC_ERROR_SEND_HEADER_FAILED); } +#if 1 + + size_t transferred = 0; + while (connected() && transferred < size) + { + transferred += stream->streamTo(*_client, size - transferred); + } + +#else + + int buff_size = HTTP_TCP_BUFFER_SIZE; int len = size; @@ -851,6 +862,8 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) return returnError(HTTPC_ERROR_TOO_LESS_RAM); } +#endif + // handle Server Response (Header) return returnError(handleHeaderResponse()); } diff --git a/tests/host/common/MockEsp.cpp b/tests/host/common/MockEsp.cpp index 21a124b80d..eded1de611 100644 --- a/tests/host/common/MockEsp.cpp +++ b/tests/host/common/MockEsp.cpp @@ -217,9 +217,14 @@ void EspClass::resetFreeContStack() { } -uint32_t EspClass::getCycleCount() +uint32_t espGetCycleCount() { timeval t; gettimeofday(&t, NULL); return (((uint64_t)t.tv_sec) * 1000000 + t.tv_usec) * (F_CPU / 1000000); } + +uint32_t EspClass::getCycleCount() +{ + return espGetCycleCount(); +} diff --git a/tests/host/common/mock.h b/tests/host/common/mock.h index 067411139f..3278ea9c9e 100644 --- a/tests/host/common/mock.h +++ b/tests/host/common/mock.h @@ -31,6 +31,29 @@ #define CORE_MOCK 1 +// + +#define ARDUINO 267 +#define ESP8266 1 +#define A0 0 +#define LED_BUILTIN 0 +#define F_CPU 80000000 +#define LWIP_OPEN_SRC +#define TCP_MSS 536 +#define LWIP_FEATURES 1 + +// + +#define D0 0 +#define D1 1 +#define D2 3 +#define D3 3 +#define D4 4 +#define D5 5 +#define D6 6 +#define D7 7 +#define D8 8 + // include host's STL before any other include file // because core definition like max() is in the way @@ -58,6 +81,9 @@ typedef uint32_t uint32; // +#include +uint32_t espGetCycleCount(); + #include // @@ -140,29 +166,6 @@ void mock_stop_spiffs (); // -#define ARDUINO 267 -#define ESP8266 1 -#define A0 0 -#define LED_BUILTIN 0 -#define F_CPU 80000000 -#define LWIP_OPEN_SRC -#define TCP_MSS 536 -#define LWIP_FEATURES 1 - -// - -#define D0 0 -#define D1 1 -#define D2 3 -#define D3 3 -#define D4 4 -#define D5 5 -#define D6 6 -#define D7 7 -#define D8 8 - -// - #include // From a7f895d482251a9a5d465ad4a4ae07872072d65d Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Fri, 12 Apr 2019 14:58:17 +0200 Subject: [PATCH 021/207] wip --- cores/esp8266/Stream.h | 50 ++++++++++++++----- .../HttpClientPost/HttpClientPost.ino | 38 ++++++++++---- 2 files changed, 66 insertions(+), 22 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index b756118217..5f34e7c9e7 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -22,6 +22,7 @@ #ifndef Stream_h #define Stream_h +#include #include #include "Print.h" @@ -36,10 +37,12 @@ */ #include -using OneShotFastMs = esp8266::polledTimeout::oneShotFastMs; -using PeriodicFastMs = esp8266::polledTimeout::periodicFastMs; -template -size_t streamMove (Tfrom& from, Tto& to, size_t maxLen = 0, OneShotFastMs timeout = OneShotFastMs::alwaysExpired); + +template +size_t streamMove (Tfrom& from, Tto& to, size_t maxLen = 0, TimeoutMs timeout = TimeoutMs::alwaysExpired); +// by default: TimeoutMs::alwaysExpired: will use Stream::_timeout class Stream: public Print { protected: @@ -61,6 +64,7 @@ class Stream: public Print { // parsing methods void setTimeout(unsigned long timeout); // sets maximum milliseconds to wait for stream data, default is 1 second + unsigned long getTimeout () { return _timeout; } bool find(const char *target); // reads data from the stream until the target string is found bool find(uint8_t *target) { @@ -120,6 +124,8 @@ class Stream: public Print { // arduino's "int Client::read(uint8_t*,size_t)": virtual int read(uint8_t* buffer, size_t maxLen) { return read((char*)buffer, maxLen); } + //////////////////// extensions: + // addition: directly access to read buffer // (currently implemented in HardwareSerial:: and WiFiClient::) @@ -141,11 +147,13 @@ class Stream: public Print { // additions: streamTo // transfer at most maxlen bytes (maxlen==0 means transfer until starvation) - // immediate return number of transfered bytes (no timeout) + // immediate return number of transfered bytes (no timeout) XXXXXXFIXME // generic implementation using arduino virtual API // (also available: virtual-less template streamMove(from,to)) virtual size_t streamTo (Print& to, size_t maxLen = 0); + //////////////////// end of extensions + protected: long parseInt(char skipChar); // as above but the given skipChar is ignored // as above but the given skipChar is ignored @@ -158,18 +166,30 @@ class Stream: public Print { #define STREAM_MOVE(from,to,...) (streamMove(from, to, ## __VA_ARGS__)) -template -size_t streamMove (Tfrom& from, Tto& to, size_t maxLen, OneShotFastMs timeout) +template +size_t streamMove (Tfrom& from, Tto& to, size_t maxLen, TimeoutMs timeout) { static constexpr auto yield_ms = 100; static constexpr auto maxStreamToOnHeap = 128; - PeriodicFastMs yieldNow(yield_ms); + PeriodicMs yieldNow(yield_ms); size_t written = 0; size_t w; -//////////////XXX use timeout + + if (timeout == TimeoutMs::alwaysExpired) + // default value => take Stream (source) value + // TODO: check for to::getTimeout() existence (w/ SFINAE or equiv.) + // and use in that case std::min(from.getTimeout(), to.getTimeout()) + timeout.reset(from.getTimeout()); + else + timeout.reset(); + +DEBUGV("---------------------------------------- %d\n", timeout.getTimeout()); + if (from.peekBufferAvailableAPI()) + { + DEBUGV(":pkbf %s:%d\r\n", __FILE__, __LINE__); // direct buffer read API available, avoid one copy - while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) + while (!timeout && (!maxLen || written < maxLen) && (w = to.availableForWrite())) { const char* pb = from.peekBuffer(); size_t r = from.availableForPeek(); @@ -184,10 +204,14 @@ size_t streamMove (Tfrom& from, Tto& to, size_t maxLen, OneShotFastMs timeout) if (yieldNow) yield(); + timeout.reset(); } + } else - // use Stream blck-read/write API - while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) + { + DEBUGV(":cpbf %s:%d\r\n", __FILE__, __LINE__); + // use Stream block-read/write API + while (!timeout && (!maxLen || written < maxLen) && (w = to.availableForWrite())) { size_t r = from.available(); if (w > r) @@ -205,7 +229,9 @@ size_t streamMove (Tfrom& from, Tto& to, size_t maxLen, OneShotFastMs timeout) if (yieldNow) yield(); + timeout.reset(); } + } return written; } diff --git a/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino b/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino index e687c0ff1a..b2bf683a44 100644 --- a/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino +++ b/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino @@ -47,26 +47,44 @@ void setup() { client.printf("Content-Disposition: form-data; name=\"%s\"; filename=\"%s\"\r\n", FILE "copy", FILE "copy"); client.println("Content-Type: application/octet-stream"); client.println(""); - + + Serial.printf("\r\n\r\n"); Serial.printf("\r\n\r\n##########################\r\n"); + + +#define TESTSTREAM 0 + +printf("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\r\n"); + auto A = millis(); - Serial.printf("\r\n\r\nCLIENT: ----> time-keep: %lums sent: %d\r\n\r\n", (millis() - A), (int)client.write(f)); - client.println(); - client.println("--glark--"); + Serial.printf("\r\n\r\nCLIENT: ----> upload duration: %lums sent: %d\r\n\r\n", + millis() - A, +#if TESTSTREAM + (int)f.streamTo(client, 0) +#else + (int)client.write(f) +#endif + ); + A = millis() - A; + +printf("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\r\n"); + Serial.println("CLIENT waiting for server ack..."); + client.println("\r\n--glark--"); while (!client.available()) yield(); - A = millis() - A; - Serial.printf("\r\n\r\n##########################\r\n"); - Serial.printf("\r\n\r\nCLIENT: ----> upload duration: +%lums\r\n", A); - Serial.printf("\r\n\r\n##########################\r\n"); - Serial.println("CLIENT: server response:"); + Serial.println("@@@@@@@@@@@@@ CLIENT: server response:"); +#if TESTSTREAM + client.setTimeout(10000); + client.streamTo(Serial, 0); +#else while (client.available()) Serial.write(client.read()); +#endif - Serial.println("CLIENT: end"); + Serial.println("@@@@@@@@@@@@@ CLIENT: end"); client.stop(); f.close(); } From b14d5f737596c7daec7c4c2159d8a740cfcb123b Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 14 Apr 2019 22:19:36 +0200 Subject: [PATCH 022/207] wip --- cores/esp8266/Stream.h | 97 +++++++++++++----------------------------- 1 file changed, 30 insertions(+), 67 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 5f34e7c9e7..6f2ace7ff4 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -36,14 +36,6 @@ readBytesBetween( pre_string, terminator, buffer, length) */ -#include - -template -size_t streamMove (Tfrom& from, Tto& to, size_t maxLen = 0, TimeoutMs timeout = TimeoutMs::alwaysExpired); -// by default: TimeoutMs::alwaysExpired: will use Stream::_timeout - class Stream: public Print { protected: unsigned long _timeout; // number of milliseconds to wait for the next char before aborting timed read @@ -129,25 +121,20 @@ class Stream: public Print { // addition: directly access to read buffer // (currently implemented in HardwareSerial:: and WiFiClient::) + // return number of byte accessible by peekBuffer() + virtual size_t availableForPeek () { return read(&oneChar, 1) != -1; } + // return a pointer to available data buffer (size = availableForPeek()) - // api availability when return value is not nullptr // semantic forbids any kind of read() before calling peekConsume() - virtual const char* peekBuffer () { return nullptr; } + virtual const char* peekBuffer () { return &oneChar; } - // should be reimplemented as constexpr where relevant - // for streamMove<> / STREAM_MOVE optimization - bool peekBufferAvailableAPI () { return !!peekBuffer(); } - - // return number of byte accessible by peekBuffer() - virtual size_t availableForPeek () { return 0; } - - // consume bytes after use (see peekBuffer) + // consume bytes after peekBuffer use virtual void peekConsume (size_t consume) { (void)consume; } // additions: streamTo // transfer at most maxlen bytes (maxlen==0 means transfer until starvation) - // immediate return number of transfered bytes (no timeout) XXXXXXFIXME + // return number of transfered bytes // generic implementation using arduino virtual API // (also available: virtual-less template streamMove(from,to)) virtual size_t streamTo (Print& to, size_t maxLen = 0); @@ -160,18 +147,21 @@ class Stream: public Print { // this allows format characters (typically commas) in values to be ignored float parseFloat(char skipChar); // as above but the given skipChar is ignored + + char oneChar; }; -#include - #define STREAM_MOVE(from,to,...) (streamMove(from, to, ## __VA_ARGS__)) -template -size_t streamMove (Tfrom& from, Tto& to, size_t maxLen, TimeoutMs timeout) +#include +#include + +template +size_t streamMove (Tfrom& from, Tto& to, size_t maxLen = 0, int readUntilChar = -1, TimeoutMs::timeType timeout = TimeoutMs::alwaysExpired) { - static constexpr auto yield_ms = 100; - static constexpr auto maxStreamToOnHeap = 128; - PeriodicMs yieldNow(yield_ms); + PeriodicMs yieldNow(100); size_t written = 0; size_t w; @@ -183,54 +173,27 @@ size_t streamMove (Tfrom& from, Tto& to, size_t maxLen, TimeoutMs timeout) else timeout.reset(); -DEBUGV("---------------------------------------- %d\n", timeout.getTimeout()); - - if (from.peekBufferAvailableAPI()) + while (!timeout && (!maxLen || written < maxLen) && (w = to.availableForWrite())) { - DEBUGV(":pkbf %s:%d\r\n", __FILE__, __LINE__); - // direct buffer read API available, avoid one copy - while (!timeout && (!maxLen || written < maxLen) && (w = to.availableForWrite())) + const char* directbuf = from.peekBuffer(); + size_t r = from.availableForPeek(); + if (w > r) + w = r; + if (readUntilChar != -1) { - const char* pb = from.peekBuffer(); - size_t r = from.availableForPeek(); - if (w > r) - w = r; - if (!w) - return written; - w = to.write(pb, w); - from.peekConsume(w); - - written += w; - - if (yieldNow) - yield(); - timeout.reset(); + const char* last = memchr(directbuf, readUntilChar, w); + if (last) + w = last - directbuf + 1; } - } - else - { - DEBUGV(":cpbf %s:%d\r\n", __FILE__, __LINE__); - // use Stream block-read/write API - while (!timeout && (!maxLen || written < maxLen) && (w = to.availableForWrite())) + if (w && ((w = to.write(directbuf, w))) { - size_t r = from.available(); - if (w > r) - w = r; - if (!w) - return written; - if (w > maxStreamToOnHeap) - w = maxStreamToOnHeap; - char temp[w]; - r = from.read(temp, w); - w = to.write(temp, r); - assert(w == r); - + from.peekConsume(w); written += w; - - if (yieldNow) - yield(); timeout.reset(); } + + if (yieldNow) + yield(); } return written; From ad5a4ea5f6e0265f2c1acffb755fa258c1172e07 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 15 Apr 2019 01:38:19 +0200 Subject: [PATCH 023/207] wip --- cores/esp8266/Print.cpp | 4 +-- cores/esp8266/Stream.cpp | 4 +-- cores/esp8266/Stream.h | 56 +++++++++++++++++++++------------------- cores/esp8266/debug.h | 6 ++--- 4 files changed, 37 insertions(+), 33 deletions(-) diff --git a/cores/esp8266/Print.cpp b/cores/esp8266/Print.cpp index 2d4e826905..d5fa702039 100644 --- a/cores/esp8266/Print.cpp +++ b/cores/esp8266/Print.cpp @@ -33,7 +33,7 @@ /* default implementation: may be overridden */ size_t Print::write(const uint8_t *buffer, size_t size) { - IAMSLOW("Print::write(buffer,len)"); + IAMSLOW(); size_t n = 0; while (size--) { @@ -298,6 +298,6 @@ size_t Print::printFloat(double number, uint8_t digits) { } int Print::availableForWrite() { - IAMSLOW("Print::availableForWrite()"); + IAMSLOW(); return 1; } diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index b0d5ef452a..a32f9d7d56 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -268,7 +268,7 @@ int Stream::read (char* buffer, size_t maxLen) return nbread; } -size_t Stream::streamTo (Print& to, size_t maxLen) +size_t Stream::streamTo (Print& to, size_t maxLen = 0, int readUntilChar = -1, unsigned long timeout_ms = 0) { - return streamMove(*this, to, maxLen); + return streamMove(*this, to, maxLen, readUntilChar, timeout_ms); } diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 6f2ace7ff4..187f9055c5 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -26,7 +26,7 @@ #include #include "Print.h" -// compatability macros for testing +// compatibility macros for testing /* #define getInt() parseInt() #define getInt(skipChar) parseInt(skipchar) @@ -43,6 +43,7 @@ class Stream: public Print { int timedRead(); // private method to read stream with timeout int timedPeek(); // private method to peek stream with timeout int peekNextDigit(); // returns the next numeric digit in the stream or -1 if timeout + char oneChar; // used by direct access extension public: virtual int available() = 0; @@ -122,22 +123,26 @@ class Stream: public Print { // (currently implemented in HardwareSerial:: and WiFiClient::) // return number of byte accessible by peekBuffer() - virtual size_t availableForPeek () { return read(&oneChar, 1) != -1; } + virtual size_t availableForPeek () { IAMSLOW(); return (oneChar = peek()) == -1? 0: 1; } // return a pointer to available data buffer (size = availableForPeek()) // semantic forbids any kind of read() before calling peekConsume() virtual const char* peekBuffer () { return &oneChar; } // consume bytes after peekBuffer use - virtual void peekConsume (size_t consume) { (void)consume; } + virtual void peekConsume (size_t consume) { (void)read(); } // additions: streamTo - // transfer at most maxlen bytes (maxlen==0 means transfer until starvation) + // transfer at most maxlen bytes // return number of transfered bytes // generic implementation using arduino virtual API // (also available: virtual-less template streamMove(from,to)) - virtual size_t streamTo (Print& to, size_t maxLen = 0); + // - maxLen==0 will transfer until read starvation or destination temporarily full + // - readUntilChar: using anything in 0..255 will stop transfer when this char is read + // - timeout_ms==0: means immediate exit if nothing more can be read even if maxLen is not reached + + virtual size_t streamTo (Print& to, size_t maxLen = 0, int readUntilChar = -1, unsigned long timeout_ms = 0); //////////////////// end of extensions @@ -147,51 +152,50 @@ class Stream: public Print { // this allows format characters (typically commas) in values to be ignored float parseFloat(char skipChar); // as above but the given skipChar is ignored - - char oneChar; }; #define STREAM_MOVE(from,to,...) (streamMove(from, to, ## __VA_ARGS__)) -#include #include template -size_t streamMove (Tfrom& from, Tto& to, size_t maxLen = 0, int readUntilChar = -1, TimeoutMs::timeType timeout = TimeoutMs::alwaysExpired) + typename TimeoutMs = esp8266::polledTimeout::oneShotMs> +size_t streamMove (Tfrom& from, Tto& to, size_t maxLen = 0, int readUntilChar = -1, TimeoutMs::timeType timeout = TimeoutMs::alwayExpired) { PeriodicMs yieldNow(100); size_t written = 0; size_t w; - if (timeout == TimeoutMs::alwaysExpired) - // default value => take Stream (source) value + if (timeout == TimeoutMs::neverExpires) + // neverExpires forbidden value => take Stream (source) value // TODO: check for to::getTimeout() existence (w/ SFINAE or equiv.) - // and use in that case std::min(from.getTimeout(), to.getTimeout()) + // and use std::min(from.getTimeout(), to.getTimeout()) timeout.reset(from.getTimeout()); else timeout.reset(); - while (!timeout && (!maxLen || written < maxLen) && (w = to.availableForWrite())) + while ((!maxLen || written < maxLen) && ((w = to.availableForWrite()) || !timeout) { - const char* directbuf = from.peekBuffer(); size_t r = from.availableForPeek(); if (w > r) w = r; - if (readUntilChar != -1) + if (w) { - const char* last = memchr(directbuf, readUntilChar, w); - if (last) - w = last - directbuf + 1; + const char* directbuf = from.peekBuffer(); + if (readUntilChar >= 0) + { + const char* last = memchr(directbuf, readUntilChar, w); + if (last) + w = last - directbuf + 1; + } + if (w && ((w = to.write(directbuf, w))) + { + from.peekConsume(w); + written += w; + timeout.reset(); + } } - if (w && ((w = to.write(directbuf, w))) - { - from.peekConsume(w); - written += w; - timeout.reset(); - } - if (yieldNow) yield(); } diff --git a/cores/esp8266/debug.h b/cores/esp8266/debug.h index 02b30a4c61..debbabda38 100644 --- a/cores/esp8266/debug.h +++ b/cores/esp8266/debug.h @@ -27,16 +27,16 @@ void __panic_func(const char* file, int line, const char* func) __attribute__((n #ifdef DEBUG_ESP_CORE extern const char* overrideme PROGMEM; -#define IAMSLOW(str) \ +#define IAMSLOW() \ do { \ static bool once = false;\ if (!once) { \ once = true; \ - DEBUGV((PGM_P)PSTR(str "%s", overrideme)); \ + DEBUGV((PGM_P)PSTR(__FUNCTION__ "%s", overrideme)); \ } \ } while (0) #else -#define IAMSLOW(str) do { (void)0; } while (0) +#define IAMSLOW() do { (void)0; } while (0) #endif #ifdef __cplusplus From acc2e246a61a0e785c90cb9f461473b75818077b Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 11 May 2019 09:04:34 +0200 Subject: [PATCH 024/207] wip --- cores/esp8266/Stream.h | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 187f9055c5..79e3ddd3e7 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -120,13 +120,13 @@ class Stream: public Print { //////////////////// extensions: // addition: directly access to read buffer - // (currently implemented in HardwareSerial:: and WiFiClient::) + // (primarily implemented in HardwareSerial:: and WiFiClient::) // return number of byte accessible by peekBuffer() - virtual size_t availableForPeek () { IAMSLOW(); return (oneChar = peek()) == -1? 0: 1; } + virtual size_t availableForPeek () { IAMSLOW(); int test = peek(); oneChar = test; return test == -1? 0: 1; } // return a pointer to available data buffer (size = availableForPeek()) - // semantic forbids any kind of read() before calling peekConsume() + // semantic forbids any kind of read() after calling peekBuffer() and before calling peekConsume() virtual const char* peekBuffer () { return &oneChar; } // consume bytes after peekBuffer use @@ -158,24 +158,22 @@ class Stream: public Print { #include -template -size_t streamMove (Tfrom& from, Tto& to, size_t maxLen = 0, int readUntilChar = -1, TimeoutMs::timeType timeout = TimeoutMs::alwayExpired) + typename TimeoutMs = esp8266::polledTimeout::oneShotFastMs> +size_t streamMove (streamT& from, printT& to, size_t maxLen = 0, int readUntilChar = -1, TimeoutMs::timeType timeout = TimeoutMs::alwaysExpired) { PeriodicMs yieldNow(100); + TimeoutMS timedOut(timeout); size_t written = 0; size_t w; if (timeout == TimeoutMs::neverExpires) - // neverExpires forbidden value => take Stream (source) value - // TODO: check for to::getTimeout() existence (w/ SFINAE or equiv.) - // and use std::min(from.getTimeout(), to.getTimeout()) - timeout.reset(from.getTimeout()); + timedOut.reset(from.getTimeout()); else - timeout.reset(); + timedOut.reset(); - while ((!maxLen || written < maxLen) && ((w = to.availableForWrite()) || !timeout) + while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) { size_t r = from.availableForPeek(); if (w > r) @@ -187,15 +185,23 @@ size_t streamMove (Tfrom& from, Tto& to, size_t maxLen = 0, int readUntilChar = { const char* last = memchr(directbuf, readUntilChar, w); if (last) - w = last - directbuf + 1; + { + size_t remain = last - directbuf + 1; + if (w > remain) + w = remain; + if (maxlen && written + w > maxlen) + w = maxlen - written; + } } if (w && ((w = to.write(directbuf, w))) { from.peekConsume(w); written += w; - timeout.reset(); + timedOut.reset(); } } + if (w == 0 && timedOut) + break; if (yieldNow) yield(); } From cbc0070c6de06eb7ea258d920e48107ef78134f7 Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Thu, 12 Sep 2019 16:04:23 +0200 Subject: [PATCH 025/207] merge continuation --- cores/esp8266/PolledTimeout.h | 6 +++--- tests/host/common/MockEsp.cpp | 7 +------ tests/host/common/mock.h | 1 - 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/cores/esp8266/PolledTimeout.h b/cores/esp8266/PolledTimeout.h index 14dd06497d..fe9c9ca4bc 100644 --- a/cores/esp8266/PolledTimeout.h +++ b/cores/esp8266/PolledTimeout.h @@ -70,12 +70,12 @@ struct TimeSourceMillis struct TimeSourceCycles { - // time policy based on espGetCycleCount() + // time policy based on ESP.getCycleCount() // this particular time measurement is intended to be called very often // (every loop, every yield) - using timeType = decltype(espGetCycleCount()); - static timeType time() {return espGetCycleCount();} + using timeType = decltype(ESP.getCycleCount()); + static timeType time() {return ESP.getCycleCount();} static constexpr timeType ticksPerSecond = F_CPU; // 80'000'000 or 160'000'000 Hz static constexpr timeType ticksPerSecondMax = 160000000; // 160MHz }; diff --git a/tests/host/common/MockEsp.cpp b/tests/host/common/MockEsp.cpp index 78736e7ab4..8ff16cd01a 100644 --- a/tests/host/common/MockEsp.cpp +++ b/tests/host/common/MockEsp.cpp @@ -220,14 +220,9 @@ void EspClass::resetFreeContStack() { } -uint32_t espGetCycleCount() +uint32_t EspClass::getCycleCount() { timeval t; gettimeofday(&t, NULL); return (((uint64_t)t.tv_sec) * 1000000 + t.tv_usec) * (F_CPU / 1000000); } - -uint32_t EspClass::getCycleCount() -{ - return espGetCycleCount(); -} diff --git a/tests/host/common/mock.h b/tests/host/common/mock.h index 46c8a8cf84..8a4e9e1c3e 100644 --- a/tests/host/common/mock.h +++ b/tests/host/common/mock.h @@ -85,7 +85,6 @@ typedef uint32_t uint32; // #include -uint32_t espGetCycleCount(); #include From 16cb3a224635d0f40489c6459fc8f4a609ec2e13 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 16 Nov 2019 00:11:51 +0100 Subject: [PATCH 026/207] wip --- cores/esp8266/FS.cpp | 2 +- cores/esp8266/FS.h | 2 +- cores/esp8266/HardwareSerial.h | 7 +- cores/esp8266/Print.h | 3 +- cores/esp8266/Stream.cpp | 8 +-- cores/esp8266/Stream.h | 64 +++++++++---------- cores/esp8266/debug.cpp | 2 +- cores/esp8266/debug.h | 4 +- .../src/ESP8266HTTPClient.cpp | 7 +- libraries/ESP8266WiFi/src/WiFiClient.h | 5 +- 10 files changed, 49 insertions(+), 55 deletions(-) diff --git a/cores/esp8266/FS.cpp b/cores/esp8266/FS.cpp index bcfbb8e814..fa841c64fa 100644 --- a/cores/esp8266/FS.cpp +++ b/cores/esp8266/FS.cpp @@ -58,7 +58,7 @@ int File::read() { return result; } -int File::read(uint8_t* buf, size_t size) { +size_t File::read(uint8_t* buf, size_t size) { if (!_p) return -1; diff --git a/cores/esp8266/FS.h b/cores/esp8266/FS.h index 757bc2355d..23d05bad83 100644 --- a/cores/esp8266/FS.h +++ b/cores/esp8266/FS.h @@ -66,7 +66,7 @@ class File : public Stream size_t readBytes(char *buffer, size_t length) override { return read((uint8_t*)buffer, length); } - int read(uint8_t* buf, size_t size); + size_t read(uint8_t* buf, size_t size); bool seek(uint32_t pos, SeekMode mode); bool seek(uint32_t pos) { return seek(pos, SeekSet); diff --git a/cores/esp8266/HardwareSerial.h b/cores/esp8266/HardwareSerial.h index cc57b48022..bf54c3e459 100644 --- a/cores/esp8266/HardwareSerial.h +++ b/cores/esp8266/HardwareSerial.h @@ -131,9 +131,6 @@ class HardwareSerial: public Stream return uart_peek_char(_uart); } - // allow optimization for streamMove/STREAM_MOVE - static constexpr bool peekBufferAvailableAPI () { return true; } - // return a pointer to available data buffer (size = available()) // semantic forbids any kind of read() before calling peekConsume() const char* peekBuffer () override @@ -159,11 +156,11 @@ class HardwareSerial: public Stream return uart_read_char(_uart); } // ::read(buffer, size): same as readBytes without timeout - int read(char* buffer, size_t size) override + size_t read(char* buffer, size_t size) override { return uart_read(_uart, buffer, size); } - int read(uint8_t* buffer, size_t size) override + size_t read(uint8_t* buffer, size_t size) override { return uart_read(_uart, (char*)buffer, size); } diff --git a/cores/esp8266/Print.h b/cores/esp8266/Print.h index 46325be798..990b67cb23 100644 --- a/cores/esp8266/Print.h +++ b/cores/esp8266/Print.h @@ -102,9 +102,8 @@ class Print { virtual void flush() { /* Empty implementation for backward compatibility */ } - // availableForWrite() must be overridden when possible ! - // overridden by HardwareSerial and WiFiClient when added in Print:: virtual int availableForWrite(); + // return type: int }; #endif diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index e2dd34db72..8368398b1a 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -22,7 +22,6 @@ #include #include -#include #include #define PARSE_TIMEOUT 1000 // default number of milli-seconds to wait @@ -262,7 +261,7 @@ String Stream::readStringUntil(char terminator) { return ret; } -int Stream::read (char* buffer, size_t maxLen) +size_t Stream::read (char* buffer, size_t maxLen) { IAMSLOW("Stream::read(buffer,len)"); @@ -272,7 +271,8 @@ int Stream::read (char* buffer, size_t maxLen) return nbread; } -size_t Stream::streamTo (Print& to, size_t maxLen = 0, int readUntilChar = -1, unsigned long timeout_ms = 0) +size_t Stream::streamTo (Print& to, unsigned long timeout_ms, size_t maxLen, int readUntilChar) { - return streamMove(*this, to, maxLen, readUntilChar, timeout_ms); + //return streamMove(*this, to, timeout_ms, maxLen, readUntilChar); + return streamMove(*this, to, timeout_ms, maxLen, readUntilChar); } diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 79e3ddd3e7..b1fe97c450 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -89,13 +89,13 @@ class Stream: public Print { float parseFloat(); // float version of parseInt - // both arduino's "size_t Stream::readBytes(char*/uint8_t*, size_t): virtual size_t readBytes(char *buffer, size_t length); // read chars from stream into buffer virtual size_t readBytes(uint8_t *buffer, size_t length) { return readBytes((char *) buffer, length); } // terminates if length characters have been read or timeout (see setTimeout) // returns the number of characters placed in the buffer (0 means no valid data found) + // return data type: size_t size_t readBytesUntil(char terminator, char *buffer, size_t length); // as readBytes with terminator character size_t readBytesUntil(char terminator, uint8_t *buffer, size_t length) { @@ -108,41 +108,34 @@ class Stream: public Print { virtual String readString(); String readStringUntil(char terminator); - // additions: Client:: read() definitions moved here, because HardwareSerial have them too - + // (Client::read() definitions moved here - HardwareSerial have them too) // read at most maxLen bytes: // returns effectively transfered bytes (can be less than maxLen) // immediate return when no more data are available (no timeout) - virtual int read(char* buffer, size_t maxLen); - // arduino's "int Client::read(uint8_t*,size_t)": - virtual int read(uint8_t* buffer, size_t maxLen) { return read((char*)buffer, maxLen); } - - //////////////////// extensions: + virtual size_t read(char* buffer, size_t maxLen); + virtual size_t read(uint8_t* buffer, size_t maxLen) { return read((char*)buffer, maxLen); } + // return data type: int - // addition: directly access to read buffer - // (primarily implemented in HardwareSerial:: and WiFiClient::) + //////////////////// extensions: direct access to input buffer - // return number of byte accessible by peekBuffer() - virtual size_t availableForPeek () { IAMSLOW(); int test = peek(); oneChar = test; return test == -1? 0: 1; } + // return number of byte accessible by peekBuffer(), can be 0 + virtual size_t availableForPeek () { IAMSLOW(); int test = peek(); oneChar = (char)test; return test == -1? 0: 1; } // return a pointer to available data buffer (size = availableForPeek()) // semantic forbids any kind of read() after calling peekBuffer() and before calling peekConsume() virtual const char* peekBuffer () { return &oneChar; } // consume bytes after peekBuffer use - virtual void peekConsume (size_t consume) { (void)read(); } - - // additions: streamTo + virtual void peekConsume (size_t consume) { if (consume) read(); } - // transfer at most maxlen bytes - // return number of transfered bytes - // generic implementation using arduino virtual API + // streamTo(): + // transfer from Stream:: to Print:: at most maxlen bytes and return number of transfered bytes + // this is the generic implementation using arduino virtual API // (also available: virtual-less template streamMove(from,to)) - // - maxLen==0 will transfer until read starvation or destination temporarily full - // - readUntilChar: using anything in 0..255 will stop transfer when this char is read - // - timeout_ms==0: means immediate exit if nothing more can be read even if maxLen is not reached - - virtual size_t streamTo (Print& to, size_t maxLen = 0, int readUntilChar = -1, unsigned long timeout_ms = 0); + // - timeout_ms==TimeoutMs::neverExpires: use getTimeout() (when 0: take what's available and immediate return) + // - maxLen==0 will transfer until input starvation or saturated output + // - readUntilChar: setting anything in 0..255 will stop transfer when this char is read (not included) + virtual size_t streamTo (Print& to, unsigned long timeout_ms = getTimeout(), size_t maxLen = 0, int readUntilChar = -1); //////////////////// end of extensions @@ -154,25 +147,28 @@ class Stream: public Print { float parseFloat(char skipChar); // as above but the given skipChar is ignored }; -#define STREAM_MOVE(from,to,...) (streamMove(from, to, ## __VA_ARGS__)) - #include +#if 0 template -size_t streamMove (streamT& from, printT& to, size_t maxLen = 0, int readUntilChar = -1, TimeoutMs::timeType timeout = TimeoutMs::alwaysExpired) +size_t streamMove (streamT& from, printT& to, TimeoutMs::timeType timeout = TimeoutMs::neverExpires, size_t maxLen = 0, int readUntilChar = -1) +#else +template <> +size_t streamMove + //(streamT& from, printT& to, TimeoutMs::timeType timeout = from.getTimeout()/*TimeoutMs::neverExpires*/, size_t maxLen = 0, int readUntilChar = -1) + (streamT& from, printT& to, TimeoutMs::timeType timeout = from.getTimeout(), size_t maxLen = 0, int readUntilChar = -1) +#endif { PeriodicMs yieldNow(100); - TimeoutMS timedOut(timeout); + //TimeoutMs timedOut(timeout == TimeoutMs::neverExpires? from.getTimeout(): timeout); + TimeoutMs timedOut(timeout); size_t written = 0; size_t w; - if (timeout == TimeoutMs::neverExpires) - timedOut.reset(from.getTimeout()); - else - timedOut.reset(); - while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) { size_t r = from.availableForPeek(); @@ -189,10 +185,10 @@ size_t streamMove (streamT& from, printT& to, size_t maxLen = 0, int readUntilCh size_t remain = last - directbuf + 1; if (w > remain) w = remain; - if (maxlen && written + w > maxlen) - w = maxlen - written; } } + if (maxlen && written + w > maxlen) + w = maxlen - written; if (w && ((w = to.write(directbuf, w))) { from.peekConsume(w); diff --git a/cores/esp8266/debug.cpp b/cores/esp8266/debug.cpp index 82f833e640..73b73499c0 100644 --- a/cores/esp8266/debug.cpp +++ b/cores/esp8266/debug.cpp @@ -21,7 +21,7 @@ #include "Arduino.h" #include "debug.h" -const char* overrideme PROGMEM = " should be overridden for better efficiency\r\n"; +const char* overrideme PROGMEM = "%s should be overridden for better efficiency\r\n"; void ICACHE_RAM_ATTR hexdump(const void *mem, uint32_t len, uint8_t cols) { const uint8_t* src = (const uint8_t*) mem; diff --git a/cores/esp8266/debug.h b/cores/esp8266/debug.h index debbabda38..b1bf9efed5 100644 --- a/cores/esp8266/debug.h +++ b/cores/esp8266/debug.h @@ -29,10 +29,10 @@ void __panic_func(const char* file, int line, const char* func) __attribute__((n extern const char* overrideme PROGMEM; #define IAMSLOW() \ do { \ - static bool once = false;\ + static bool once = false; \ if (!once) { \ once = true; \ - DEBUGV((PGM_P)PSTR(__FUNCTION__ "%s", overrideme)); \ + DEBUGV(overrideme, (PGM_P)PSTR(__FUNCTION__)); \ } \ } while (0) #else diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 6f4af6128c..24442eaa10 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -759,9 +759,14 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) #if 1 size_t transferred = 0; + while (connected() && transferred < size) + transferred += streamMove(*stream, *_client, stream->getTimeout(), size - transferred); + + if (transferred < size) { - transferred += stream->streamTo(*_client, size - transferred); + DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] short write, asked for %d but got %d failed.\n", size, transferred); + return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); } #else diff --git a/libraries/ESP8266WiFi/src/WiFiClient.h b/libraries/ESP8266WiFi/src/WiFiClient.h index 4226760942..82798d8a56 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.h +++ b/libraries/ESP8266WiFi/src/WiFiClient.h @@ -66,7 +66,7 @@ class WiFiClient : public Client, public SList { virtual int available() override; virtual int read() override; - virtual int read(uint8_t *buf, size_t size) override; + virtual int read(char *buf, size_t size) override; virtual int peek() override; virtual size_t peekBytes(uint8_t *buffer, size_t length); size_t peekBytes(char *buffer, size_t length) { @@ -120,9 +120,6 @@ class WiFiClient : public Client, public SList { bool getSync() const; void setSync(bool sync); - // allow optimization for streamMove/STREAM_MOVE - static constexpr bool peekBufferAvailableAPI () { return true; } - // return number of byte accessible by peekBuffer() virtual size_t availableForPeek () override; From cab509c2df200e4ef921b23f77cdf915447f193b Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 16 Nov 2019 01:15:00 +0100 Subject: [PATCH 027/207] wip --- libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h | 10 ---------- libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h | 12 ------------ 2 files changed, 22 deletions(-) diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h b/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h index bfb99d06ad..8b525e753f 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h @@ -84,18 +84,8 @@ class WiFiClientSecure : public WiFiClient { } // disallow optimization for streamMove/STREAM_MOVE - static constexpr bool peekBufferAvailableAPI () { return false; } - - // return a pointer to available data buffer (size = availableForPeek()) - // semantic forbids any kind of read() before calling peekConsume() virtual const char* peekBuffer () { return nullptr; } - // return number of byte accessible by peekBuffer() - virtual size_t availableForPeek () { return 0; } - - // consume bytes after use (see peekBuffer) - virtual void peekConsume (size_t consume) { (void)consume; } - friend class WiFiServerSecure; // Needs access to custom constructor below protected: // Only called by WiFiServerSecure diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h index 4d7c5766a7..e05b9cd261 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h @@ -131,21 +131,9 @@ class WiFiClientSecure : public WiFiClient { static bool probeMaxFragmentLength(const char *hostname, uint16_t port, uint16_t len); static bool probeMaxFragmentLength(const String& host, uint16_t port, uint16_t len); - // peek buffer not implemented yet: - // disallow optimization for streamMove/STREAM_MOVE - static constexpr bool peekBufferAvailableAPI () { return false; } - - // return a pointer to available data buffer (size = availableForPeek()) - // semantic forbids any kind of read() before calling peekConsume() virtual const char* peekBuffer () { return nullptr; } - // return number of byte accessible by peekBuffer() - virtual size_t availableForPeek () { return 0; } - - // consume bytes after use (see peekBuffer) - virtual void peekConsume (size_t consume) { (void)consume; } - //////////////////////////////////////////////////// // AxTLS API deprecated warnings to help upgrading From 076a5eeb2f53d2e2c63ad7e4aa16cd6c70a40ba5 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 17 Nov 2019 22:38:12 +0100 Subject: [PATCH 028/207] wip --- cores/esp8266/Stream.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index b1fe97c450..ba47552d98 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -118,7 +118,7 @@ class Stream: public Print { //////////////////// extensions: direct access to input buffer - // return number of byte accessible by peekBuffer(), can be 0 + // return number of byte accessible by peekBuffer() virtual size_t availableForPeek () { IAMSLOW(); int test = peek(); oneChar = (char)test; return test == -1? 0: 1; } // return a pointer to available data buffer (size = availableForPeek()) From 04fdf7640247c6668e8a73fab6c90d255fe9a8b9 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 20 Nov 2019 01:46:32 +0100 Subject: [PATCH 029/207] wip --- cores/esp8266/CircularBuffer.h | 174 --------------------------------- cores/esp8266/HardwareSerial.h | 5 + cores/esp8266/Stream.cpp | 91 ++++++++++++++++- cores/esp8266/Stream.h | 82 +++------------- 4 files changed, 108 insertions(+), 244 deletions(-) delete mode 100644 cores/esp8266/CircularBuffer.h diff --git a/cores/esp8266/CircularBuffer.h b/cores/esp8266/CircularBuffer.h deleted file mode 100644 index 4f3bc19d0b..0000000000 --- a/cores/esp8266/CircularBuffer.h +++ /dev/null @@ -1,174 +0,0 @@ -/* - CircularBuffer.h - yet another circular buffer, - compatible with Arduino Stream transfer API - Copyright (c) 2019 david gauchard. All right reserved. - - This library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with this library; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - */ - -#include - -// CONSTst (CONST-for-stream) should be 'const', -// Stream&Print should be updated accordingly (or Ardui-not). -#define CONSTst //const - -template -class CircularBuffer: public Stream -{ -protected: - - idx_t m_posr, m_posw; - uint8_t m_buffer[m_len]; - bool m_full; - - void inc_posr (idx_t contig = 1) - { - if ((m_posr += contig) == m_len) - m_posr = 0; - m_full = false; - } - - void inc_posw (idx_t contig = 1) - { - if ((m_posw += contig) == m_len) - m_posw = 0; - if (m_posw == m_posr) - m_full = true; - } - -public: - - CircularBuffer (): m_posr(0), m_posw(0), m_full(false) { } - - size_t length () const - { - return m_len; - } - - bool full () const - { - return m_full; - } - - bool empty () const - { - return m_posw == m_posr && !m_full; - } - - // char/byte oriented - - int peek () CONSTst override - { - return empty()? -1 : m_buffer[m_posr]; - } - - int read () override - { - if (empty()) - return -1; - int c = m_buffer[m_posr]; - inc_posr(); - return c; - } - - size_t write (uint8_t c) override - { - if (full()) - return 0; - m_buffer[m_posw] = c; - inc_posw(); - return 1; - } - - idx_t availableForWrite () CONSTst override - { - if (full()) - return 0; - if (m_posr > m_posw) - return m_posr - m_posw; - return m_len - m_posw; - } - - idx_t available () CONSTst override - { - if (full()) - return m_len; - if (m_posr > m_posw) - return m_len - m_posr; - return m_posw - m_posr; - } - - // buffer oriented - - size_t peek (uint8_t* buffer, size_t len) const /*override*/ - { - if (empty()) - return 0; - idx_t chunk1 = availableForRead(); - if (chunk1 > len) - chunk1 = len; - memcpy(buffer, &m_buffer[m_posr], chunk1); - return chunk1; - } - - size_t read (char* buffer, size_t maxLen) override - { - idx_t copied = 0; - while (copied < maxLen) - { - idx_t contig = availableForRead(); - if (!contig) - break; - if (contig > maxLen - copied) - contig = maxLen - copied; - memcpy(buffer + copied, &m_buffer[m_posr], contig); - copied += contig; - inc_posr(contig); - } - return copied; - } - - size_t write (const uint8_t* buffer, size_t maxLen) override - { - idx_t copied = 0; - while (copied < maxLen) - { - idx_t contig = availableForWrite(); - if (!contig) - break; - if (contig > maxLen - copied) - contig = maxLen - copied; - memcpy(&m_buffer[m_posw], buffer + copied, contig); - copied += contig; - inc_posw(contig); - } - return copied; - } - - virtual const char* peekBuffer () override - { - return &m_buffer[m_posr]; - } - - virtual void peekConsume (size_t consume) override - { -#ifdef DEBUG_ESP_CORE - if (consume > available()) - DEBUGV("consume overflow\r\n"); -#endif - inc_posr(consume); - } - -}; diff --git a/cores/esp8266/HardwareSerial.h b/cores/esp8266/HardwareSerial.h index bf54c3e459..96a947ffd0 100644 --- a/cores/esp8266/HardwareSerial.h +++ b/cores/esp8266/HardwareSerial.h @@ -131,6 +131,11 @@ class HardwareSerial: public Stream return uart_peek_char(_uart); } + bool peekBufferAPI () override + { + return true; + } + // return a pointer to available data buffer (size = available()) // semantic forbids any kind of read() before calling peekConsume() const char* peekBuffer () override diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index 8368398b1a..81865899b8 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -271,8 +271,93 @@ size_t Stream::read (char* buffer, size_t maxLen) return nbread; } -size_t Stream::streamTo (Print& to, unsigned long timeout_ms, size_t maxLen, int readUntilChar) +size_t Stream::to (Print& to, + esp8266::polledTimeout::oneShotFastMs::timeType timeout = from.getTimeout(), + size_t maxLen = 0, + int readUntilChar = -1) { - //return streamMove(*this, to, timeout_ms, maxLen, readUntilChar); - return streamMove(*this, to, timeout_ms, maxLen, readUntilChar); + esp8266::polledTimeout::periodicFastMs yieldNow(100); + esp8266::polledTimeout::oneShotFastMs timedOut(timeout); + size_t written = 0; + size_t w; + + if (peekBufferAPI()) + + // peek-buffer API + + while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) + { + size_t r = availableForPeek(); + if (w > r) + w = r; + if (w) + { + const char* directbuf = peekBuffer(); + if (readUntilChar >= 0) + { + const char* last = memchr(directbuf, readUntilChar, w); + if (last) + { + size_t remain = last - directbuf + 1; + w = std::min(remain, w); + } + } + if (maxlen && written + w > maxlen) + w = maxlen - written; + if (w && ((w = to.write(directbuf, w))) + { + from.peekConsume(w); + written += w; + timedOut.reset(); + } + } + if (w == 0 && timedOut) + break; + if (yieldNow) + yield(); + } + + else if (readUntilChar >= 0) + + // regular Stream API + // no other choice than reading byte by byte + + while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) + { + char c; + size_t r = read(&c, 1); + if (r) + { + if (c == readUntilChar) + break; + w = to.write(&c, 1); + assert(w); + written += 1; + timedOut.reset(); + } + else if (timedOut) + break; + } + + else + + // regular Stream API + // use an intermediary buffer + + while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) + { + w = std::min(w, available()); + w = std::min(w, 64); + char temp[w]; + size_t r = read(temp, w); + w = to.write(temp, r); + assert(r == w); + written += w; + if (w) + timedOut.reset(); + else if (timedOut) + break; + } + + return written; } diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index ba47552d98..6ac5bc3f9f 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -43,7 +43,6 @@ class Stream: public Print { int timedRead(); // private method to read stream with timeout int timedPeek(); // private method to peek stream with timeout int peekNextDigit(); // returns the next numeric digit in the stream or -1 if timeout - char oneChar; // used by direct access extension public: virtual int available() = 0; @@ -117,25 +116,32 @@ class Stream: public Print { // return data type: int //////////////////// extensions: direct access to input buffer + + // inform user or ::to() on effective implementation + virtual bool peekBufferAPI () { return false; } // return number of byte accessible by peekBuffer() - virtual size_t availableForPeek () { IAMSLOW(); int test = peek(); oneChar = (char)test; return test == -1? 0: 1; } + virtual size_t availableForPeek () { return 0; } // return a pointer to available data buffer (size = availableForPeek()) // semantic forbids any kind of read() after calling peekBuffer() and before calling peekConsume() - virtual const char* peekBuffer () { return &oneChar; } + virtual const char* peekBuffer () { return nullptr; } // consume bytes after peekBuffer use - virtual void peekConsume (size_t consume) { if (consume) read(); } + virtual void peekConsume (size_t consume) { (void)consume; } - // streamTo(): + //////////////////// extensions: Stream streams + + // Stream::to() // transfer from Stream:: to Print:: at most maxlen bytes and return number of transfered bytes - // this is the generic implementation using arduino virtual API - // (also available: virtual-less template streamMove(from,to)) + // (uses 1-copy peekBuffer API when available, or transfer through a 2-copy local stack space) // - timeout_ms==TimeoutMs::neverExpires: use getTimeout() (when 0: take what's available and immediate return) // - maxLen==0 will transfer until input starvation or saturated output - // - readUntilChar: setting anything in 0..255 will stop transfer when this char is read (not included) - virtual size_t streamTo (Print& to, unsigned long timeout_ms = getTimeout(), size_t maxLen = 0, int readUntilChar = -1); + // - readUntilChar: setting anything in 0..255 will stop transfer when this char is read (swallowed, not copied) + size_t to (Print& to, + esp8266::polledTimeout::oneShotFastMs::timeType timeout = from.getTimeout(), + size_t maxLen = 0, + int readUntilChar = -1); //////////////////// end of extensions @@ -147,62 +153,4 @@ class Stream: public Print { float parseFloat(char skipChar); // as above but the given skipChar is ignored }; -#include - -#if 0 -template -size_t streamMove (streamT& from, printT& to, TimeoutMs::timeType timeout = TimeoutMs::neverExpires, size_t maxLen = 0, int readUntilChar = -1) -#else -template <> -size_t streamMove - //(streamT& from, printT& to, TimeoutMs::timeType timeout = from.getTimeout()/*TimeoutMs::neverExpires*/, size_t maxLen = 0, int readUntilChar = -1) - (streamT& from, printT& to, TimeoutMs::timeType timeout = from.getTimeout(), size_t maxLen = 0, int readUntilChar = -1) -#endif -{ - PeriodicMs yieldNow(100); - //TimeoutMs timedOut(timeout == TimeoutMs::neverExpires? from.getTimeout(): timeout); - TimeoutMs timedOut(timeout); - size_t written = 0; - size_t w; - - while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) - { - size_t r = from.availableForPeek(); - if (w > r) - w = r; - if (w) - { - const char* directbuf = from.peekBuffer(); - if (readUntilChar >= 0) - { - const char* last = memchr(directbuf, readUntilChar, w); - if (last) - { - size_t remain = last - directbuf + 1; - if (w > remain) - w = remain; - } - } - if (maxlen && written + w > maxlen) - w = maxlen - written; - if (w && ((w = to.write(directbuf, w))) - { - from.peekConsume(w); - written += w; - timedOut.reset(); - } - } - if (w == 0 && timedOut) - break; - if (yieldNow) - yield(); - } - - return written; -} - #endif From 489f0ebd966a87c0bceab3043d806b5a3f735ca3 Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Wed, 20 Nov 2019 12:39:10 +0100 Subject: [PATCH 030/207] wip --- .gitmodules | 6 +++ cores/esp8266/Arduino.h | 5 -- cores/esp8266/HardwareSerial.h | 6 +-- cores/esp8266/PolledTimeout.h | 19 +++---- cores/esp8266/Stream.cpp | 51 +++++++++++-------- cores/esp8266/Stream.h | 13 +++-- cores/esp8266/core_esp8266_features.h | 5 ++ cores/esp8266/debug.cpp | 7 ++- cores/esp8266/debug.h | 4 +- cores/esp8266/uart.cpp | 5 ++ libraries/ESP8266WiFi/src/WiFiClient.cpp | 4 ++ libraries/ESP8266WiFi/src/WiFiClient.h | 9 +++- .../ESP8266WiFi/src/WiFiClientSecureAxTLS.h | 4 +- .../ESP8266WiFi/src/WiFiClientSecureBearSSL.h | 4 +- 14 files changed, 92 insertions(+), 50 deletions(-) diff --git a/.gitmodules b/.gitmodules index ca26e7d93b..1814ac7468 100644 --- a/.gitmodules +++ b/.gitmodules @@ -19,3 +19,9 @@ [submodule "tools/esptool"] path = tools/esptool url = https://github.com/espressif/esptool.git +[submodule "libraries/Ethernet"] + path = libraries/Ethernet + url = https://github.com/d-a-v/Ethernet.git +[submodule "Ethernet"] + path = libraries/Ethernet + url = https://github.com/d-a-v/Ethernet.git diff --git a/cores/esp8266/Arduino.h b/cores/esp8266/Arduino.h index 54919ba100..8948181594 100644 --- a/cores/esp8266/Arduino.h +++ b/cores/esp8266/Arduino.h @@ -184,11 +184,6 @@ void analogWrite(uint8_t pin, int val); void analogWriteFreq(uint32_t freq); void analogWriteRange(uint32_t range); -unsigned long millis(void); -unsigned long micros(void); -uint64_t micros64(void); -void delay(unsigned long); -void delayMicroseconds(unsigned int us); unsigned long pulseIn(uint8_t pin, uint8_t state, unsigned long timeout); unsigned long pulseInLong(uint8_t pin, uint8_t state, unsigned long timeout); diff --git a/cores/esp8266/HardwareSerial.h b/cores/esp8266/HardwareSerial.h index 96a947ffd0..e8bd4aeda5 100644 --- a/cores/esp8266/HardwareSerial.h +++ b/cores/esp8266/HardwareSerial.h @@ -131,7 +131,7 @@ class HardwareSerial: public Stream return uart_peek_char(_uart); } - bool peekBufferAPI () override + virtual bool peekBufferAPI () const override { return true; } @@ -161,11 +161,11 @@ class HardwareSerial: public Stream return uart_read_char(_uart); } // ::read(buffer, size): same as readBytes without timeout - size_t read(char* buffer, size_t size) override + size_t read(char* buffer, size_t size) // should override, see Stream.h { return uart_read(_uart, buffer, size); } - size_t read(uint8_t* buffer, size_t size) override + size_t read(uint8_t* buffer, size_t size) // should override, see Stream.h { return uart_read(_uart, (char*)buffer, size); } diff --git a/cores/esp8266/PolledTimeout.h b/cores/esp8266/PolledTimeout.h index fe9c9ca4bc..96398aa4ac 100644 --- a/cores/esp8266/PolledTimeout.h +++ b/cores/esp8266/PolledTimeout.h @@ -23,9 +23,10 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ -#include - -#include +#include // std::numeric_limits<> +#include // std::is_unsigned<> +#include // IRAM_ATTR +#include // esp_get_cycle_count(), delay() namespace esp8266 { @@ -70,12 +71,12 @@ struct TimeSourceMillis struct TimeSourceCycles { - // time policy based on ESP.getCycleCount() + // time policy based on esp_get_cycle_count() // this particular time measurement is intended to be called very often // (every loop, every yield) - using timeType = decltype(ESP.getCycleCount()); - static timeType time() {return ESP.getCycleCount();} + using timeType = decltype(esp_get_cycle_count()); + static timeType time() {return esp_get_cycle_count();} static constexpr timeType ticksPerSecond = F_CPU; // 80'000'000 or 160'000'000 Hz static constexpr timeType ticksPerSecondMax = 160000000; // 160MHz }; @@ -259,14 +260,14 @@ using periodic = polledTimeout::timeoutTemplate /*__attribute__((deprecate using oneShotMs = polledTimeout::timeoutTemplate; using periodicMs = polledTimeout::timeoutTemplate; -// Time policy based on ESP.getCycleCount(), and intended to be called very often: +// Time policy based on esp_get_cycle_count(), and intended to be called very often: // "Fast" versions sacrifices time range for improved precision and reduced execution time (by 86%) -// (cpu cycles for ::expired(): 372 (millis()) vs 52 (ESP.getCycleCount())) +// (cpu cycles for ::expired(): 372 (millis()) vs 52 (esp_get_cycle_count())) // timeMax() values: // Ms: max is 26843 ms (26.8 s) // Us: max is 26843545 us (26.8 s) // Ns: max is 1073741823 ns ( 1.07 s) -// (time policy based on ESP.getCycleCount() is intended to be called very often) +// (time policy based on esp_get_cycle_count() is intended to be called very often) using oneShotFastMs = polledTimeout::timeoutTemplate; using periodicFastMs = polledTimeout::timeoutTemplate; diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index 81865899b8..26892a4f0f 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -22,7 +22,7 @@ #include #include -#include +#include #define PARSE_TIMEOUT 1000 // default number of milli-seconds to wait #define NO_SKIP_CHAR 1 // a magic char not found in a valid ASCII numeric field @@ -261,25 +261,30 @@ String Stream::readStringUntil(char terminator) { return ret; } +#if 0 size_t Stream::read (char* buffer, size_t maxLen) { - IAMSLOW("Stream::read(buffer,len)"); + IAMSLOW(); size_t nbread = 0; while (nbread < maxLen && available()) buffer[nbread++] = read(); return nbread; } +#endif size_t Stream::to (Print& to, - esp8266::polledTimeout::oneShotFastMs::timeType timeout = from.getTimeout(), - size_t maxLen = 0, - int readUntilChar = -1) + esp8266::polledTimeout::oneShotFastMs::timeType timeout, + size_t maxLen, + int readUntilChar) { esp8266::polledTimeout::periodicFastMs yieldNow(100); esp8266::polledTimeout::oneShotFastMs timedOut(timeout); size_t written = 0; size_t w; + + if (timeout == esp8266::polledTimeout::oneShotFastMs::neverExpires) + timeout = getTimeout(); if (peekBufferAPI()) @@ -287,31 +292,30 @@ size_t Stream::to (Print& to, while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) { - size_t r = availableForPeek(); - if (w > r) - w = r; + if (maxLen) + w = std::min(w, maxLen - written); + w = std::min(w, availableForPeek()); if (w) { const char* directbuf = peekBuffer(); + bool ignore = false; if (readUntilChar >= 0) { - const char* last = memchr(directbuf, readUntilChar, w); + const char* last = (const char*)memchr(directbuf, readUntilChar, w); if (last) { - size_t remain = last - directbuf + 1; - w = std::min(remain, w); + w = std::min((size_t)(last - directbuf + 1), w); + ignore = true; } } - if (maxlen && written + w > maxlen) - w = maxlen - written; - if (w && ((w = to.write(directbuf, w))) + if (w && ((w = to.write(directbuf, w)))) { - from.peekConsume(w); + peekConsume(w + ignore); written += w; timedOut.reset(); } } - if (w == 0 && timedOut) + else if (timedOut) break; if (yieldNow) yield(); @@ -324,19 +328,20 @@ size_t Stream::to (Print& to, while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) { - char c; - size_t r = read(&c, 1); - if (r) + int c = read(); + if (c != -1) { if (c == readUntilChar) break; - w = to.write(&c, 1); + w = to.write(c); assert(w); written += 1; timedOut.reset(); } else if (timedOut) break; + if (yieldNow) + yield(); } else @@ -346,8 +351,8 @@ size_t Stream::to (Print& to, while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) { - w = std::min(w, available()); - w = std::min(w, 64); + w = std::min(w, (size_t)available()); + w = std::min(w, 64U); char temp[w]; size_t r = read(temp, w); w = to.write(temp, r); @@ -357,6 +362,8 @@ size_t Stream::to (Print& to, timedOut.reset(); else if (timedOut) break; + if (yieldNow) + yield(); } return written; diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 6ac5bc3f9f..39da902d6b 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -25,6 +25,7 @@ #include #include #include "Print.h" +#include // compatibility macros for testing /* @@ -107,6 +108,11 @@ class Stream: public Print { virtual String readString(); String readStringUntil(char terminator); +#if 0 +// conflicting returned type: it is `int` in arduino's `Client::` +// and `size_t` in esp8266 API (serial, FS), so holding this back for now because +// changing every read()/write() `size_t` return type to `int` will be a breaking change + // (Client::read() definitions moved here - HardwareSerial have them too) // read at most maxLen bytes: // returns effectively transfered bytes (can be less than maxLen) @@ -114,11 +120,12 @@ class Stream: public Print { virtual size_t read(char* buffer, size_t maxLen); virtual size_t read(uint8_t* buffer, size_t maxLen) { return read((char*)buffer, maxLen); } // return data type: int +#endif //////////////////// extensions: direct access to input buffer - // inform user or ::to() on effective implementation - virtual bool peekBufferAPI () { return false; } + // inform user or ::to() on effective buffered peek API implementation + virtual bool peekBufferAPI () const { return false; } // return number of byte accessible by peekBuffer() virtual size_t availableForPeek () { return 0; } @@ -139,7 +146,7 @@ class Stream: public Print { // - maxLen==0 will transfer until input starvation or saturated output // - readUntilChar: setting anything in 0..255 will stop transfer when this char is read (swallowed, not copied) size_t to (Print& to, - esp8266::polledTimeout::oneShotFastMs::timeType timeout = from.getTimeout(), + esp8266::polledTimeout::oneShotFastMs::timeType timeout = esp8266::polledTimeout::oneShotFastMs::neverExpires, /* ->getTimeout() */ size_t maxLen = 0, int readUntilChar = -1); diff --git a/cores/esp8266/core_esp8266_features.h b/cores/esp8266/core_esp8266_features.h index d3b70f3dcc..1154ea1f80 100644 --- a/cores/esp8266/core_esp8266_features.h +++ b/cores/esp8266/core_esp8266_features.h @@ -111,6 +111,11 @@ extern "C" { #endif void precache(void *f, uint32_t bytes); +unsigned long millis(void); +unsigned long micros(void); +uint64_t micros64(void); +void delay(unsigned long); +void delayMicroseconds(unsigned int us); #ifdef __cplusplus } diff --git a/cores/esp8266/debug.cpp b/cores/esp8266/debug.cpp index 73b73499c0..8a13e7dbc3 100644 --- a/cores/esp8266/debug.cpp +++ b/cores/esp8266/debug.cpp @@ -21,7 +21,12 @@ #include "Arduino.h" #include "debug.h" -const char* overrideme PROGMEM = "%s should be overridden for better efficiency\r\n"; +#ifdef DEBUG_ESP_CORE +void iamslow (const char* what) +{ + DEBUGV("%s should be overridden for better efficiency\r\n", what); +} +#endif void ICACHE_RAM_ATTR hexdump(const void *mem, uint32_t len, uint8_t cols) { const uint8_t* src = (const uint8_t*) mem; diff --git a/cores/esp8266/debug.h b/cores/esp8266/debug.h index b1bf9efed5..1670445f5e 100644 --- a/cores/esp8266/debug.h +++ b/cores/esp8266/debug.h @@ -26,13 +26,13 @@ void __panic_func(const char* file, int line, const char* func) __attribute__((n #define panic() __panic_func(PSTR(__FILE__), __LINE__, __func__) #ifdef DEBUG_ESP_CORE -extern const char* overrideme PROGMEM; +extern void iamslow (const char* what); #define IAMSLOW() \ do { \ static bool once = false; \ if (!once) { \ once = true; \ - DEBUGV(overrideme, (PGM_P)PSTR(__FUNCTION__)); \ + iamslow((PGM_P)FPSTR(__FUNCTION__)); \ } \ } while (0) #else diff --git a/cores/esp8266/uart.cpp b/cores/esp8266/uart.cpp index 832c33a775..d528a6a9a2 100644 --- a/cores/esp8266/uart.cpp +++ b/cores/esp8266/uart.cpp @@ -243,6 +243,11 @@ uart_peek_char(uart_t* uart) // return number of byte accessible by uart_peek_buffer() size_t uart_peek_available (uart_t* uart) { + // path for further optimization: + // - return already copied buffer pointer (= older data) + // - or return fifo when buffer is empty but then any move from fifo to + // buffer should be blocked until peek_consume is called + ETS_UART_INTR_DISABLE(); uart_rx_copy_fifo_to_buffer_unsafe(uart); ETS_UART_INTR_ENABLE(); diff --git a/libraries/ESP8266WiFi/src/WiFiClient.cpp b/libraries/ESP8266WiFi/src/WiFiClient.cpp index eb2a91c0f1..1a09d95e2d 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.cpp +++ b/libraries/ESP8266WiFi/src/WiFiClient.cpp @@ -424,6 +424,10 @@ uint8_t WiFiClient::getKeepAliveCount () const return _client->getKeepAliveCount(); } +bool WiFiClient::peekBufferAPI () const +{ + return true; +} // return a pointer to available data buffer (size = availableForPeek()) // semantic forbids any kind of read() before calling peekConsume() diff --git a/libraries/ESP8266WiFi/src/WiFiClient.h b/libraries/ESP8266WiFi/src/WiFiClient.h index 82798d8a56..5e557d6682 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.h +++ b/libraries/ESP8266WiFi/src/WiFiClient.h @@ -66,7 +66,12 @@ class WiFiClient : public Client, public SList { virtual int available() override; virtual int read() override; - virtual int read(char *buf, size_t size) override; + virtual int read(char *buf, size_t size); // should override, see Stream.h + virtual int read(uint8_t *buf, size_t size) // should override, see Stream.h + { + return read((char*)buf, size); + } + virtual int peek() override; virtual size_t peekBytes(uint8_t *buffer, size_t length); size_t peekBytes(char *buffer, size_t length) { @@ -120,6 +125,8 @@ class WiFiClient : public Client, public SList { bool getSync() const; void setSync(bool sync); + virtual bool peekBufferAPI () const override; + // return number of byte accessible by peekBuffer() virtual size_t availableForPeek () override; diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h b/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h index 8b525e753f..0edbeb889b 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h @@ -83,8 +83,8 @@ class WiFiClientSecure : public WiFiClient { return loadCACert(file, file.size()); } - // disallow optimization for streamMove/STREAM_MOVE - virtual const char* peekBuffer () { return nullptr; } + // disallow buffered peek API + virtual bool peekBufferAPI () const override { return false; } friend class WiFiServerSecure; // Needs access to custom constructor below protected: diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h index e05b9cd261..afafe5a263 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h @@ -131,8 +131,8 @@ class WiFiClientSecure : public WiFiClient { static bool probeMaxFragmentLength(const char *hostname, uint16_t port, uint16_t len); static bool probeMaxFragmentLength(const String& host, uint16_t port, uint16_t len); - // disallow optimization for streamMove/STREAM_MOVE - virtual const char* peekBuffer () { return nullptr; } + // disallow buffered peek API (for now) + virtual bool peekBufferAPI () const override { return false; } //////////////////////////////////////////////////// // AxTLS API deprecated warnings to help upgrading From 0161c505bf96758cb7db2cb65566216f8457455d Mon Sep 17 00:00:00 2001 From: david gauchard Date: Thu, 21 Nov 2019 13:07:31 +0100 Subject: [PATCH 031/207] wip --- cores/esp8266/Stream.cpp | 11 ++++++++--- cores/esp8266/Stream.h | 20 +++++++++----------- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index 26892a4f0f..d3cd2f274a 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -261,17 +261,22 @@ String Stream::readStringUntil(char terminator) { return ret; } -#if 0 -size_t Stream::read (char* buffer, size_t maxLen) +// read what can be read, immediate exit on unavailable data +// prototype should be int `int Stream::read(char* buffer, size_t maxLen)` like Arduino `int Client::read(buf, len)` +int Stream::readNow (char* buffer, size_t maxLen) { IAMSLOW(); size_t nbread = 0; while (nbread < maxLen && available()) + { + int c = read(); + if (c == -1) + break; buffer[nbread++] = read(); + } return nbread; } -#endif size_t Stream::to (Print& to, esp8266::polledTimeout::oneShotFastMs::timeType timeout, diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 39da902d6b..db0e1fd190 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -108,19 +108,17 @@ class Stream: public Print { virtual String readString(); String readStringUntil(char terminator); -#if 0 -// conflicting returned type: it is `int` in arduino's `Client::` -// and `size_t` in esp8266 API (serial, FS), so holding this back for now because -// changing every read()/write() `size_t` return type to `int` will be a breaking change - - // (Client::read() definitions moved here - HardwareSerial have them too) - // read at most maxLen bytes: - // returns effectively transfered bytes (can be less than maxLen) + // ::read(buf, len): conflicting returned type: it is `int` in arduino's `Client::` + // and `size_t` in esp8266 API (serial, FS), so holding this back for now because + // changing every read()/write() `size_t` return type to `int` will be a breaking change + // adding int ::readNow(buf, len) for now (following Client::read(buf, len)) + // + // (::read() definitions from Client:: moved here - HardwareSerial have them too) + // read at most maxLen bytes, returns effectively transfered bytes (can be less than maxLen) // immediate return when no more data are available (no timeout) - virtual size_t read(char* buffer, size_t maxLen); - virtual size_t read(uint8_t* buffer, size_t maxLen) { return read((char*)buffer, maxLen); } + virtual int readNow(char* buffer, size_t maxLen); + virtual int readNow(uint8_t* buffer, size_t maxLen) { return read((char*)buffer, maxLen); } // return data type: int -#endif //////////////////// extensions: direct access to input buffer From e386dcf9dee924ed1536a358aa95ea785a539034 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Thu, 21 Nov 2019 22:11:17 +0100 Subject: [PATCH 032/207] wip --- cores/esp8266/Client.h | 5 +++++ cores/esp8266/FS.h | 5 +++++ cores/esp8266/HardwareSerial.h | 6 ++++++ cores/esp8266/Stream.cpp | 4 ++-- cores/esp8266/Stream.h | 24 ++++++++++++------------ cores/esp8266/StreamString.cpp | 6 ++++++ cores/esp8266/StreamString.h | 11 +++++++++++ libraries/ESP8266WiFi/src/WiFiClient.h | 4 ++++ 8 files changed, 51 insertions(+), 14 deletions(-) diff --git a/cores/esp8266/Client.h b/cores/esp8266/Client.h index 6cb99a04b6..ca676961d0 100644 --- a/cores/esp8266/Client.h +++ b/cores/esp8266/Client.h @@ -47,6 +47,11 @@ class Client: public Stream { return addr.raw_address(); } #endif + + // substitute for `virtual int ::read(buf, len)` in `Stream::` + virtual int readNow (char* buffer, size_t len) override { + return read(buf, len); + } }; #endif diff --git a/cores/esp8266/FS.h b/cores/esp8266/FS.h index 23d05bad83..4985793dd1 100644 --- a/cores/esp8266/FS.h +++ b/cores/esp8266/FS.h @@ -114,6 +114,11 @@ class File : public Stream time_t getLastWrite(); void setTimeCallback(time_t (*cb)(void)); + // substitute for `virtual int ::read(buf, len)` in `Stream::` + virtual int readNow (char* buffer, size_t len) override { + return read((uint8_t*)buffer, len); + } + protected: FileImplPtr _p; diff --git a/cores/esp8266/HardwareSerial.h b/cores/esp8266/HardwareSerial.h index e8bd4aeda5..6686405203 100644 --- a/cores/esp8266/HardwareSerial.h +++ b/cores/esp8266/HardwareSerial.h @@ -136,6 +136,12 @@ class HardwareSerial: public Stream return true; } + // substitute for virtual int ::read(buf, len) + virtual int readNow (char* buffer, size_t len) override + { + return HardwareSerial::read(buffer, len); + } + // return a pointer to available data buffer (size = available()) // semantic forbids any kind of read() before calling peekConsume() const char* peekBuffer () override diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index d3cd2f274a..1d0dc2b858 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -262,7 +262,7 @@ String Stream::readStringUntil(char terminator) { } // read what can be read, immediate exit on unavailable data -// prototype should be int `int Stream::read(char* buffer, size_t maxLen)` like Arduino `int Client::read(buf, len)` +// prototype should be `int Stream::read(char* buffer, size_t maxLen)` like Arduino `int Client::read(buf, len)` int Stream::readNow (char* buffer, size_t maxLen) { IAMSLOW(); @@ -359,7 +359,7 @@ size_t Stream::to (Print& to, w = std::min(w, (size_t)available()); w = std::min(w, 64U); char temp[w]; - size_t r = read(temp, w); + size_t r = readNow(temp, w); w = to.write(temp, r); assert(r == w); written += w; diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index db0e1fd190..cd64d77b8a 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -108,21 +108,21 @@ class Stream: public Print { virtual String readString(); String readStringUntil(char terminator); - // ::read(buf, len): conflicting returned type: it is `int` in arduino's `Client::` - // and `size_t` in esp8266 API (serial, FS), so holding this back for now because + // ::read(buf, len): conflicting returned type: + // - `int` in arduino's `Client::` + // - `size_t` in esp8266 API (serial, FS) // changing every read()/write() `size_t` return type to `int` will be a breaking change - // adding int ::readNow(buf, len) for now (following Client::read(buf, len)) + // => adding int ::readNow(buf, len) for now (following official Client::read(buf, len)) // - // (::read() definitions from Client:: moved here - HardwareSerial have them too) - // read at most maxLen bytes, returns effectively transfered bytes (can be less than maxLen) + // ::readNow(buf, len) + // read at most len bytes, returns effectively transfered bytes (can be less than len) // immediate return when no more data are available (no timeout) - virtual int readNow(char* buffer, size_t maxLen); - virtual int readNow(uint8_t* buffer, size_t maxLen) { return read((char*)buffer, maxLen); } - // return data type: int + virtual int readNow(char* buffer, size_t len); + int readNow(uint8_t* buffer, size_t len) { return readNow((char*)buffer, len); } //////////////////// extensions: direct access to input buffer - // inform user or ::to() on effective buffered peek API implementation + // inform user and ::to() on effective buffered peek API implementation virtual bool peekBufferAPI () const { return false; } // return number of byte accessible by peekBuffer() @@ -138,13 +138,13 @@ class Stream: public Print { //////////////////// extensions: Stream streams // Stream::to() - // transfer from Stream:: to Print:: at most maxlen bytes and return number of transfered bytes - // (uses 1-copy peekBuffer API when available, or transfer through a 2-copy local stack space) + // transfer from `Stream::` to `Print::` at most maxlen bytes and return number of transfered bytes + // (uses 1-copy peekBuffer API when available, or transfer through a 2-copies local stack space) // - timeout_ms==TimeoutMs::neverExpires: use getTimeout() (when 0: take what's available and immediate return) // - maxLen==0 will transfer until input starvation or saturated output // - readUntilChar: setting anything in 0..255 will stop transfer when this char is read (swallowed, not copied) size_t to (Print& to, - esp8266::polledTimeout::oneShotFastMs::timeType timeout = esp8266::polledTimeout::oneShotFastMs::neverExpires, /* ->getTimeout() */ + esp8266::polledTimeout::oneShotFastMs::timeType timeout = esp8266::polledTimeout::oneShotFastMs::neverExpires, /* =>getTimeout() */ size_t maxLen = 0, int readUntilChar = -1); diff --git a/cores/esp8266/StreamString.cpp b/cores/esp8266/StreamString.cpp index 24cfe0dd1a..70633d396e 100644 --- a/cores/esp8266/StreamString.cpp +++ b/cores/esp8266/StreamString.cpp @@ -66,3 +66,9 @@ int StreamString::peek() { void StreamString::flush() { } +int StreamString::read (char* buffer, size_t len) /*should override*/ { + size_t l = std::min(len, length()); + memcpy(buffer, String::buffer(), l); + remove(0, l); + return l; +} diff --git a/cores/esp8266/StreamString.h b/cores/esp8266/StreamString.h index 2e81fa14a0..2660c1bc73 100644 --- a/cores/esp8266/StreamString.h +++ b/cores/esp8266/StreamString.h @@ -33,6 +33,17 @@ class StreamString: public Stream, public String { int read() override; int peek() override; void flush() override; + + virtual bool peekBufferAPI () const override { return true; } + virtual size_t availableForPeek () override { return String::length(); } + virtual const char* peekBuffer () override { return String::buffer(); } + virtual void peekConsume (size_t consume) override { String::remove(0, consume); } + virtual int read (char* buffer, size_t len) /*should override*/; + + // substitute for `virtual int ::read(buf, len)` in `Stream::` + virtual int readNow (char* buffer, size_t len) override { + return read(buffer, len); + } }; diff --git a/libraries/ESP8266WiFi/src/WiFiClient.h b/libraries/ESP8266WiFi/src/WiFiClient.h index 5e557d6682..5d912c02ba 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.h +++ b/libraries/ESP8266WiFi/src/WiFiClient.h @@ -125,6 +125,10 @@ class WiFiClient : public Client, public SList { bool getSync() const; void setSync(bool sync); + // substitute for virtual int ::read(buf, len) + virtual int readNow (char* buffer, size_t len) override; + + // peek buffer API is present virtual bool peekBufferAPI () const override; // return number of byte accessible by peekBuffer() From d18411bc269e851ea488b9e52361586774f5c5fd Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 22 Nov 2019 00:42:06 +0100 Subject: [PATCH 033/207] wip, +example --- cores/esp8266/Client.h | 2 +- .../examples/WiFiEcho/WiFiEcho.ino | 98 +++++++++++++++++++ libraries/ESP8266WiFi/src/WiFiClient.h | 4 +- .../ESP8266WiFi/src/include/ClientContext.h | 6 +- 4 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino diff --git a/cores/esp8266/Client.h b/cores/esp8266/Client.h index ca676961d0..27e52f68ef 100644 --- a/cores/esp8266/Client.h +++ b/cores/esp8266/Client.h @@ -50,7 +50,7 @@ class Client: public Stream { // substitute for `virtual int ::read(buf, len)` in `Stream::` virtual int readNow (char* buffer, size_t len) override { - return read(buf, len); + return read((uint8_t*)buffer, len); } }; diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino new file mode 100644 index 0000000000..2e9c190e98 --- /dev/null +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino @@ -0,0 +1,98 @@ +/* + WiFiEcho - Echo tester + + released to public domain +*/ + +#include + +#include // std::min + +#ifndef STASSID +#define STASSID "your-ssid" +#define STAPSK "your-password" +#endif + +#define STACK_PROTECTOR 128 + +const int port = 23; +int t = 3; + +WiFiServer server(port); +WiFiClient client; + +void setup() { + + Serial.begin(115200); + Serial.println(ESP.getFullVersion()); + + WiFi.mode(WIFI_STA); + WiFi.begin(STASSID, STAPSK); + Serial.print("\nConnecting to "); + Serial.println(STASSID); + while (WiFi.status() != WL_CONNECTED) { + Serial.print('.'); + delay(500); + } + Serial.println(); + Serial.print("connected, address="); + Serial.println(WiFi.localIP()); + + //start server + server.begin(); + //server.setNoDelay(true); + + Serial.print("Ready! Use 'telnet "); + Serial.print(WiFi.localIP()); + Serial.printf(" %d' to connect\n", port); +} + +void loop() { + + //check if there are any new clients + if (server.hasClient()) { + client = server.available(); + Serial.println("New client"); + } + + if (Serial.available()) + switch (Serial.read()) { + case '1': t = 1; break; + case '2': t = 2; break; + case '3': t = 3; break; + } + + if (t == 1) + + // byte by byte + + while (client.available() && client.availableForWrite()) + // working char by char is not very efficient + { + client.write(client.read()); + } + + else if (t == 2) + + // block by block through a local buffer (2 copies) + + while (client.available() && client.availableForWrite()) { + size_t maxTo = std::min(client.available(), client.availableForWrite()); + maxTo = std::min(maxTo, (size_t)STACK_PROTECTOR); + uint8_t buf[maxTo]; + size_t tcp_got = client.read(buf, maxTo); + size_t tcp_sent = client.write(buf, tcp_got); + if (tcp_sent != maxTo) { + Serial.printf("len mismatch: available:%zd tcp-read:%zd serial-write:%zd\n", maxTo, tcp_got, tcp_sent); + } + } + + else if (t == 3) + + // stream to print, possibly with only one copy + + { + client.to(client, 0); // immediate return (no timeout) + } + +} diff --git a/libraries/ESP8266WiFi/src/WiFiClient.h b/libraries/ESP8266WiFi/src/WiFiClient.h index 5d912c02ba..7847eee2fe 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.h +++ b/libraries/ESP8266WiFi/src/WiFiClient.h @@ -126,7 +126,9 @@ class WiFiClient : public Client, public SList { void setSync(bool sync); // substitute for virtual int ::read(buf, len) - virtual int readNow (char* buffer, size_t len) override; + virtual int readNow (char* buffer, size_t len) override { + return WiFiClient::read(buffer, len); + } // peek buffer API is present virtual bool peekBufferAPI () const override; diff --git a/libraries/ESP8266WiFi/src/include/ClientContext.h b/libraries/ESP8266WiFi/src/include/ClientContext.h index aa5ca1cbd9..2c8888f1ce 100644 --- a/libraries/ESP8266WiFi/src/include/ClientContext.h +++ b/libraries/ESP8266WiFi/src/include/ClientContext.h @@ -437,7 +437,7 @@ class ClientContext { if (!_rx_buf) return 0; - return _rx_buf->tot_len - _rx_buf_offset; + return _rx_buf->len - _rx_buf_offset; } // consume bytes after use (see peekBuffer) @@ -575,8 +575,6 @@ class ClientContext void _consume(size_t size) { - if(_pcb) - tcp_recved(_pcb, size); ptrdiff_t left = _rx_buf->len - _rx_buf_offset - size; if(left > 0) { _rx_buf_offset += size; @@ -593,6 +591,8 @@ class ClientContext pbuf_ref(_rx_buf); pbuf_free(head); } + if(_pcb) + tcp_recved(_pcb, size); } err_t _recv(tcp_pcb* pcb, pbuf* pb, err_t err) From 85b1b4fe8de69a7354b3f05d8aa6326935b64deb Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 23 Nov 2019 00:14:44 +0100 Subject: [PATCH 034/207] fix emu-on-host, fix httpclient --- cores/esp8266/Stream.cpp | 2 +- cores/esp8266/debug.cpp | 8 ++++---- libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp | 2 +- tests/host/Makefile | 1 + tests/host/common/ClientContextSocket.cpp | 4 +++- tests/host/common/include/ClientContext.h | 5 +---- tests/host/common/mock.h | 5 ++++- 7 files changed, 15 insertions(+), 12 deletions(-) diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index 1d0dc2b858..1ee496191a 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -361,8 +361,8 @@ size_t Stream::to (Print& to, char temp[w]; size_t r = readNow(temp, w); w = to.write(temp, r); - assert(r == w); written += w; + assert(r == w); if (w) timedOut.reset(); else if (timedOut) diff --git a/cores/esp8266/debug.cpp b/cores/esp8266/debug.cpp index 8a13e7dbc3..d4f396dc2e 100644 --- a/cores/esp8266/debug.cpp +++ b/cores/esp8266/debug.cpp @@ -30,14 +30,14 @@ void iamslow (const char* what) void ICACHE_RAM_ATTR hexdump(const void *mem, uint32_t len, uint8_t cols) { const uint8_t* src = (const uint8_t*) mem; - os_printf("\n[HEXDUMP] Address: 0x%08X len: 0x%X (%d)", (ptrdiff_t)src, len, len); + os_printf_plus("\n[HEXDUMP] Address: 0x%08X len: 0x%X (%d)", (ptrdiff_t)src, len, len); for(uint32_t i = 0; i < len; i++) { if(i % cols == 0) { - os_printf("\n[0x%08X] 0x%08X: ", (ptrdiff_t)src, i); + os_printf_plus("\n[0x%08X] 0x%08X: ", (ptrdiff_t)src, i); yield(); } - os_printf("%02X ", *src); + os_printf_plus("%02X ", *src); src++; } - os_printf("\n"); + os_printf_plus("\n"); } diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 5ccf9b93e2..c78deb8cb9 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -760,7 +760,7 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) size_t transferred = 0; while (connected() && transferred < size) - transferred += streamMove(*stream, *_client, stream->getTimeout(), size - transferred); + transferred += stream->to(*_client, stream->getTimeout(), size - transferred); if (transferred < size) { diff --git a/tests/host/Makefile b/tests/host/Makefile index aaf8a88d91..79ed966e75 100644 --- a/tests/host/Makefile +++ b/tests/host/Makefile @@ -243,6 +243,7 @@ ARDUINO_LIBS := \ IPAddress.cpp \ Updater.cpp \ base64.cpp \ + debug.cpp \ ) \ $(addprefix ../../libraries/ESP8266WiFi/src/,\ ESP8266WiFi.cpp \ diff --git a/tests/host/common/ClientContextSocket.cpp b/tests/host/common/ClientContextSocket.cpp index b2366fa72c..86c3fec40c 100644 --- a/tests/host/common/ClientContextSocket.cpp +++ b/tests/host/common/ClientContextSocket.cpp @@ -179,7 +179,7 @@ ssize_t mockWrite (int sock, const uint8_t* data, size_t size, int timeout_ms) #endif if (ret == -1) { - fprintf(stderr, MOCK "ClientContext::read: write(%d): %s\n", sock, strerror(errno)); + fprintf(stderr, MOCK "ClientContext::write/send(%d): %s\n", sock, strerror(errno)); return -1; } sent += ret; @@ -187,6 +187,8 @@ ssize_t mockWrite (int sock, const uint8_t* data, size_t size, int timeout_ms) fprintf(stderr, MOCK "ClientContext::write: sent %d bytes (%zd / %zd)\n", ret, sent, size); } } +#ifdef DEBUG_ESP_WIFI fprintf(stderr, MOCK "ClientContext::write: total sent %zd bytes\n", sent); +#endif return sent; } diff --git a/tests/host/common/include/ClientContext.h b/tests/host/common/include/ClientContext.h index 198991bdc4..ade3a2e785 100644 --- a/tests/host/common/include/ClientContext.h +++ b/tests/host/common/include/ClientContext.h @@ -323,10 +323,7 @@ class ClientContext void peekConsume (size_t consume) { assert(consume <= _inbufsize); - size_t move = consume; - if (consume + move > sizeof _inbuf) - move = sizeof _inbuf - consume; - memmove(_inbuf, _inbuf + consume, move); + memmove(_inbuf, _inbuf + consume, _inbufsize - consume); _inbufsize -= consume; } diff --git a/tests/host/common/mock.h b/tests/host/common/mock.h index 8a4e9e1c3e..dde3fa13bb 100644 --- a/tests/host/common/mock.h +++ b/tests/host/common/mock.h @@ -85,6 +85,9 @@ typedef uint32_t uint32; // #include +#include + +inline uint32_t esp_get_cycle_count() { return millis(); } #include @@ -105,7 +108,7 @@ typedef uint32_t uint32; extern "C" { #endif int ets_printf (const char* fmt, ...) __attribute__ ((format (printf, 1, 2))); -#define os_printf_plus printf +#define os_printf_plus ets_printf int mockverbose (const char* fmt, ...) __attribute__ ((format (printf, 1, 2))); From d1a1c883e942b371536fc14f796eefa85197fa00 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 23 Nov 2019 00:24:25 +0100 Subject: [PATCH 035/207] update hostname when emu-on-host --- tests/host/common/MockEsp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/host/common/MockEsp.cpp b/tests/host/common/MockEsp.cpp index 8ff16cd01a..f966527970 100644 --- a/tests/host/common/MockEsp.cpp +++ b/tests/host/common/MockEsp.cpp @@ -208,7 +208,7 @@ uint32_t EspClass::getFlashChipSize(void) String EspClass::getFullVersion () { - return "host-emulation"; + return "emulation-on-host"; } uint32_t EspClass::getFreeContStack() From efd30520b33fcc87ced226c45fd6dd253e8f1322 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 23 Nov 2019 00:49:14 +0100 Subject: [PATCH 036/207] update example --- tests/host/emutest-FSBrowser+ClientPoster.sh | 30 +++++++++---------- .../HttpClientPost/HttpClientPost.ino | 8 ++--- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/host/emutest-FSBrowser+ClientPoster.sh b/tests/host/emutest-FSBrowser+ClientPoster.sh index 12df7a4e05..31617ca3bf 100755 --- a/tests/host/emutest-FSBrowser+ClientPoster.sh +++ b/tests/host/emutest-FSBrowser+ClientPoster.sh @@ -7,16 +7,16 @@ # what it does: # # Run FSbrowser official example -# use it to upload a random (big) file to emulated SPIFFS -# Stop it, copy the emulated SPIFFS file for another HTTPClientPost sketch +# use it to upload a random (big) file to emulated FS +# Stop it, copy the emulated FS file for another HTTPClientPost sketch # Restart the Server # Start the Poster, it will upload a copy to the server # end set -e -sizekb=16000 # SPIFFS SIZE in KBytes (file size will be the third of this value) -DEBUG="D=1" # comment this or not +sizekb=16000 # FS SIZE in KBytes (file size will be the third of this value) +#DEBUG="D=1" # comment this or not options="-f -b" sleep=5 @@ -28,9 +28,9 @@ make -j $DEBUG otherexamples/HttpClientPost/HttpClientPost killall -9 FSBrowser HttpClientPost || true -# remove SPIFFS -rm -f FSBrowser-spiffs${sizekb}KB -./bin/FSBrowser/FSBrowser $options -S ${sizekb} & PIDFSBrowser=$! +# remove FS +rm -f ./bin/FSBrowser/FSBrowser-littlefs${sizekb}KB +./bin/FSBrowser/FSBrowser $options -S 0 -L ${sizekb} & PIDFSBrowser=$! echo "------------------------------" echo " let server start..." @@ -42,35 +42,35 @@ dd if=/dev/urandom of=randfile bs=${fileb} count=1 sleep ${sleep} echo "------------------------------" -echo " Uploading $file to SPIFFS" +echo " Uploading $file to FS" echo "------------------------------" curl -F "file=@$PWD/randfile" 127.0.0.1:9080/edit echo "------------------------------" -echo " Uploading to SPIFFS (done)" +echo " Uploading to FS (done)" echo " (wait&kill FSBrowser) " echo "------------------------------" sleep ${sleep} kill -INT $PIDFSBrowser sleep ${sleep} -# FSBrowser has generated SPIFFS backup file, copy it for the other sketch +# FSBrowser has generated littlefs backup file, copy it for the other sketch # This sketch will repost this file to the FSBrowser, so we can debug/tune -# copy SPIFFS to http client sketch -cp ./bin/FSBrowser/FSBrowser-spiffs${sizekb}KB ./bin/HttpClientPost/HttpClientPost-spiffs${sizekb}KB -ls -al ./bin/FSBrowser/FSBrowser-spiffs${sizekb}KB ./bin/HttpClientPost/HttpClientPost-spiffs${sizekb}KB +# copy FS to http client sketch +cp ./bin/FSBrowser/FSBrowser-littlefs${sizekb}KB ./bin/HttpClientPost/HttpClientPost-littlefs${sizekb}KB +ls -al ./bin/FSBrowser/FSBrowser-littlefs${sizekb}KB ./bin/HttpClientPost/HttpClientPost-littlefs${sizekb}KB echo "------------------------------" echo " let server start again..." echo "------------------------------" -./bin/FSBrowser/FSBrowser $options -S ${sizekb} & PIDFSBrowser=$! +./bin/FSBrowser/FSBrowser $options -S 0 -L ${sizekb} & PIDFSBrowser=$! sleep ${sleep} echo "------------------------------" echo " start uploader sketch" echo "------------------------------" # run http client poster sketch -./bin/HttpClientPost/HttpClientPost $options -S ${sizekb} & PIDClient=$! +./bin/HttpClientPost/HttpClientPost $options -S 0 -L ${sizekb} & PIDClient=$! echo "------------------------------" echo "Let run the sketches, press enter to kill" diff --git a/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino b/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino index b2bf683a44..2cae0e480d 100644 --- a/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino +++ b/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino @@ -3,7 +3,7 @@ #include #include -#include +#include #define FILE "/randfile" #define URL "http://127.0.0.1:9080/edit" @@ -24,10 +24,10 @@ void setup() { delay(250); } - if (!SPIFFS.begin()) - Serial.println("CLIENT: failed to start SPIFFS!"); + if (!LittleFS.begin()) + Serial.println("CLIENT: failed to start LittleFS!"); - auto f = SPIFFS.open(FILE, "r"); + auto f = LittleFS.open(FILE, "r"); const auto f_size = f.size(); Serial.printf("CLIENT: size to upload: %d\n", (int)f_size); From b12d29e12a8564eeed78c376954d66324a67d52f Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 23 Nov 2019 00:55:50 +0100 Subject: [PATCH 037/207] update example --- .../HttpClientPost/HttpClientPost.ino | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino b/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino index 2cae0e480d..763a22d6a1 100644 --- a/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino +++ b/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino @@ -48,28 +48,19 @@ void setup() { client.println("Content-Type: application/octet-stream"); client.println(""); - - Serial.printf("\r\n\r\n"); - Serial.printf("\r\n\r\n##########################\r\n"); - - -#define TESTSTREAM 0 - -printf("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\r\n"); +#define TESTSTREAM 1 auto A = millis(); Serial.printf("\r\n\r\nCLIENT: ----> upload duration: %lums sent: %d\r\n\r\n", millis() - A, #if TESTSTREAM - (int)f.streamTo(client, 0) + (int)f.to(client, 0) #else (int)client.write(f) #endif ); A = millis() - A; -printf("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\r\n"); - Serial.println("CLIENT waiting for server ack..."); client.println("\r\n--glark--"); while (!client.available()) @@ -78,7 +69,7 @@ printf("$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$$\ Serial.println("@@@@@@@@@@@@@ CLIENT: server response:"); #if TESTSTREAM client.setTimeout(10000); - client.streamTo(Serial, 0); + client.to(Serial, 0); #else while (client.available()) Serial.write(client.read()); From cf5c2e4347dd0b885deba9903ced50d38b47a0ce Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 24 Nov 2019 09:01:20 +0100 Subject: [PATCH 038/207] fix untilchar --- cores/esp8266/Stream.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index 1ee496191a..75259bd9f9 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -303,21 +303,23 @@ size_t Stream::to (Print& to, if (w) { const char* directbuf = peekBuffer(); - bool ignore = false; + bool foundChar = false; if (readUntilChar >= 0) { const char* last = (const char*)memchr(directbuf, readUntilChar, w); if (last) { w = std::min((size_t)(last - directbuf + 1), w); - ignore = true; + foundChar = true; } } if (w && ((w = to.write(directbuf, w)))) { - peekConsume(w + ignore); + peekConsume(w + foundChar); written += w; timedOut.reset(); + if (foundChar) + break; } } else if (timedOut) From 266ff1a5bf8e57d72e6021206ea8b63ecb9e0587 Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Thu, 28 Nov 2019 01:24:59 +0100 Subject: [PATCH 039/207] update Stream::to() API (breaking) (param order and readUntil semantics) --- cores/esp8266/Stream.cpp | 6 +++--- cores/esp8266/Stream.h | 4 ++-- libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp | 2 +- libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino | 2 +- tests/host/otherexamples/HttpClientPost/HttpClientPost.ino | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index 75259bd9f9..0be70b8698 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -279,8 +279,8 @@ int Stream::readNow (char* buffer, size_t maxLen) } size_t Stream::to (Print& to, - esp8266::polledTimeout::oneShotFastMs::timeType timeout, size_t maxLen, + esp8266::polledTimeout::oneShotFastMs::timeType timeout, int readUntilChar) { esp8266::polledTimeout::periodicFastMs yieldNow(100); @@ -315,7 +315,7 @@ size_t Stream::to (Print& to, } if (w && ((w = to.write(directbuf, w)))) { - peekConsume(w + foundChar); + peekConsume(w); written += w; timedOut.reset(); if (foundChar) @@ -338,9 +338,9 @@ size_t Stream::to (Print& to, int c = read(); if (c != -1) { + w = to.write(c); if (c == readUntilChar) break; - w = to.write(c); assert(w); written += 1; timedOut.reset(); diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index cd64d77b8a..c22cde5584 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -142,10 +142,10 @@ class Stream: public Print { // (uses 1-copy peekBuffer API when available, or transfer through a 2-copies local stack space) // - timeout_ms==TimeoutMs::neverExpires: use getTimeout() (when 0: take what's available and immediate return) // - maxLen==0 will transfer until input starvation or saturated output - // - readUntilChar: setting anything in 0..255 will stop transfer when this char is read (swallowed, not copied) + // - readUntilChar: setting anything in 0..255 will stop transfer when this char is read *and copied too*. size_t to (Print& to, - esp8266::polledTimeout::oneShotFastMs::timeType timeout = esp8266::polledTimeout::oneShotFastMs::neverExpires, /* =>getTimeout() */ size_t maxLen = 0, + esp8266::polledTimeout::oneShotFastMs::timeType timeout = esp8266::polledTimeout::oneShotFastMs::neverExpires /* =>getTimeout() */, int readUntilChar = -1); //////////////////// end of extensions diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 912802a97d..349a8c052a 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -771,7 +771,7 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) size_t transferred = 0; while (connected() && transferred < size) - transferred += stream->to(*_client, stream->getTimeout(), size - transferred); + transferred += stream->to(*_client, size - transferred); // default timeout from *stream if (transferred < size) { diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino index 2e9c190e98..e830955bc3 100644 --- a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino @@ -92,7 +92,7 @@ void loop() { // stream to print, possibly with only one copy { - client.to(client, 0); // immediate return (no timeout) + client.to(client, 0, 0); // immediate return (no timeout) } } diff --git a/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino b/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino index 763a22d6a1..faa6943ad6 100644 --- a/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino +++ b/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino @@ -54,7 +54,7 @@ void setup() { Serial.printf("\r\n\r\nCLIENT: ----> upload duration: %lums sent: %d\r\n\r\n", millis() - A, #if TESTSTREAM - (int)f.to(client, 0) + (int)f.to(client, 0, 0) #else (int)client.write(f) #endif From ebcf5538fc96b6190bd5ff4897d963db8385b821 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 13 Dec 2019 00:26:01 +0100 Subject: [PATCH 040/207] merge String and StreamString --- cores/esp8266/Print.h | 4 +- cores/esp8266/StreamString.cpp | 74 -------------------------- cores/esp8266/StreamString.h | 24 ++------- cores/esp8266/WString.cpp | 96 ++++++++++++++++++++++++++++++++++ cores/esp8266/WString.h | 47 ++++++++++++++++- 5 files changed, 147 insertions(+), 98 deletions(-) delete mode 100644 cores/esp8266/StreamString.cpp diff --git a/cores/esp8266/Print.h b/cores/esp8266/Print.h index 990b67cb23..541deefe3c 100644 --- a/cores/esp8266/Print.h +++ b/cores/esp8266/Print.h @@ -23,9 +23,11 @@ #include #include -#include "WString.h" #include "Printable.h" +class String; +class __FlashStringHelper; + #define DEC 10 #define HEX 16 #define OCT 8 diff --git a/cores/esp8266/StreamString.cpp b/cores/esp8266/StreamString.cpp deleted file mode 100644 index 70633d396e..0000000000 --- a/cores/esp8266/StreamString.cpp +++ /dev/null @@ -1,74 +0,0 @@ -/** - StreamString.cpp - - Copyright (c) 2015 Markus Sattler. All rights reserved. - This file is part of the esp8266 core for Arduino environment. - - This library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with this library; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - - */ - -#include -#include "StreamString.h" - -size_t StreamString::write(const uint8_t *data, size_t size) { - if(size && data) { - const unsigned int newlen = length() + size; - if(reserve(newlen + 1)) { - memcpy((void *) (wbuffer() + len()), (const void *) data, size); - setLen(newlen); - *(wbuffer() + newlen) = 0x00; // add null for string end - return size; - } - DEBUGV(":stream2string: OOM (%d->%d)\n", length(), newlen+1); - } - return 0; -} - -size_t StreamString::write(uint8_t data) { - return concat((char) data); -} - -int StreamString::available() { - return length(); -} - -int StreamString::read() { - if(length()) { - char c = charAt(0); - remove(0, 1); - return c; - - } - return -1; -} - -int StreamString::peek() { - if(length()) { - char c = charAt(0); - return c; - } - return -1; -} - -void StreamString::flush() { -} - -int StreamString::read (char* buffer, size_t len) /*should override*/ { - size_t l = std::min(len, length()); - memcpy(buffer, String::buffer(), l); - remove(0, l); - return l; -} diff --git a/cores/esp8266/StreamString.h b/cores/esp8266/StreamString.h index 2660c1bc73..6aa72a96cb 100644 --- a/cores/esp8266/StreamString.h +++ b/cores/esp8266/StreamString.h @@ -23,28 +23,10 @@ #ifndef STREAMSTRING_H_ #define STREAMSTRING_H_ +#include "WString.h" -class StreamString: public Stream, public String { -public: - size_t write(const uint8_t *buffer, size_t size) override; - size_t write(uint8_t data) override; - - int available() override; - int read() override; - int peek() override; - void flush() override; - - virtual bool peekBufferAPI () const override { return true; } - virtual size_t availableForPeek () override { return String::length(); } - virtual const char* peekBuffer () override { return String::buffer(); } - virtual void peekConsume (size_t consume) override { String::remove(0, consume); } - virtual int read (char* buffer, size_t len) /*should override*/; - - // substitute for `virtual int ::read(buf, len)` in `Stream::` - virtual int readNow (char* buffer, size_t len) override { - return read(buffer, len); - } -}; +// StreamString has been integrated insite String +using StreamString = String; #endif /* STREAMSTRING_H_ */ diff --git a/cores/esp8266/WString.cpp b/cores/esp8266/WString.cpp index bab537737e..62a58396d4 100644 --- a/cores/esp8266/WString.cpp +++ b/cores/esp8266/WString.cpp @@ -6,6 +6,9 @@ Modified by Ivan Grokhotkov, 2014 - esp8266 support Modified by Michael C. Miller, 2015 - esp8266 progmem support + imported: + StringStream Copyright (c) 2015 Markus Sattler. All rights reserved. + This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either @@ -854,3 +857,96 @@ double String::toDouble(void) const // global empty string to allow returning const String& with nothing const String emptyString; + +///////////////////////////////////////////// +// Stream API: + +//// imported from former StreamString.cpp + +size_t String::write(const uint8_t *data, size_t size) +{ + if(size && data) + { + const unsigned int newlen = length() + size; + if(reserve(newlen + 1)) + { + memcpy((void *) (wbuffer() + len()), (const void *) data, size); + setLen(newlen); + *(wbuffer() + newlen) = 0x00; // add null for string end + return size; + } + DEBUGV(":stream2string: OOM (%d->%d)\n", length(), newlen+1); + } + return 0; +} + +size_t String::write(uint8_t data) +{ + return concat((char) data); +} + +int String::available() +{ + return length(); +} + +int String::read() +{ + if(length()) + { + char c = charAt(0); + if (peekPointer < 0) + remove(0, 1); + else + peekPointer++; + return c; + } + return -1; +} + +int String::peek() +{ + if(length()) + { + char c = charAt(0); + return c; + } + return -1; +} + +void String::flush() +{ +} + +//// Stream's peekBufferAPI + +void String::peekConsume (size_t consume) +{ + if (peekPointer < 0) + // string is really consumed + remove(0, consume); + else + // only the pointer is moved + peekPointer = std::min(length(), peekPointer + consume); +} + +int String::read (char* buffer, size_t len) +{ + if (peekPointer < 0) + { + // string is really consumed + size_t l = std::min(len, length()); + memcpy(buffer, String::buffer(), l); + remove(0, l); + return l; + } + + if (peekPointer >= (int)length()) + return 0; + + // only the pointer is moved + size_t l = std::min(len, length() - peekPointer); + memcpy(buffer, String::buffer() + peekPointer, l); + peekPointer += l; + return l; +} diff --git a/cores/esp8266/WString.h b/cores/esp8266/WString.h index fb2f0797f0..bb054db2d4 100644 --- a/cores/esp8266/WString.h +++ b/cores/esp8266/WString.h @@ -28,6 +28,8 @@ #include #include +#include + // An inherited class for holding the result of a concatenation. These // result objects are assumed to be writable by subsequent concatenations. class StringSumHelper; @@ -39,7 +41,7 @@ class __FlashStringHelper; #define F(string_literal) (FPSTR(PSTR(string_literal))) // The string class -class String { +class String: public Stream { // use a function pointer to allow for "if (s)" without the // complications of an operator bool(). for more information, see: // http://www.artima.com/cppsource/safebool.html @@ -279,7 +281,7 @@ class String { protected: // Contains the string info when we're not in SSO mode - struct _ptr { + struct _ptr { char * buff; uint16_t cap; uint16_t len; @@ -319,6 +321,47 @@ class String { #ifdef __GXX_EXPERIMENTAL_CXX0X__ void move(String &rhs); #endif + + ///////////////////////////////////////////// + // Stream API: + +protected: + + // peekPointer is used with peekBufferAPI, + // on peekConsume(), string can either: + // - be really consumed -- case when peekPointer==-1 + // - marked as read -- peekPointer is increased + // (marked to not be consumed by default) + int peekPointer = 0; + +public: + + //// imported from former StreamString class: + + size_t write(const uint8_t *buffer, size_t size) override; + size_t write(uint8_t data) override; + + int available() override; + int read() override; + int peek() override; + void flush() override; + + //// peekBuffer API: + + virtual bool peekBufferAPI () const override { return true; } + virtual size_t availableForPeek () override { return String::length(); } + virtual const char* peekBuffer () override { return String::buffer(); } + //ditchme virtual void peekConsume (size_t consume) override { String::remove(0, consume); } + virtual void peekConsume (size_t consume) override; + virtual int read (char* buffer, size_t len) /*should override*/; + + void peekPointerSetConsume () { peekPointer = -1; } + void peekPointerReset (int pointer = 0) { peekPointer = pointer; } + + // substitute for `virtual int ::read(buf, len)` in `Stream::` + virtual int readNow (char* buffer, size_t len) override { + return read(buffer, len); + } }; class StringSumHelper: public String { From c030a8563659318e97b903aa21bdc48e9d875789 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 13 Dec 2019 00:31:16 +0100 Subject: [PATCH 041/207] stream string fusion fixes --- libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h | 4 +--- tests/host/Makefile | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h index 1d7548f0f7..1ebb0b5b61 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h @@ -32,7 +32,7 @@ #include #include - +#include #include #ifdef DEBUG_ESP_HTTP_CLIENT @@ -135,8 +135,6 @@ class TransportTraits; typedef std::unique_ptr TransportTraitsPtr; #endif -class StreamString; - class HTTPClient { public: diff --git a/tests/host/Makefile b/tests/host/Makefile index 79ed966e75..33ead97844 100644 --- a/tests/host/Makefile +++ b/tests/host/Makefile @@ -51,7 +51,6 @@ endif $(shell mkdir -p $(BINDIR)) CORE_CPP_FILES := $(addprefix $(CORE_PATH)/,\ - StreamString.cpp \ Stream.cpp \ WString.cpp \ Print.cpp \ From 4eb7263213606fcaa69c09fe2523b2cd798b6e32 Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Fri, 13 Dec 2019 10:45:39 +0100 Subject: [PATCH 042/207] fix Print --- cores/esp8266/Print.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cores/esp8266/Print.h b/cores/esp8266/Print.h index 541deefe3c..7fadf00004 100644 --- a/cores/esp8266/Print.h +++ b/cores/esp8266/Print.h @@ -22,6 +22,8 @@ #include #include +#include +#include #include "Printable.h" From 2b0cfe13c72a2008549c1299f660701a1a0ef066 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 13 Dec 2019 23:51:17 +0100 Subject: [PATCH 043/207] fix example --- .../examples/WiFiTelnetToSerial/WiFiTelnetToSerial.ino | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/ESP8266WiFi/examples/WiFiTelnetToSerial/WiFiTelnetToSerial.ino b/libraries/ESP8266WiFi/examples/WiFiTelnetToSerial/WiFiTelnetToSerial.ino index 9102af12d8..1f0112a3c1 100644 --- a/libraries/ESP8266WiFi/examples/WiFiTelnetToSerial/WiFiTelnetToSerial.ino +++ b/libraries/ESP8266WiFi/examples/WiFiTelnetToSerial/WiFiTelnetToSerial.ino @@ -194,7 +194,7 @@ void loop() { len = std::min(len, (size_t)STACK_PROTECTOR); if (len) { uint8_t sbuf[len]; - size_t serial_got = Serial.readBytes(sbuf, len); + int serial_got = Serial.readBytes(sbuf, len); // push UART data to all connected telnet clients for (int i = 0; i < MAX_SRV_CLIENTS; i++) // if client.availableForWrite() was 0 (congested) From 388a43cdd5a9d967efbc47b3a7e38c7185295fcb Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 14 Dec 2019 00:51:36 +0100 Subject: [PATCH 044/207] + streamdev, + simplify http client --- cores/esp8266/Stream.h | 2 +- cores/esp8266/StreamDev.h | 93 +++++++++++++++++++ .../src/ESP8266HTTPClient.cpp | 10 +- 3 files changed, 103 insertions(+), 2 deletions(-) create mode 100644 cores/esp8266/StreamDev.h diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index c22cde5584..325ece55b9 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -140,8 +140,8 @@ class Stream: public Print { // Stream::to() // transfer from `Stream::` to `Print::` at most maxlen bytes and return number of transfered bytes // (uses 1-copy peekBuffer API when available, or transfer through a 2-copies local stack space) - // - timeout_ms==TimeoutMs::neverExpires: use getTimeout() (when 0: take what's available and immediate return) // - maxLen==0 will transfer until input starvation or saturated output + // - timeout_ms==TimeoutMs::neverExpires: use getTimeout() (when 0: take what's available and immediate return) // - readUntilChar: setting anything in 0..255 will stop transfer when this char is read *and copied too*. size_t to (Print& to, size_t maxLen = 0, diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h new file mode 100644 index 0000000000..319bdda8d3 --- /dev/null +++ b/cores/esp8266/StreamDev.h @@ -0,0 +1,93 @@ + +#ifndef __STREAMDEV_H +#define __STREAMDEV_H + +#include + +/////////////////////////////////////////////// +// - black hole in, swallow everything, availableForWrite = infinite +// - black hole out, nothing to read, available = 0 + +class StreamNull: public Stream +{ +public: + + // Print + virtual size_t write(uint8_t) override { return 1; } + virtual size_t write(const uint8_t* buffer, size_t size) override { (void)buffer; return size; } + virtual int availableForWrite() override { return 32767; } + + // Stream + virtual int available() override { return 0; } + virtual int read() override { return -1; } + virtual int peek() override { return -1; } + virtual size_t readBytes(char* buffer, size_t len) override { (void)buffer; (void)len; return 0; } + virtual int readNow(char* buffer, size_t len) override { (void)buffer; (void)len; return 0; } +}; + +/////////////////////////////////////////////// +// /dev/zero +// - black hole in, swallow everything, availableForWrite = infinite +// - white hole out, gives infinity to read, available = infinite + +class StreamZero: public StreamNull +{ +protected: + char _x; +public: + StreamZero (char x = 0): _x(x) { } + + // Stream + virtual int available() override { return 32767; } + virtual int read() override { return _x; } + virtual int peek() override { return _x; } + virtual size_t readBytes(char* buffer, size_t len) override { memset(buffer, _x, len); return len; } + virtual int readNow(char* buffer, size_t len) override { memset(buffer, _x, len); return len; } +}; + +/////////////////////////////////////////////// +// static buffer (in flash or ram) +// - black hole in, swallow everything, availableForWrite = infinite +// - Stream buffer out + +class StreamPtr: public StreamNull +{ +protected: + const char* _buffer; + size_t _size; + bool _in_flash; + size_t _peekPointer = 0; + +public: + StreamPtr (const char* buffer, size_t size, bool in_flash = false): _buffer(buffer), _size(size), _in_flash(in_flash) { } + StreamPtr (const uint8_t* buffer, size_t size, bool in_flash = false): _buffer((const char*)buffer), _size(size), _in_flash(in_flash) { } + StreamPtr (const __FlashStringHelper* buffer, size_t size): _buffer(reinterpret_cast(buffer)), _size(size), _in_flash(true) { } + + void peekPointerReset (int pointer = 0) { _peekPointer = pointer; } + + // Stream + virtual int available() override { return availableForPeek(); } + virtual int read() override { return _peekPointer < _size? _buffer[_peekPointer++]: -1; } + virtual int peek() override { return _peekPointer < _size? _buffer[_peekPointer]: -1; } + virtual size_t readBytes(char* buffer, size_t len) override + { + if (_peekPointer < _size) + { + size_t cpylen = std::min(_size - _peekPointer, len); + memcpy_P(buffer, _buffer + _peekPointer, cpylen); + _peekPointer += cpylen; + return cpylen; + } + return 0; + } + virtual int readNow(char* buffer, size_t len) override { return readBytes(buffer, len); } + + // peekBuffer + virtual bool peekBufferAPI () const override { return !_in_flash; } + virtual size_t availableForPeek () override { return _peekPointer < _size? _size - _peekPointer: 0; } + virtual const char* peekBuffer () { return _peekPointer < _size? _buffer + _peekPointer: nullptr; } + virtual void peekConsume (size_t consume) { _peekPointer += consume; } +}; + +/////////////////////////////////////////////// +#endif // __STREAMDEV_H diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 349a8c052a..1f9bd322c6 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -30,7 +30,7 @@ #include #endif -#include +#include #include #if HTTPCLIENT_1_1_COMPATIBLE @@ -676,6 +676,13 @@ int HTTPClient::sendRequest(const char * type, const uint8_t * payload, size_t s return returnError(HTTPC_ERROR_SEND_HEADER_FAILED); } +#if 1 + + size_t sent = StreamPtr(payload, size).to(*_client); // all of it, with timeout + if (sent < size) + return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); + +#else // send Payload if needed if (payload && size > 0) { size_t bytesWritten = 0; @@ -693,6 +700,7 @@ int HTTPClient::sendRequest(const char * type, const uint8_t * payload, size_t s size -= written; } } +#endif // handle Server Response (Header) code = handleHeaderResponse(); From f95d98ed727c6732bf934151d05f391f528609dc Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 14 Dec 2019 16:22:02 +0100 Subject: [PATCH 045/207] wip --- cores/esp8266/Stream.cpp | 2 + .../src/ESP8266HTTPClient.cpp | 68 +++++++++++++------ 2 files changed, 49 insertions(+), 21 deletions(-) diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index 0be70b8698..866fcbf306 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -212,6 +212,8 @@ float Stream::parseFloat(char skipChar) { // the buffer is NOT null terminated. // size_t Stream::readBytes(char *buffer, size_t length) { + IAMSLOW(); + size_t count = 0; while(count < length) { int c = timedRead(); diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 1f9bd322c6..08e895e5cc 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -678,8 +678,8 @@ int HTTPClient::sendRequest(const char * type, const uint8_t * payload, size_t s #if 1 - size_t sent = StreamPtr(payload, size).to(*_client); // all of it, with timeout - if (sent < size) + // all of it, with timeout + if (StreamPtr(payload, size).to(*_client) < size) return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); #else @@ -776,12 +776,8 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) #if 1 - size_t transferred = 0; - - while (connected() && transferred < size) - transferred += stream->to(*_client, size - transferred); // default timeout from *stream - - if (transferred < size) + // all of it, with timeout + if (stream->to(*_client, size) < size) { DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] short write, asked for %d but got %d failed.\n", size, transferred); return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); @@ -970,8 +966,13 @@ int HTTPClient::writeToStream(Stream * stream) int ret = 0; if(_transferEncoding == HTTPC_TE_IDENTITY) { +#if 1 + // len < 0: all of it, with timeout + // len >= 0: max:len, with timeout + ret = _client->to(stream, (size_t)(len < 0? 0: len)); +#else ret = writeToStreamDataBlock(stream, len); - +#endif // have we an error? if(ret < 0) { return returnError(ret); @@ -997,11 +998,18 @@ int HTTPClient::writeToStream(Stream * stream) // data left? if(len > 0) { +#if 1 + // read len bytes with timeout + int r = _client->to(stream, len); + if (r < len) + // not all data transferred + return returnError(HTTPC_ERROR_READ_TIMEOUT); +#else int r = writeToStreamDataBlock(stream, len); - if(r < 0) { + if(r < 0) // error in writeToStreamDataBlock return returnError(r); - } +#endif ret += r; } else { @@ -1194,9 +1202,13 @@ bool HTTPClient::connect(void) } else { DEBUG_HTTPCLIENT("[HTTP-Client] connect: already connected, try reuse!\n"); } +#if 1 + _client->to(StreamNull(), 0, 0); // clear _client's output +#else while(_client->available() > 0) { _client->read(); } +#endif return true; } @@ -1251,22 +1263,28 @@ bool HTTPClient::sendHeader(const char * type) return false; } - String header = String(type) + ' ' + (_uri.length() ? _uri : F("/")) + F(" HTTP/1."); + String header; + header.reserve(128); - if(_useHTTP10) { - header += '0'; - } else { - header += '1'; - } + header += type; + header += ' '; + if (_uri.length()) + header += _uri; + else + header += '/'; + header += F(" HTTP/1."); + header += '0' + !_useHTTP10; - header += String(F("\r\nHost: ")) + _host; + header += F("\r\nHost: "); + jeader += _host; if (_port != 80 && _port != 443) { header += ':'; header += String(_port); } - header += String(F("\r\nUser-Agent: ")) + _userAgent + - F("\r\nConnection: "); + header += F("\r\nUser-Agent: "); + header += _userAgent; + header += F("\r\nConnection: "); if(_reuse) { header += F("keep-alive"); @@ -1285,11 +1303,17 @@ bool HTTPClient::sendHeader(const char * type) header += "\r\n"; } - header += _headers + "\r\n"; + header += _headers; + header += F("\r\n"); DEBUG_HTTPCLIENT("[HTTP-Client] sending request header\n-----\n%s-----\n", header.c_str()); +#if 1 + // all of it, with timeout + return header.to(*_client) == header.length(); +#else return (_client->write((const uint8_t *) header.c_str(), header.length()) == header.length()); +#endif } /** @@ -1401,6 +1425,7 @@ int HTTPClient::handleHeaderResponse() return HTTPC_ERROR_CONNECTION_LOST; } +#if 0 /** * write one Data Block to Stream * @param stream Stream * @@ -1504,6 +1529,7 @@ int HTTPClient::writeToStreamDataBlock(Stream * stream, int size) return bytesWritten; } +#endif /** * called to handle error return, may disconnect the connection if still exists From 6fb5001b8ad58a0adbafb73b84b44d4dcd5256b4 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 14 Dec 2019 17:56:39 +0100 Subject: [PATCH 046/207] wip --- cores/esp8266/StreamDev.h | 5 ++++- libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 319bdda8d3..62fe88d930 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -74,7 +74,10 @@ class StreamPtr: public StreamNull if (_peekPointer < _size) { size_t cpylen = std::min(_size - _peekPointer, len); - memcpy_P(buffer, _buffer + _peekPointer, cpylen); + if (_in_flash) + memcpy_P(buffer, _buffer + _peekPointer, cpylen); + else + memcpy(buffer, _buffer + _peekPointer, cpylen); _peekPointer += cpylen; return cpylen; } diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index 395267cd53..5edfb4cb1d 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -435,7 +435,15 @@ void ESP8266WebServerTemplate::send(int code, const char* content_ty //if(code == 200 && content.length() == 0 && _contentLength == CONTENT_LENGTH_NOT_SET) // _contentLength = CONTENT_LENGTH_UNKNOWN; _prepareHeader(header, code, content_type, content.length()); +#if 1 + size_t sent = StreamPtr(header.c_str(), header.length()).to(_currentClient); // all of it, with timeout +#ifdef DEBUG_ESP_HTTP_SERVER + if (sent != header.length()) + DEBUG_OUTPUT.println("HTTPServer: error: sent %zd on %zd bytes\n", sent, header.length()); +#endif +#else _currentClient.write((const uint8_t *)header.c_str(), header.length()); +#endif if(content.length()) sendContent(content); } From c1321a3e1561e9555ff821e9bfc00872a366e5b7 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 14 Dec 2019 18:24:29 +0100 Subject: [PATCH 047/207] fixes --- libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 08e895e5cc..652d9a8361 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -969,7 +969,7 @@ int HTTPClient::writeToStream(Stream * stream) #if 1 // len < 0: all of it, with timeout // len >= 0: max:len, with timeout - ret = _client->to(stream, (size_t)(len < 0? 0: len)); + ret = _client->to(*stream, (size_t)(len < 0? 0: len)); #else ret = writeToStreamDataBlock(stream, len); #endif @@ -1000,7 +1000,7 @@ int HTTPClient::writeToStream(Stream * stream) if(len > 0) { #if 1 // read len bytes with timeout - int r = _client->to(stream, len); + int r = _client->to(*stream, len); if (r < len) // not all data transferred return returnError(HTTPC_ERROR_READ_TIMEOUT); @@ -1203,7 +1203,8 @@ bool HTTPClient::connect(void) DEBUG_HTTPCLIENT("[HTTP-Client] connect: already connected, try reuse!\n"); } #if 1 - _client->to(StreamNull(), 0, 0); // clear _client's output + StreamNull devnull; + _client->to(devnull, 0, 0); // clear _client's output #else while(_client->available() > 0) { _client->read(); @@ -1273,10 +1274,10 @@ bool HTTPClient::sendHeader(const char * type) else header += '/'; header += F(" HTTP/1."); - header += '0' + !_useHTTP10; + header += (char)('0' + !_useHTTP10); header += F("\r\nHost: "); - jeader += _host; + header += _host; if (_port != 80 && _port != 443) { header += ':'; From b2e5b2a3e2a3c23c495a492b19bcaa64d1eeec22 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 15 Dec 2019 02:16:47 +0100 Subject: [PATCH 048/207] wip --- cores/esp8266/Print.h | 4 ++ cores/esp8266/Stream.cpp | 99 --------------------------------------- cores/esp8266/Stream.h | 10 ++-- cores/esp8266/StreamDev.h | 2 + 4 files changed, 13 insertions(+), 102 deletions(-) diff --git a/cores/esp8266/Print.h b/cores/esp8266/Print.h index 7fadf00004..5ba38374d1 100644 --- a/cores/esp8266/Print.h +++ b/cores/esp8266/Print.h @@ -108,6 +108,10 @@ class Print { virtual int availableForWrite(); // return type: int + + // by default timeout is enabled (outgoing network,serial.. data) + // (children can override to false) + virtual bool outputTimeoutPossible () const { return true; } }; #endif diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index 866fcbf306..50771b97a3 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -22,7 +22,6 @@ #include #include -#include #define PARSE_TIMEOUT 1000 // default number of milli-seconds to wait #define NO_SKIP_CHAR 1 // a magic char not found in a valid ASCII numeric field @@ -279,101 +278,3 @@ int Stream::readNow (char* buffer, size_t maxLen) } return nbread; } - -size_t Stream::to (Print& to, - size_t maxLen, - esp8266::polledTimeout::oneShotFastMs::timeType timeout, - int readUntilChar) -{ - esp8266::polledTimeout::periodicFastMs yieldNow(100); - esp8266::polledTimeout::oneShotFastMs timedOut(timeout); - size_t written = 0; - size_t w; - - if (timeout == esp8266::polledTimeout::oneShotFastMs::neverExpires) - timeout = getTimeout(); - - if (peekBufferAPI()) - - // peek-buffer API - - while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) - { - if (maxLen) - w = std::min(w, maxLen - written); - w = std::min(w, availableForPeek()); - if (w) - { - const char* directbuf = peekBuffer(); - bool foundChar = false; - if (readUntilChar >= 0) - { - const char* last = (const char*)memchr(directbuf, readUntilChar, w); - if (last) - { - w = std::min((size_t)(last - directbuf + 1), w); - foundChar = true; - } - } - if (w && ((w = to.write(directbuf, w)))) - { - peekConsume(w); - written += w; - timedOut.reset(); - if (foundChar) - break; - } - } - else if (timedOut) - break; - if (yieldNow) - yield(); - } - - else if (readUntilChar >= 0) - - // regular Stream API - // no other choice than reading byte by byte - - while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) - { - int c = read(); - if (c != -1) - { - w = to.write(c); - if (c == readUntilChar) - break; - assert(w); - written += 1; - timedOut.reset(); - } - else if (timedOut) - break; - if (yieldNow) - yield(); - } - - else - - // regular Stream API - // use an intermediary buffer - - while ((!maxLen || written < maxLen) && (w = to.availableForWrite())) - { - w = std::min(w, (size_t)available()); - w = std::min(w, 64U); - char temp[w]; - size_t r = readNow(temp, w); - w = to.write(temp, r); - written += w; - assert(r == w); - if (w) - timedOut.reset(); - else if (timedOut) - break; - if (yieldNow) - yield(); - } - - return written; -} diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 325ece55b9..b9de87e858 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -24,8 +24,9 @@ #include #include -#include "Print.h" +#include #include +#include // ssize_t // compatibility macros for testing /* @@ -135,16 +136,19 @@ class Stream: public Print { // consume bytes after peekBuffer use virtual void peekConsume (size_t consume) { (void)consume; } + // by default timeout is enabled (incoming network,serial.. data) + virtual bool inputTimeoutPossible () const { return true; } + //////////////////// extensions: Stream streams // Stream::to() // transfer from `Stream::` to `Print::` at most maxlen bytes and return number of transfered bytes // (uses 1-copy peekBuffer API when available, or transfer through a 2-copies local stack space) - // - maxLen==0 will transfer until input starvation or saturated output + // - maxLen<0 will transfer until input starvation or saturated output // - timeout_ms==TimeoutMs::neverExpires: use getTimeout() (when 0: take what's available and immediate return) // - readUntilChar: setting anything in 0..255 will stop transfer when this char is read *and copied too*. size_t to (Print& to, - size_t maxLen = 0, + ssize_t len = -1, esp8266::polledTimeout::oneShotFastMs::timeType timeout = esp8266::polledTimeout::oneShotFastMs::neverExpires /* =>getTimeout() */, int readUntilChar = -1); diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 62fe88d930..0199c512b7 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -23,6 +23,8 @@ class StreamNull: public Stream virtual int peek() override { return -1; } virtual size_t readBytes(char* buffer, size_t len) override { (void)buffer; (void)len; return 0; } virtual int readNow(char* buffer, size_t len) override { (void)buffer; (void)len; return 0; } + virtual bool outputTimeoutPossible () const override { return false; } + virtual bool inputTimeoutPossible () const override { return false; } }; /////////////////////////////////////////////// From dde713dcc65f280157431dced3c04d9b421eb6d1 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 15 Dec 2019 02:17:32 +0100 Subject: [PATCH 049/207] wip --- cores/esp8266/WString.cpp | 37 +++++++++++++------ cores/esp8266/WString.h | 9 ++--- .../src/ESP8266HTTPClient.cpp | 23 +++++++----- .../examples/WiFiEcho/WiFiEcho.ino | 16 ++++---- tests/host/Makefile | 1 + .../HttpClientPost/HttpClientPost.ino | 2 +- 6 files changed, 53 insertions(+), 35 deletions(-) diff --git a/cores/esp8266/WString.cpp b/cores/esp8266/WString.cpp index 62a58396d4..b5d5c6d690 100644 --- a/cores/esp8266/WString.cpp +++ b/cores/esp8266/WString.cpp @@ -887,30 +887,36 @@ size_t String::write(uint8_t data) int String::available() { - return length(); + return peekPointer < 0? length(): length() - peekPointer; } int String::read() { - if(length()) + if (peekPointer < 0) { - char c = charAt(0); - if (peekPointer < 0) + if (length()) + { + char c = charAt(0); remove(0, 1); - else - peekPointer++; - return c; + return c; + } } + else if (peekPointer < (int)length()) + return charAt(peekPointer++); + return -1; } int String::peek() { - if(length()) + if (peekPointer < 0) { - char c = charAt(0); - return c; + if (length()) + return charAt(0); } + else if (peekPointer < (int)length()) + return charAt(peekPointer); + return -1; } @@ -920,6 +926,15 @@ void String::flush() //// Stream's peekBufferAPI +const char* String::peekBuffer () +{ + if (peekPointer < 0) + return buffer(); + if (peekPointer < (int)length()) + return buffer() + peekPointer; + return nullptr; +} + void String::peekConsume (size_t consume) { if (peekPointer < 0) @@ -934,7 +949,7 @@ int String::read (char* buffer, size_t len) { if (peekPointer < 0) { - // string is really consumed + // string will be consumed size_t l = std::min(len, length()); memcpy(buffer, String::buffer(), l); remove(0, l); diff --git a/cores/esp8266/WString.h b/cores/esp8266/WString.h index bb054db2d4..4b6bdf9c53 100644 --- a/cores/esp8266/WString.h +++ b/cores/esp8266/WString.h @@ -336,8 +336,6 @@ class String: public Stream { public: - //// imported from former StreamString class: - size_t write(const uint8_t *buffer, size_t size) override; size_t write(uint8_t data) override; @@ -349,10 +347,11 @@ class String: public Stream { //// peekBuffer API: virtual bool peekBufferAPI () const override { return true; } - virtual size_t availableForPeek () override { return String::length(); } - virtual const char* peekBuffer () override { return String::buffer(); } - //ditchme virtual void peekConsume (size_t consume) override { String::remove(0, consume); } + virtual size_t availableForPeek () override { return available(); } + virtual const char* peekBuffer () override; virtual void peekConsume (size_t consume) override; + virtual bool outputTimeoutPossible () const override { return false; } + virtual bool inputTimeoutPossible () const override { return false; } virtual int read (char* buffer, size_t len) /*should override*/; void peekPointerSetConsume () { peekPointer = -1; } diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 652d9a8361..eaf93ba9da 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -679,7 +679,7 @@ int HTTPClient::sendRequest(const char * type, const uint8_t * payload, size_t s #if 1 // all of it, with timeout - if (StreamPtr(payload, size).to(*_client) < size) + if (size && StreamPtr(payload, size).to(*_client) != size) return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); #else @@ -777,7 +777,7 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) #if 1 // all of it, with timeout - if (stream->to(*_client, size) < size) + if (stream->to(*_client, size) != size) { DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] short write, asked for %d but got %d failed.\n", size, transferred); return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); @@ -966,13 +966,15 @@ int HTTPClient::writeToStream(Stream * stream) int ret = 0; if(_transferEncoding == HTTPC_TE_IDENTITY) { -#if 1 +Serial.printf("##### 1 %d %d\n", (int)len, (int)millis()); +#if 0 // test ok // len < 0: all of it, with timeout // len >= 0: max:len, with timeout - ret = _client->to(*stream, (size_t)(len < 0? 0: len)); + ret = _client->to(*stream, len, 1000); #else ret = writeToStreamDataBlock(stream, len); #endif +Serial.printf("##### 2 %d %d\n", (int)ret, (int)millis()); // have we an error? if(ret < 0) { return returnError(ret); @@ -998,14 +1000,17 @@ int HTTPClient::writeToStream(Stream * stream) // data left? if(len > 0) { -#if 1 +Serial.printf("##### 3 %d %d\n", (int)len, (int)millis()); +#if 0 // testok // read len bytes with timeout - int r = _client->to(*stream, len); - if (r < len) + int r = _client->to(*stream, len, 100); +Serial.printf("##### 4 %d %d\n", (int)r, (int)millis()); + if (r != len) // not all data transferred return returnError(HTTPC_ERROR_READ_TIMEOUT); #else int r = writeToStreamDataBlock(stream, len); +Serial.printf("##### 4 %d %d\n", (int)r, (int)millis()); if(r < 0) // error in writeToStreamDataBlock return returnError(r); @@ -1204,7 +1209,7 @@ bool HTTPClient::connect(void) } #if 1 StreamNull devnull; - _client->to(devnull, 0, 0); // clear _client's output + _client->to(devnull, -1, 0); // clear _client's output (all of it, no timeout) #else while(_client->available() > 0) { _client->read(); @@ -1426,7 +1431,7 @@ int HTTPClient::handleHeaderResponse() return HTTPC_ERROR_CONNECTION_LOST; } -#if 0 +#if 1 /** * write one Data Block to Stream * @param stream Stream * diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino index e830955bc3..58ea1dacfd 100644 --- a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino @@ -63,19 +63,18 @@ void loop() { } if (t == 1) - + { // byte by byte - while (client.available() && client.availableForWrite()) - // working char by char is not very efficient { + // working char by char is not efficient client.write(client.read()); } + } else if (t == 2) - + { // block by block through a local buffer (2 copies) - while (client.available() && client.availableForWrite()) { size_t maxTo = std::min(client.available(), client.availableForWrite()); maxTo = std::min(maxTo, (size_t)STACK_PROTECTOR); @@ -86,13 +85,12 @@ void loop() { Serial.printf("len mismatch: available:%zd tcp-read:%zd serial-write:%zd\n", maxTo, tcp_got, tcp_sent); } } + } else if (t == 3) - - // stream to print, possibly with only one copy - { - client.to(client, 0, 0); // immediate return (no timeout) + // stream to print, possibly with only one copy + client.to(client, -1, 0); // all of it, immediate return (no timeout) } } diff --git a/tests/host/Makefile b/tests/host/Makefile index 33ead97844..0eda8f145f 100644 --- a/tests/host/Makefile +++ b/tests/host/Makefile @@ -51,6 +51,7 @@ endif $(shell mkdir -p $(BINDIR)) CORE_CPP_FILES := $(addprefix $(CORE_PATH)/,\ + StreamPtr.cpp \ Stream.cpp \ WString.cpp \ Print.cpp \ diff --git a/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino b/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino index faa6943ad6..8d4e289e1d 100644 --- a/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino +++ b/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino @@ -54,7 +54,7 @@ void setup() { Serial.printf("\r\n\r\nCLIENT: ----> upload duration: %lums sent: %d\r\n\r\n", millis() - A, #if TESTSTREAM - (int)f.to(client, 0, 0) + (int)f.to(client, -1, 0) #else (int)client.write(f) #endif From 9cc11b6a97bcd9897fc0e0cc1744f6e5204121c3 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 15 Dec 2019 21:59:54 +0100 Subject: [PATCH 050/207] wip --- cores/esp8266/Stream.h | 2 +- libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp | 8 ++++---- libraries/ESP8266WiFi/src/WiFiClient.h | 2 +- libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h | 1 + tests/host/Makefile | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index b9de87e858..5dd23d23f4 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -147,7 +147,7 @@ class Stream: public Print { // - maxLen<0 will transfer until input starvation or saturated output // - timeout_ms==TimeoutMs::neverExpires: use getTimeout() (when 0: take what's available and immediate return) // - readUntilChar: setting anything in 0..255 will stop transfer when this char is read *and copied too*. - size_t to (Print& to, + size_t to (Print* to, ssize_t len = -1, esp8266::polledTimeout::oneShotFastMs::timeType timeout = esp8266::polledTimeout::oneShotFastMs::neverExpires /* =>getTimeout() */, int readUntilChar = -1); diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index eaf93ba9da..485511fbae 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -679,7 +679,7 @@ int HTTPClient::sendRequest(const char * type, const uint8_t * payload, size_t s #if 1 // all of it, with timeout - if (size && StreamPtr(payload, size).to(*_client) != size) + if (size && StreamPtr(payload, size).to(_client) != size) return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); #else @@ -777,7 +777,7 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) #if 1 // all of it, with timeout - if (stream->to(*_client, size) != size) + if (stream->to(_client, size) != size) { DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] short write, asked for %d but got %d failed.\n", size, transferred); return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); @@ -1209,7 +1209,7 @@ bool HTTPClient::connect(void) } #if 1 StreamNull devnull; - _client->to(devnull, -1, 0); // clear _client's output (all of it, no timeout) + _client->to(&devnull, -1, 0); // clear _client's output (all of it, no timeout) #else while(_client->available() > 0) { _client->read(); @@ -1316,7 +1316,7 @@ bool HTTPClient::sendHeader(const char * type) #if 1 // all of it, with timeout - return header.to(*_client) == header.length(); + return header.to(_client) == header.length(); #else return (_client->write((const uint8_t *) header.c_str(), header.length()) == header.length()); #endif diff --git a/libraries/ESP8266WiFi/src/WiFiClient.h b/libraries/ESP8266WiFi/src/WiFiClient.h index 7847eee2fe..85b740dc54 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.h +++ b/libraries/ESP8266WiFi/src/WiFiClient.h @@ -127,7 +127,7 @@ class WiFiClient : public Client, public SList { // substitute for virtual int ::read(buf, len) virtual int readNow (char* buffer, size_t len) override { - return WiFiClient::read(buffer, len); + return read(buffer, len); } // peek buffer API is present diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h index afafe5a263..946bc54e6a 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h @@ -133,6 +133,7 @@ class WiFiClientSecure : public WiFiClient { // disallow buffered peek API (for now) virtual bool peekBufferAPI () const override { return false; } + //virtual int readNow (char* buffer, size_t len) override { return read((uint8_t*)buffer, len); } //////////////////////////////////////////////////// // AxTLS API deprecated warnings to help upgrading diff --git a/tests/host/Makefile b/tests/host/Makefile index 0eda8f145f..0996944b92 100644 --- a/tests/host/Makefile +++ b/tests/host/Makefile @@ -51,7 +51,7 @@ endif $(shell mkdir -p $(BINDIR)) CORE_CPP_FILES := $(addprefix $(CORE_PATH)/,\ - StreamPtr.cpp \ + StreamDev.cpp \ Stream.cpp \ WString.cpp \ Print.cpp \ From c28e50f3dc41d879b45ad6a6aeff6c310f17c28f Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 15 Dec 2019 22:02:43 +0100 Subject: [PATCH 051/207] wip --- .../src/ESP8266HTTPClient.cpp | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 485511fbae..fd7244ae1b 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -952,7 +952,6 @@ WiFiClient* HTTPClient::getStreamPtr(void) */ int HTTPClient::writeToStream(Stream * stream) { - if(!stream) { return returnError(HTTPC_ERROR_NO_STREAM); } @@ -966,15 +965,15 @@ int HTTPClient::writeToStream(Stream * stream) int ret = 0; if(_transferEncoding == HTTPC_TE_IDENTITY) { -Serial.printf("##### 1 %d %d\n", (int)len, (int)millis()); -#if 0 // test ok +Serial.printf("##### 1 %dms pb=%i len=%d\n", (int)millis(), (int)_client->peekBufferAPI(), (int)len); +#if 1 // testok // len < 0: all of it, with timeout // len >= 0: max:len, with timeout - ret = _client->to(*stream, len, 1000); + ret = _client->to(stream, len, 1000); #else ret = writeToStreamDataBlock(stream, len); #endif -Serial.printf("##### 2 %d %d\n", (int)ret, (int)millis()); +Serial.printf("##### 2 %dms pb=%i ret=%d\n", (int)millis(), (int)_client->peekBufferAPI(), (int)ret); // have we an error? if(ret < 0) { return returnError(ret); @@ -1000,17 +999,17 @@ Serial.printf("##### 2 %d %d\n", (int)ret, (int)millis()); // data left? if(len > 0) { -Serial.printf("##### 3 %d %d\n", (int)len, (int)millis()); -#if 0 // testok +Serial.printf("##### 3 %dms pb=%i len=%d\n", (int)millis(), (int)_client->peekBufferAPI(), (int)len); +#if 1 // testok // read len bytes with timeout - int r = _client->to(*stream, len, 100); -Serial.printf("##### 4 %d %d\n", (int)r, (int)millis()); + int r = _client->to(stream, len, 100); +Serial.printf("##### 4 %dms pb=%i ret=%d\n", (int)millis(), (int)_client->peekBufferAPI(), (int)r); if (r != len) // not all data transferred return returnError(HTTPC_ERROR_READ_TIMEOUT); #else int r = writeToStreamDataBlock(stream, len); -Serial.printf("##### 4 %d %d\n", (int)r, (int)millis()); +Serial.printf("##### 4 %dms pb=%i ret=%d\n", (int)millis(), (int)_client->peekBufferAPI(), (int)r); if(r < 0) // error in writeToStreamDataBlock return returnError(r); @@ -1067,7 +1066,9 @@ const String& HTTPClient::getString(void) } } +Serial.printf("blob1 c=%p\n", _client); writeToStream(_payload.get()); +Serial.printf("blob2\n"); return *_payload; } From b2dfcfaeb2cf92132b524d27eabe03cbcd4e82d6 Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Mon, 16 Dec 2019 15:32:45 +0100 Subject: [PATCH 052/207] back to 2.7.0-dev --- README.md | 4 ++-- package.json | 2 +- platform.txt | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6907c7f900..b135c682f5 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Arduino core for ESP8266 WiFi chip # Quick links -- [Latest release documentation](https://arduino-esp8266.readthedocs.io/en/2.6.2/) +- [Latest release documentation](https://arduino-esp8266.readthedocs.io/en/2.6.3/) - [Current "git version" documentation](https://arduino-esp8266.readthedocs.io/en/latest/) - [Install git version](https://arduino-esp8266.readthedocs.io/en/latest/installing.html#using-git-version) ([sources](doc/installing.rst#using-git-version)) @@ -36,7 +36,7 @@ Starting with 1.6.4, Arduino allows installation of third-party platform package #### Latest release [![Latest release](https://img.shields.io/github/release/esp8266/Arduino.svg)](https://github.com/esp8266/Arduino/releases/latest/) Boards manager link: `https://arduino.esp8266.com/stable/package_esp8266com_index.json` -Documentation: [https://arduino-esp8266.readthedocs.io/en/2.6.2/](https://arduino-esp8266.readthedocs.io/en/2.6.2/) +Documentation: [https://arduino-esp8266.readthedocs.io/en/2.6.3/](https://arduino-esp8266.readthedocs.io/en/2.6.3/) ### Using git version [![Linux build status](https://travis-ci.org/esp8266/Arduino.svg)](https://travis-ci.org/esp8266/Arduino) diff --git a/package.json b/package.json index 34a5efeeb1..5ad2898f74 100644 --- a/package.json +++ b/package.json @@ -2,5 +2,5 @@ "name": "framework-arduinoespressif8266", "description": "Arduino Wiring-based Framework (ESP8266 Core)", "url": "https://github.com/esp8266/Arduino", - "version": "2.6.3" + "version": "2.7.0-dev" } diff --git a/platform.txt b/platform.txt index afcc65c002..678be1ba9d 100644 --- a/platform.txt +++ b/platform.txt @@ -5,8 +5,8 @@ # For more info: # https://github.com/arduino/Arduino/wiki/Arduino-IDE-1.5-3rd-party-Hardware-specification -name=ESP8266 Boards (2.6.3) -version=2.6.3 +name=ESP8266 Boards (2.7.0-dev) +version=2.7.0-dev # These will be removed by the packager script when doing a JSON release runtime.tools.xtensa-lx106-elf-gcc.path={runtime.platform.path}/tools/xtensa-lx106-elf From f5cdd1088ca436dc74d921b9b91095f08c43c4d6 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 17 Dec 2019 23:53:26 +0100 Subject: [PATCH 053/207] wip --- cores/esp8266/WString.cpp | 6 ++-- cores/esp8266/debug.h | 1 + .../src/ESP8266HTTPClient.cpp | 32 +++++++++++-------- tests/host/common/ClientContextSocket.cpp | 17 ++++++++-- tests/host/common/include/ClientContext.h | 2 +- 5 files changed, 39 insertions(+), 19 deletions(-) diff --git a/cores/esp8266/WString.cpp b/cores/esp8266/WString.cpp index b5d5c6d690..9c388d4608 100644 --- a/cores/esp8266/WString.cpp +++ b/cores/esp8266/WString.cpp @@ -942,7 +942,7 @@ void String::peekConsume (size_t consume) remove(0, consume); else // only the pointer is moved - peekPointer = std::min(length(), peekPointer + consume); + peekPointer = std::min((size_t)length(), peekPointer + consume); } int String::read (char* buffer, size_t len) @@ -950,7 +950,7 @@ int String::read (char* buffer, size_t len) if (peekPointer < 0) { // string will be consumed - size_t l = std::min(len, length()); + size_t l = std::min(len, (size_t)length()); memcpy(buffer, String::buffer(), l); remove(0, l); return l; @@ -960,7 +960,7 @@ int String::read (char* buffer, size_t len) return 0; // only the pointer is moved - size_t l = std::min(len, length() - peekPointer); + size_t l = std::min(len, (size_t)(length() - peekPointer)); memcpy(buffer, String::buffer() + peekPointer, l); peekPointer += l; return l; diff --git a/cores/esp8266/debug.h b/cores/esp8266/debug.h index 1670445f5e..d17c51b00c 100644 --- a/cores/esp8266/debug.h +++ b/cores/esp8266/debug.h @@ -32,6 +32,7 @@ extern void iamslow (const char* what); static bool once = false; \ if (!once) { \ once = true; \ + if (__GXX_RTTI) DEBUGV(typeid(*this).name()); \ iamslow((PGM_P)FPSTR(__FUNCTION__)); \ } \ } while (0) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index fd7244ae1b..0349f9fd3c 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -676,7 +676,7 @@ int HTTPClient::sendRequest(const char * type, const uint8_t * payload, size_t s return returnError(HTTPC_ERROR_SEND_HEADER_FAILED); } -#if 1 +#if 1 // mock64ok // all of it, with timeout if (size && StreamPtr(payload, size).to(_client) != size) @@ -774,10 +774,11 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) return returnError(HTTPC_ERROR_SEND_HEADER_FAILED); } -#if 1 +#if 1 // mock64ok // all of it, with timeout - if (stream->to(_client, size) != size) + size_t transferred = stream->to(_client, size); + if (transferred != size) { DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] short write, asked for %d but got %d failed.\n", size, transferred); return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); @@ -965,7 +966,12 @@ int HTTPClient::writeToStream(Stream * stream) int ret = 0; if(_transferEncoding == HTTPC_TE_IDENTITY) { -Serial.printf("##### 1 %dms pb=%i len=%d\n", (int)millis(), (int)_client->peekBufferAPI(), (int)len); +fprintf(stderr, "##### 1 %dms pb=%i len=%d\n", (int)millis(), (int)_client->peekBufferAPI(), (int)len); +//fprintf(stderr, "_client:avr=%d avp=%d %p %s\n", (int)_client->available(), _client->availableForPeek(), _client, typeid(*_client).name()); +fprintf(stderr, "_client:avr=%d\n", (int)_client->available()); +fprintf(stderr, "_client:avp=%d\n", _client->availableForPeek()); +fprintf(stderr, "_client:%p\n", _client); +fprintf(stderr, "_client:%s\n", typeid(*_client).name()); #if 1 // testok // len < 0: all of it, with timeout // len >= 0: max:len, with timeout @@ -973,7 +979,7 @@ Serial.printf("##### 1 %dms pb=%i len=%d\n", (int)millis(), (int)_client->peekBu #else ret = writeToStreamDataBlock(stream, len); #endif -Serial.printf("##### 2 %dms pb=%i ret=%d\n", (int)millis(), (int)_client->peekBufferAPI(), (int)ret); +fprintf(stderr, "##### 2 %dms pb=%i ret=%d\n", (int)millis(), (int)_client->peekBufferAPI(), (int)ret); // have we an error? if(ret < 0) { return returnError(ret); @@ -999,17 +1005,17 @@ Serial.printf("##### 2 %dms pb=%i ret=%d\n", (int)millis(), (int)_client->peekBu // data left? if(len > 0) { -Serial.printf("##### 3 %dms pb=%i len=%d\n", (int)millis(), (int)_client->peekBufferAPI(), (int)len); -#if 1 // testok +fprintf(stderr, "##### 3 %dms pb=%i len=%d\n", (int)millis(), (int)_client->peekBufferAPI(), (int)len); +#if 0 // testok // read len bytes with timeout int r = _client->to(stream, len, 100); -Serial.printf("##### 4 %dms pb=%i ret=%d\n", (int)millis(), (int)_client->peekBufferAPI(), (int)r); +fprintf(stderr, "##### 4 %dms pb=%i ret=%d\n", (int)millis(), (int)_client->peekBufferAPI(), (int)r); if (r != len) // not all data transferred return returnError(HTTPC_ERROR_READ_TIMEOUT); #else int r = writeToStreamDataBlock(stream, len); -Serial.printf("##### 4 %dms pb=%i ret=%d\n", (int)millis(), (int)_client->peekBufferAPI(), (int)r); +fprintf(stderr, "##### 4 %dms pb=%i ret=%d\n", (int)millis(), (int)_client->peekBufferAPI(), (int)r); if(r < 0) // error in writeToStreamDataBlock return returnError(r); @@ -1066,9 +1072,9 @@ const String& HTTPClient::getString(void) } } -Serial.printf("blob1 c=%p\n", _client); +fprintf(stderr, "blob1 c=%p\n", _client); writeToStream(_payload.get()); -Serial.printf("blob2\n"); +fprintf(stderr, "blob2\n"); return *_payload; } @@ -1208,7 +1214,7 @@ bool HTTPClient::connect(void) } else { DEBUG_HTTPCLIENT("[HTTP-Client] connect: already connected, try reuse!\n"); } -#if 1 +#if 1 // mock64ok StreamNull devnull; _client->to(&devnull, -1, 0); // clear _client's output (all of it, no timeout) #else @@ -1315,7 +1321,7 @@ bool HTTPClient::sendHeader(const char * type) DEBUG_HTTPCLIENT("[HTTP-Client] sending request header\n-----\n%s-----\n", header.c_str()); -#if 1 +#if 1 // mock64ok // all of it, with timeout return header.to(_client) == header.length(); #else diff --git a/tests/host/common/ClientContextSocket.cpp b/tests/host/common/ClientContextSocket.cpp index 86c3fec40c..627ea5f9a2 100644 --- a/tests/host/common/ClientContextSocket.cpp +++ b/tests/host/common/ClientContextSocket.cpp @@ -89,7 +89,8 @@ ssize_t mockFillInBuf (int sock, char* ccinbuf, size_t& ccinbufsize) if (ret == 0) { // connection closed - return -1; + // nothing is read + return 0; } if (ret == -1) @@ -97,16 +98,21 @@ ssize_t mockFillInBuf (int sock, char* ccinbuf, size_t& ccinbufsize) if (errno != EAGAIN) { fprintf(stderr, MOCK "ClientContext::(read/peek fd=%i): filling buffer for %zd bytes: %s\n", sock, maxread, strerror(errno)); +assert(0); + // error return -1; } ret = 0; } + ccinbufsize += ret; return ret; } ssize_t mockPeekBytes (int sock, char* dst, size_t usersize, int timeout_ms, char* ccinbuf, size_t& ccinbufsize) { + // usersize==0: availableForPeek() + if (usersize > CCBUFSIZE) mockverbose("CCBUFSIZE(%d) should be increased by %zd bytes (-> %zd)\n", CCBUFSIZE, usersize - CCBUFSIZE, usersize); @@ -114,7 +120,7 @@ ssize_t mockPeekBytes (int sock, char* dst, size_t usersize, int timeout_ms, cha size_t retsize = 0; do { - if (usersize <= ccinbufsize) + if (usersize && usersize <= ccinbufsize) { // data already buffered retsize = usersize; @@ -123,7 +129,14 @@ ssize_t mockPeekBytes (int sock, char* dst, size_t usersize, int timeout_ms, cha // check incoming data data if (mockFillInBuf(sock, ccinbuf, ccinbufsize) < 0) + { return -1; + } + + if (usersize == 0 && ccinbufsize) + // availableForPeek + return ccinbufsize; + if (usersize <= ccinbufsize) { // data just received diff --git a/tests/host/common/include/ClientContext.h b/tests/host/common/include/ClientContext.h index ade3a2e785..2dcbeb185d 100644 --- a/tests/host/common/include/ClientContext.h +++ b/tests/host/common/include/ClientContext.h @@ -310,7 +310,7 @@ class ClientContext // return number of byte accessible by peekBuffer() size_t availableForPeek () { - ssize_t ret = mockPeekBytes(_sock, nullptr, sizeof _inbuf, 0, _inbuf, _inbufsize); + ssize_t ret = mockPeekBytes(_sock, nullptr, 0, 0, _inbuf, _inbufsize); if (ret < 0) { abort(); From c3148c405826d5adb536f71926573c7e2be48d45 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 18 Dec 2019 23:10:04 +0100 Subject: [PATCH 054/207] fix --- cores/esp8266/Stream.h | 2 +- cores/esp8266/StreamDev.h | 2 +- cores/esp8266/WString.h | 4 +++ .../src/ESP8266HTTPClient.cpp | 31 ++++++------------- libraries/ESP8266WiFi/src/WiFiClient.h | 4 +-- .../ESP8266WiFi/src/WiFiClientSecureBearSSL.h | 3 +- 6 files changed, 19 insertions(+), 27 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 5dd23d23f4..6aa53e8bea 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -122,7 +122,7 @@ class Stream: public Print { int readNow(uint8_t* buffer, size_t len) { return readNow((char*)buffer, len); } //////////////////// extensions: direct access to input buffer - + // inform user and ::to() on effective buffered peek API implementation virtual bool peekBufferAPI () const { return false; } diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 0199c512b7..5ec689e2b4 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -59,7 +59,7 @@ class StreamPtr: public StreamNull size_t _size; bool _in_flash; size_t _peekPointer = 0; - + public: StreamPtr (const char* buffer, size_t size, bool in_flash = false): _buffer(buffer), _size(size), _in_flash(in_flash) { } StreamPtr (const uint8_t* buffer, size_t size, bool in_flash = false): _buffer((const char*)buffer), _size(size), _in_flash(in_flash) { } diff --git a/cores/esp8266/WString.h b/cores/esp8266/WString.h index 4b6bdf9c53..173963be06 100644 --- a/cores/esp8266/WString.h +++ b/cores/esp8266/WString.h @@ -361,6 +361,10 @@ class String: public Stream { virtual int readNow (char* buffer, size_t len) override { return read(buffer, len); } + + //// Print + virtual int availableForWrite() override { return 256; } + // or biggestChunk()/4 or reserved-length? }; class StringSumHelper: public String { diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 0349f9fd3c..8e927b62e9 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -676,12 +676,10 @@ int HTTPClient::sendRequest(const char * type, const uint8_t * payload, size_t s return returnError(HTTPC_ERROR_SEND_HEADER_FAILED); } -#if 1 // mock64ok - +#if 1 // all of it, with timeout if (size && StreamPtr(payload, size).to(_client) != size) return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); - #else // send Payload if needed if (payload && size > 0) { @@ -774,7 +772,7 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) return returnError(HTTPC_ERROR_SEND_HEADER_FAILED); } -#if 1 // mock64ok +#if 1 // all of it, with timeout size_t transferred = stream->to(_client, size); @@ -786,7 +784,6 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) #else - int buff_size = HTTP_TCP_BUFFER_SIZE; int len = size; @@ -951,6 +948,7 @@ WiFiClient* HTTPClient::getStreamPtr(void) * @param stream Stream * * @return bytes written ( negative values are error codes ) */ + int HTTPClient::writeToStream(Stream * stream) { if(!stream) { @@ -966,12 +964,6 @@ int HTTPClient::writeToStream(Stream * stream) int ret = 0; if(_transferEncoding == HTTPC_TE_IDENTITY) { -fprintf(stderr, "##### 1 %dms pb=%i len=%d\n", (int)millis(), (int)_client->peekBufferAPI(), (int)len); -//fprintf(stderr, "_client:avr=%d avp=%d %p %s\n", (int)_client->available(), _client->availableForPeek(), _client, typeid(*_client).name()); -fprintf(stderr, "_client:avr=%d\n", (int)_client->available()); -fprintf(stderr, "_client:avp=%d\n", _client->availableForPeek()); -fprintf(stderr, "_client:%p\n", _client); -fprintf(stderr, "_client:%s\n", typeid(*_client).name()); #if 1 // testok // len < 0: all of it, with timeout // len >= 0: max:len, with timeout @@ -979,7 +971,6 @@ fprintf(stderr, "_client:%s\n", typeid(*_client).name()); #else ret = writeToStreamDataBlock(stream, len); #endif -fprintf(stderr, "##### 2 %dms pb=%i ret=%d\n", (int)millis(), (int)_client->peekBufferAPI(), (int)ret); // have we an error? if(ret < 0) { return returnError(ret); @@ -1005,20 +996,18 @@ fprintf(stderr, "##### 2 %dms pb=%i ret=%d\n", (int)millis(), (int)_client->peek // data left? if(len > 0) { -fprintf(stderr, "##### 3 %dms pb=%i len=%d\n", (int)millis(), (int)_client->peekBufferAPI(), (int)len); -#if 0 // testok +#if 1 // read len bytes with timeout int r = _client->to(stream, len, 100); -fprintf(stderr, "##### 4 %dms pb=%i ret=%d\n", (int)millis(), (int)_client->peekBufferAPI(), (int)r); if (r != len) // not all data transferred return returnError(HTTPC_ERROR_READ_TIMEOUT); #else int r = writeToStreamDataBlock(stream, len); -fprintf(stderr, "##### 4 %dms pb=%i ret=%d\n", (int)millis(), (int)_client->peekBufferAPI(), (int)r); - if(r < 0) + if(r < 0) { // error in writeToStreamDataBlock return returnError(r); + } #endif ret += r; } else { @@ -1072,9 +1061,7 @@ const String& HTTPClient::getString(void) } } -fprintf(stderr, "blob1 c=%p\n", _client); writeToStream(_payload.get()); -fprintf(stderr, "blob2\n"); return *_payload; } @@ -1214,7 +1201,7 @@ bool HTTPClient::connect(void) } else { DEBUG_HTTPCLIENT("[HTTP-Client] connect: already connected, try reuse!\n"); } -#if 1 // mock64ok +#if 1 StreamNull devnull; _client->to(&devnull, -1, 0); // clear _client's output (all of it, no timeout) #else @@ -1321,7 +1308,7 @@ bool HTTPClient::sendHeader(const char * type) DEBUG_HTTPCLIENT("[HTTP-Client] sending request header\n-----\n%s-----\n", header.c_str()); -#if 1 // mock64ok +#if 1 // all of it, with timeout return header.to(_client) == header.length(); #else @@ -1438,7 +1425,7 @@ int HTTPClient::handleHeaderResponse() return HTTPC_ERROR_CONNECTION_LOST; } -#if 1 +#if 0 /** * write one Data Block to Stream * @param stream Stream * diff --git a/libraries/ESP8266WiFi/src/WiFiClient.h b/libraries/ESP8266WiFi/src/WiFiClient.h index 85b740dc54..6665970bf2 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.h +++ b/libraries/ESP8266WiFi/src/WiFiClient.h @@ -69,9 +69,9 @@ class WiFiClient : public Client, public SList { virtual int read(char *buf, size_t size); // should override, see Stream.h virtual int read(uint8_t *buf, size_t size) // should override, see Stream.h { - return read((char*)buf, size); + return WiFiClient::read((char*)buf, size); } - + virtual int peek() override; virtual size_t peekBytes(uint8_t *buffer, size_t length); size_t peekBytes(char *buffer, size_t length) { diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h index 946bc54e6a..bbe8f62244 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h @@ -54,6 +54,7 @@ class WiFiClientSecure : public WiFiClient { } size_t write(Stream& stream); // Note this is not virtual int read(uint8_t *buf, size_t size) override; + int read(char *buf, size_t size) override { return read((uint8_t*)buf, size); } int available() override; int read() override; int peek() override; @@ -133,7 +134,7 @@ class WiFiClientSecure : public WiFiClient { // disallow buffered peek API (for now) virtual bool peekBufferAPI () const override { return false; } - //virtual int readNow (char* buffer, size_t len) override { return read((uint8_t*)buffer, len); } + virtual int readNow (char* buffer, size_t len) override { return read((uint8_t*)buffer, len); } //////////////////////////////////////////////////// // AxTLS API deprecated warnings to help upgrading From 40b782c5cb542f82d9953928d06af5877dc855ac Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 18 Dec 2019 23:32:37 +0100 Subject: [PATCH 055/207] wip --- cores/esp8266/Stream.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 6aa53e8bea..deaa24ea09 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -110,16 +110,17 @@ class Stream: public Print { String readStringUntil(char terminator); // ::read(buf, len): conflicting returned type: - // - `int` in arduino's `Client::` - // - `size_t` in esp8266 API (serial, FS) + // - `int` in arduino's Client:: + // - `size_t` in esp8266 API (HardwareSerial::, FS::) + // - not existent in arduino's Stream:: // changing every read()/write() `size_t` return type to `int` will be a breaking change - // => adding int ::readNow(buf, len) for now (following official Client::read(buf, len)) + // => adding int ::readNow(buf, len) for now (following official `int Client::read(buf, len))` // - // ::readNow(buf, len) + // int ::readNow(buf, len) // read at most len bytes, returns effectively transfered bytes (can be less than len) // immediate return when no more data are available (no timeout) virtual int readNow(char* buffer, size_t len); - int readNow(uint8_t* buffer, size_t len) { return readNow((char*)buffer, len); } + virtual int readNow(uint8_t* buffer, size_t len) final { return readNow((char*)buffer, len); } //////////////////// extensions: direct access to input buffer From 643dd6d30e7d3e7b0fb04abd4d2189960a226ba2 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 18 Dec 2019 23:38:56 +0100 Subject: [PATCH 056/207] fixes --- cores/esp8266/debug.h | 2 +- tests/host/common/ClientContextSocket.cpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cores/esp8266/debug.h b/cores/esp8266/debug.h index d17c51b00c..d9066435bd 100644 --- a/cores/esp8266/debug.h +++ b/cores/esp8266/debug.h @@ -32,7 +32,7 @@ extern void iamslow (const char* what); static bool once = false; \ if (!once) { \ once = true; \ - if (__GXX_RTTI) DEBUGV(typeid(*this).name()); \ + /*if (__GXX_RTTI) DEBUGV(typeid(*this).name());*/ \ iamslow((PGM_P)FPSTR(__FUNCTION__)); \ } \ } while (0) diff --git a/tests/host/common/ClientContextSocket.cpp b/tests/host/common/ClientContextSocket.cpp index 627ea5f9a2..b76c9a2a91 100644 --- a/tests/host/common/ClientContextSocket.cpp +++ b/tests/host/common/ClientContextSocket.cpp @@ -98,7 +98,6 @@ ssize_t mockFillInBuf (int sock, char* ccinbuf, size_t& ccinbufsize) if (errno != EAGAIN) { fprintf(stderr, MOCK "ClientContext::(read/peek fd=%i): filling buffer for %zd bytes: %s\n", sock, maxread, strerror(errno)); -assert(0); // error return -1; } From 07b52b57a145d07f4eaa52b1cdcdf6f8c83ff989 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 18 Dec 2019 23:42:17 +0100 Subject: [PATCH 057/207] missing file --- cores/esp8266/StreamDev.cpp | 177 ++++++++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 cores/esp8266/StreamDev.cpp diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp new file mode 100644 index 0000000000..cac6c20f65 --- /dev/null +++ b/cores/esp8266/StreamDev.cpp @@ -0,0 +1,177 @@ + +#include +#include + +using esp8266::polledTimeout::oneShotFastMs; +using esp8266::polledTimeout::periodicFastMs; +#include +using namespace std; +size_t Stream::to (Print* to, + ssize_t len, + oneShotFastMs::timeType timeout, + int readUntilChar) +{ + if (len == 0) + return 0; // avoid timeout for no requested data + + // There are two useful timeout: + // - read (network, serial, ...) + // - write (network, serial, ...) + // However + // - getTimeout() is for reading only + // - there is no getOutputTimeout() api + // So we use getTimeout() for both, + // (also when inputTimeoutPossible() is false) + + oneShotFastMs timedOut(timeout == oneShotFastMs::neverExpires? getTimeout(): timeout); + periodicFastMs yieldNow(5); // yield about every 5ms + size_t written = 0; + size_t maxLen = std::max((decltype(len))0, len); + +#define D 1 +#if D + Serial.printf("#### %dms: pb=%d ruc=%d ml=%d to=%d otp=%d itp=%d avr=%d avp=%d avw=%d\n", + (int)millis(), + (int)peekBufferAPI(), + (int)readUntilChar, + (int)maxLen, + (int)timedOut.getTimeout(), + (int)inputTimeoutPossible(), + (int)outputTimeoutPossible(), + (int)available(), + (int)availableForPeek(), + (int)to->availableForWrite()); +#endif + + if (peekBufferAPI()) + + // peek-buffer API + + while (!maxLen || written < maxLen) + { + size_t avpk = availableForPeek(); + if (avpk == 0 && !inputTimeoutPossible()) + // no more data to read, ever + break; + + size_t w = to->availableForWrite(); + if (w == 0 && !outputTimeoutPossible()) + // no more data can be written, ever + break; + + w = std::min(w, avpk); + if (maxLen) + w = std::min(w, maxLen - written); + if (w) + { + const char* directbuf = peekBuffer(); + bool foundChar = false; + if (readUntilChar >= 0) + { + const char* last = (const char*)memchr(directbuf, readUntilChar, w); + if (last) + { + w = std::min((size_t)(last - directbuf + 1), w); + foundChar = true; + } + } + if (w && ((w = to->write(directbuf, w)))) + { + peekConsume(w); + written += w; + timedOut.reset(); + if (foundChar) + break; + } + } + else if (timedOut) + break; + if (yieldNow) + yield(); + } + + else if (readUntilChar >= 0) + + // regular Stream API + // no other choice than reading byte by byte + + while (!maxLen || written < maxLen) + { + size_t avpk = availableForPeek(); + if (avpk == 0 && !inputTimeoutPossible()) + // no more data to read, ever + break; + + size_t w = to->availableForWrite(); + if (w == 0 && !outputTimeoutPossible()) + // no more data can be written, ever + break; + + int c = read(); + if (c != -1) + { + w = to->write(c); + if (c == readUntilChar) + break; + assert(w); + written += 1; + timedOut.reset(); + } + else if (timedOut) + break; + if (yieldNow) + yield(); + } + + else + + // regular Stream API + // use an intermediary buffer + + while (!maxLen || written < maxLen) + { + size_t avr = available(); + if (avr == 0 && !inputTimeoutPossible()) + // no more data to read, ever + break; + + size_t w = to->availableForWrite(); + if (w == 0 && !to->outputTimeoutPossible()) + // no more data can be written, ever + break; + w = std::min(w, avr); + w = std::min(w, (decltype(w))64); + if (w) + { + char temp[w]; + ssize_t r = readNow(temp, w); + if (r < 0) + { + // do something? + break; + } + if ((size_t)r < w) + { + Serial.printf(":to read(=%zd)write(temp, r); + written += w; + assert((size_t)r == w); + if (w) + timedOut.reset(); + } + if (timedOut) + break; + if (yieldNow) + yield(); + } + +#if D + Serial.printf("#### %dms: transf=%d\n", (int)millis(), (int)written); + if (timedOut) + Serial.printf("#### Timeout!\n"); +#endif + + return written; +} From 429ba2f129d8c0c806965ab0783c1b2e7713133c Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 18 Dec 2019 23:49:52 +0100 Subject: [PATCH 058/207] wip --- cores/esp8266/StreamDev.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index cac6c20f65..095b0f836b 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -4,8 +4,7 @@ using esp8266::polledTimeout::oneShotFastMs; using esp8266::polledTimeout::periodicFastMs; -#include -using namespace std; + size_t Stream::to (Print* to, ssize_t len, oneShotFastMs::timeType timeout, From 64c2656f2c940c9ef5246fec1784785e305285d1 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Thu, 19 Dec 2019 01:08:18 +0100 Subject: [PATCH 059/207] wip --- cores/esp8266/Stream.h | 14 ++++++++------ cores/esp8266/StreamDev.cpp | 8 ++++---- libraries/ESP8266WiFi/src/WiFiClient.h | 2 +- libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h | 1 + .../ESP8266WiFi/src/WiFiClientSecureBearSSL.h | 2 +- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index deaa24ea09..267223986f 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -131,10 +131,12 @@ class Stream: public Print { virtual size_t availableForPeek () { return 0; } // return a pointer to available data buffer (size = availableForPeek()) - // semantic forbids any kind of read() after calling peekBuffer() and before calling peekConsume() + // semantic forbids any kind of read() + // after calling peekBuffer() + // and before calling peekConsume() virtual const char* peekBuffer () { return nullptr; } - // consume bytes after peekBuffer use + // consume bytes after peekBuffer() use virtual void peekConsume (size_t consume) { (void)consume; } // by default timeout is enabled (incoming network,serial.. data) @@ -148,10 +150,10 @@ class Stream: public Print { // - maxLen<0 will transfer until input starvation or saturated output // - timeout_ms==TimeoutMs::neverExpires: use getTimeout() (when 0: take what's available and immediate return) // - readUntilChar: setting anything in 0..255 will stop transfer when this char is read *and copied too*. - size_t to (Print* to, - ssize_t len = -1, - esp8266::polledTimeout::oneShotFastMs::timeType timeout = esp8266::polledTimeout::oneShotFastMs::neverExpires /* =>getTimeout() */, - int readUntilChar = -1); + virtual size_t to (Print* to, + ssize_t len = -1, + esp8266::polledTimeout::oneShotFastMs::timeType timeout = esp8266::polledTimeout::oneShotFastMs::neverExpires /* =>getTimeout() */, + int readUntilChar = -1) final; //////////////////// end of extensions diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index 095b0f836b..bc6581ae14 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -27,7 +27,7 @@ size_t Stream::to (Print* to, size_t written = 0; size_t maxLen = std::max((decltype(len))0, len); -#define D 1 +#define D 0 #if D Serial.printf("#### %dms: pb=%d ruc=%d ml=%d to=%d otp=%d itp=%d avr=%d avp=%d avw=%d\n", (int)millis(), @@ -57,7 +57,7 @@ size_t Stream::to (Print* to, if (w == 0 && !outputTimeoutPossible()) // no more data can be written, ever break; - + w = std::min(w, avpk); if (maxLen) w = std::min(w, maxLen - written); @@ -105,7 +105,7 @@ size_t Stream::to (Print* to, if (w == 0 && !outputTimeoutPossible()) // no more data can be written, ever break; - + int c = read(); if (c != -1) { @@ -123,7 +123,7 @@ size_t Stream::to (Print* to, } else - + // regular Stream API // use an intermediary buffer diff --git a/libraries/ESP8266WiFi/src/WiFiClient.h b/libraries/ESP8266WiFi/src/WiFiClient.h index 6665970bf2..04deb2ad8c 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.h +++ b/libraries/ESP8266WiFi/src/WiFiClient.h @@ -127,7 +127,7 @@ class WiFiClient : public Client, public SList { // substitute for virtual int ::read(buf, len) virtual int readNow (char* buffer, size_t len) override { - return read(buffer, len); + return WiFiClient::read(buffer, len); } // peek buffer API is present diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h b/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h index 0edbeb889b..6c78e01773 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureAxTLS.h @@ -85,6 +85,7 @@ class WiFiClientSecure : public WiFiClient { // disallow buffered peek API virtual bool peekBufferAPI () const override { return false; } + virtual int readNow (char* buffer, size_t len) override { return WiFiClientSecure::read((uint8_t*)buffer, len); } friend class WiFiServerSecure; // Needs access to custom constructor below protected: diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h index bbe8f62244..5e288a54f5 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h @@ -134,7 +134,7 @@ class WiFiClientSecure : public WiFiClient { // disallow buffered peek API (for now) virtual bool peekBufferAPI () const override { return false; } - virtual int readNow (char* buffer, size_t len) override { return read((uint8_t*)buffer, len); } + virtual int readNow (char* buffer, size_t len) override { return WiFiClientSecure::read((uint8_t*)buffer, len); } //////////////////////////////////////////////////// // AxTLS API deprecated warnings to help upgrading From 6b14a136d425815e7a62cb86f14b484cfa8b58a9 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Thu, 19 Dec 2019 23:39:30 +0100 Subject: [PATCH 060/207] peekbuffer for ssl --- .../ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp | 13 +++++++++++++ .../ESP8266WiFi/src/WiFiClientSecureBearSSL.h | 14 ++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp index 773f68cfdc..7620ba1854 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp @@ -368,6 +368,19 @@ int WiFiClientSecure::read(uint8_t *buf, size_t size) { return 0; // If we're connected, no error but no read. } +// return a pointer to available data buffer (size = availableForPeek()) +// semantic forbids any kind of read() before calling peekConsume() +const char* WiFiClientSecure::peekBuffer () +{ + return (const char*)_recvapp_buf; +} + +// consume bytes after use (see peekBuffer) +void WiFiClientSecure::peekConsume (size_t consume) +{ + br_ssl_engine_recvapp_ack(_eng, consume); +} + int WiFiClientSecure::read() { uint8_t c; if (1 == read(&c, 1)) { diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h index 5e288a54f5..0312857c4a 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h @@ -132,10 +132,20 @@ class WiFiClientSecure : public WiFiClient { static bool probeMaxFragmentLength(const char *hostname, uint16_t port, uint16_t len); static bool probeMaxFragmentLength(const String& host, uint16_t port, uint16_t len); - // disallow buffered peek API (for now) - virtual bool peekBufferAPI () const override { return false; } + // peek buffer API is present + virtual bool peekBufferAPI () const override { return true; } virtual int readNow (char* buffer, size_t len) override { return WiFiClientSecure::read((uint8_t*)buffer, len); } + // return number of byte accessible by peekBuffer() + virtual size_t availableForPeek () override { return WiFiClientSecure::available(); } + + // return a pointer to available data buffer (size = availableForPeek()) + // semantic forbids any kind of read() before calling peekConsume() + virtual const char* peekBuffer () override; + + // consume bytes after use (see peekBuffer) + virtual void peekConsume (size_t consume) override; + //////////////////////////////////////////////////// // AxTLS API deprecated warnings to help upgrading From 32f8871c31c94475aee18ad4ed3de0ce4de50df6 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 20 Dec 2019 01:28:30 +0100 Subject: [PATCH 061/207] small fix in device test --- tests/device/test_sw_WiFiServer/test_sw_WiFiServer.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/device/test_sw_WiFiServer/test_sw_WiFiServer.py b/tests/device/test_sw_WiFiServer/test_sw_WiFiServer.py index b64de33e12..86552547cd 100644 --- a/tests/device/test_sw_WiFiServer/test_sw_WiFiServer.py +++ b/tests/device/test_sw_WiFiServer/test_sw_WiFiServer.py @@ -11,6 +11,7 @@ def setup_echo_server(e): global stop_client_thread global client_thread def echo_client_thread(): + time.sleep(1) # let some time for mDNS to start server_address = socket.gethostbyname('esp8266-wfs-test.local') count = 0 while count < 5 and not stop_client_thread: From dfa48c66bfe008ed36a96e062b522a146a57dcb5 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 22 Dec 2019 01:55:33 +0100 Subject: [PATCH 062/207] clean httpclient --- cores/esp8266/Stream.h | 12 +- cores/esp8266/StreamDev.cpp | 60 ++-- .../src/ESP8266HTTPClient.cpp | 297 ++---------------- tests/host/emutest-FSBrowser+ClientPoster.sh | 85 ----- .../HttpClientPost/HttpClientPost.ino | 85 ----- 5 files changed, 67 insertions(+), 472 deletions(-) delete mode 100755 tests/host/emutest-FSBrowser+ClientPoster.sh delete mode 100644 tests/host/otherexamples/HttpClientPost/HttpClientPost.ino diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 267223986f..817d4ed890 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -151,9 +151,19 @@ class Stream: public Print { // - timeout_ms==TimeoutMs::neverExpires: use getTimeout() (when 0: take what's available and immediate return) // - readUntilChar: setting anything in 0..255 will stop transfer when this char is read *and copied too*. virtual size_t to (Print* to, - ssize_t len = -1, + const ssize_t len = -1, esp8266::polledTimeout::oneShotFastMs::timeType timeout = esp8266::polledTimeout::oneShotFastMs::neverExpires /* =>getTimeout() */, int readUntilChar = -1) final; + enum + { + TOSTREAM_SUCCESS = 0, + TOSTREAM_TIMED_OUT, + TOSTREAM_READ_ERROR, + TOSTREAM_WRITE_ERROR, + TOSTREAM_SHORT, + } _last_to = TOSTREAM_SUCCESS; + + decltype(_last_to) getLastTo () const { return _last_to; } //////////////////// end of extensions diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index bc6581ae14..5282688c69 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -6,14 +6,16 @@ using esp8266::polledTimeout::oneShotFastMs; using esp8266::polledTimeout::periodicFastMs; size_t Stream::to (Print* to, - ssize_t len, + const ssize_t len, oneShotFastMs::timeType timeout, int readUntilChar) { + _last_to = TOSTREAM_SUCCESS; + if (len == 0) return 0; // avoid timeout for no requested data - // There are two useful timeout: + // There are two timeouts: // - read (network, serial, ...) // - write (network, serial, ...) // However @@ -22,25 +24,16 @@ size_t Stream::to (Print* to, // So we use getTimeout() for both, // (also when inputTimeoutPossible() is false) + // "neverExpires (default, impossible)" is translated to default timeout oneShotFastMs timedOut(timeout == oneShotFastMs::neverExpires? getTimeout(): timeout); - periodicFastMs yieldNow(5); // yield about every 5ms + + // yield about every 5ms (XXX SHOULD BE A SYSTEM-WIDE CONSTANT?) + periodicFastMs yieldNow(5); + size_t written = 0; - size_t maxLen = std::max((decltype(len))0, len); -#define D 0 -#if D - Serial.printf("#### %dms: pb=%d ruc=%d ml=%d to=%d otp=%d itp=%d avr=%d avp=%d avw=%d\n", - (int)millis(), - (int)peekBufferAPI(), - (int)readUntilChar, - (int)maxLen, - (int)timedOut.getTimeout(), - (int)inputTimeoutPossible(), - (int)outputTimeoutPossible(), - (int)available(), - (int)availableForPeek(), - (int)to->availableForWrite()); -#endif + // len==-1 => maxLen=0 <=> until starvation + size_t maxLen = std::max((decltype(len))0, len); if (peekBufferAPI()) @@ -110,11 +103,15 @@ size_t Stream::to (Print* to, if (c != -1) { w = to->write(c); - if (c == readUntilChar) + if (w != 1) + { + _last_to = TOSTREAM_WRITE_ERROR; break; - assert(w); + } written += 1; timedOut.reset(); + if (c == readUntilChar) + break; } else if (timedOut) break; @@ -139,14 +136,14 @@ size_t Stream::to (Print* to, // no more data can be written, ever break; w = std::min(w, avr); - w = std::min(w, (decltype(w))64); + w = std::min(w, (decltype(w))64); //XXX FIXME 64 is a constant if (w) { char temp[w]; ssize_t r = readNow(temp, w); if (r < 0) { - // do something? + _last_to = TOSTREAM_READ_ERROR; break; } if ((size_t)r < w) @@ -156,7 +153,11 @@ size_t Stream::to (Print* to, } w = to->write(temp, r); written += w; - assert((size_t)r == w); + if ((size_t)r != w) + { + _last_to = TOSTREAM_WRITE_ERROR; + break; + } if (w) timedOut.reset(); } @@ -166,11 +167,12 @@ size_t Stream::to (Print* to, yield(); } -#if D - Serial.printf("#### %dms: transf=%d\n", (int)millis(), (int)written); - if (timedOut) - Serial.printf("#### Timeout!\n"); -#endif - + if (_last_to == TOSTREAM_SUCCESS) + { + if (timedOut) + _last_to = TOSTREAM_TIMED_OUT; + else if (len > 0 && written != len) + _last_to = TOSTREAM_SHORT; + } return written; } diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 8e927b62e9..4ede59499f 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -33,6 +33,19 @@ #include #include +int TO2HTTPC (int streamToError) +{ + switch (streamToError) + { + case Stream::TOSTREAM_TIMED_OUT: return HTTPC_ERROR_READ_TIMEOUT; + case Stream::TOSTREAM_READ_ERROR: return HTTPC_ERROR_NO_STREAM; + case Stream::TOSTREAM_WRITE_ERROR: return HTTPC_ERROR_STREAM_WRITE; + case Stream::TOSTREAM_SHORT: return HTTPC_ERROR_STREAM_WRITE; + default: + case Stream::TOSTREAM_SUCCESS: return 0; + } +} + #if HTTPCLIENT_1_1_COMPATIBLE class TransportTraits { @@ -676,29 +689,9 @@ int HTTPClient::sendRequest(const char * type, const uint8_t * payload, size_t s return returnError(HTTPC_ERROR_SEND_HEADER_FAILED); } -#if 1 - // all of it, with timeout + // transfer all of it, with timeout if (size && StreamPtr(payload, size).to(_client) != size) return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); -#else - // send Payload if needed - if (payload && size > 0) { - size_t bytesWritten = 0; - const uint8_t *p = payload; - while (bytesWritten < size) { - int written; - int towrite = std::min((int)size, (int)HTTP_TCP_BUFFER_SIZE); - written = _client->write(p + bytesWritten, towrite); - if (written < 0) { - return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); - } else if (written == 0) { - return returnError(HTTPC_ERROR_CONNECTION_LOST); - } - bytesWritten += written; - size -= written; - } - } -#endif // handle Server Response (Header) code = handleHeaderResponse(); @@ -772,9 +765,7 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) return returnError(HTTPC_ERROR_SEND_HEADER_FAILED); } -#if 1 - - // all of it, with timeout + // transfer all of it, with timeout size_t transferred = stream->to(_client, size); if (transferred != size) { @@ -782,117 +773,6 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); } -#else - - int buff_size = HTTP_TCP_BUFFER_SIZE; - - int len = size; - int bytesWritten = 0; - - if(len == 0) { - len = -1; - } - - // if possible create smaller buffer then HTTP_TCP_BUFFER_SIZE - if((len > 0) && (len < HTTP_TCP_BUFFER_SIZE)) { - buff_size = len; - } - - // create buffer for read - uint8_t * buff = (uint8_t *) malloc(buff_size); - - if(buff) { - // read all data from stream and send it to server - while(connected() && (stream->available() > 0) && (len > 0 || len == -1)) { - - // get available data size - int sizeAvailable = stream->available(); - - if(sizeAvailable) { - - int readBytes = sizeAvailable; - - // read only the asked bytes - if(len > 0 && readBytes > len) { - readBytes = len; - } - - // not read more the buffer can handle - if(readBytes > buff_size) { - readBytes = buff_size; - } - - // read data - int bytesRead = stream->readBytes(buff, readBytes); - - // write it to Stream - int bytesWrite = _client->write((const uint8_t *) buff, bytesRead); - bytesWritten += bytesWrite; - - // are all Bytes a writen to stream ? - if(bytesWrite != bytesRead) { - DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] short write, asked for %d but got %d retry...\n", bytesRead, bytesWrite); - - // check for write error - if(_client->getWriteError()) { - DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] stream write error %d\n", _client->getWriteError()); - - //reset write error for retry - _client->clearWriteError(); - } - - // some time for the stream - delay(1); - - int leftBytes = (readBytes - bytesWrite); - - // retry to send the missed bytes - bytesWrite = _client->write((const uint8_t *) (buff + bytesWrite), leftBytes); - bytesWritten += bytesWrite; - - if(bytesWrite != leftBytes) { - // failed again - DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] short write, asked for %d but got %d failed.\n", leftBytes, bytesWrite); - free(buff); - return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); - } - } - - // check for write error - if(_client->getWriteError()) { - DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] stream write error %d\n", _client->getWriteError()); - free(buff); - return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); - } - - // count bytes to read left - if(len > 0) { - len -= readBytes; - } - - delay(0); - } else { - delay(1); - } - } - - free(buff); - - if(size && (int) size != bytesWritten) { - DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] Stream payload bytesWritten %d and size %zd mismatch!.\n", bytesWritten, size); - DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] ERROR SEND PAYLOAD FAILED!"); - return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); - } else { - DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] Stream payload written: %d\n", bytesWritten); - } - - } else { - DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] not enough ram! need %d\n", HTTP_TCP_BUFFER_SIZE); - return returnError(HTTPC_ERROR_TOO_LESS_RAM); - } - -#endif - // handle Server Response (Header) return returnError(handleHeaderResponse()); } @@ -964,16 +844,13 @@ int HTTPClient::writeToStream(Stream * stream) int ret = 0; if(_transferEncoding == HTTPC_TE_IDENTITY) { -#if 1 // testok - // len < 0: all of it, with timeout + // len < 0: transfer all of it, with timeout // len >= 0: max:len, with timeout - ret = _client->to(stream, len, 1000); -#else - ret = writeToStreamDataBlock(stream, len); -#endif + ret = _client->to(stream, len); + // have we an error? - if(ret < 0) { - return returnError(ret); + if(_client->getLastTo() != Stream::TOSTREAM_SUCCESS) { + return returnError(TO2HTTPC(ret)); } } else if(_transferEncoding == HTTPC_TE_CHUNKED) { int size = 0; @@ -996,19 +873,11 @@ int HTTPClient::writeToStream(Stream * stream) // data left? if(len > 0) { -#if 1 // read len bytes with timeout - int r = _client->to(stream, len, 100); - if (r != len) + int r = _client->to(stream, len); + if (_client->getLastTo() != Stream::TOSTREAM_SUCCESS) // not all data transferred - return returnError(HTTPC_ERROR_READ_TIMEOUT); -#else - int r = writeToStreamDataBlock(stream, len); - if(r < 0) { - // error in writeToStreamDataBlock - return returnError(r); - } -#endif + return returnError(TO2HTTPC(ret)); ret += r; } else { @@ -1201,14 +1070,8 @@ bool HTTPClient::connect(void) } else { DEBUG_HTTPCLIENT("[HTTP-Client] connect: already connected, try reuse!\n"); } -#if 1 StreamNull devnull; _client->to(&devnull, -1, 0); // clear _client's output (all of it, no timeout) -#else - while(_client->available() > 0) { - _client->read(); - } -#endif return true; } @@ -1308,12 +1171,8 @@ bool HTTPClient::sendHeader(const char * type) DEBUG_HTTPCLIENT("[HTTP-Client] sending request header\n-----\n%s-----\n", header.c_str()); -#if 1 - // all of it, with timeout - return header.to(_client) == header.length(); -#else - return (_client->write((const uint8_t *) header.c_str(), header.length()) == header.length()); -#endif + // transfer all of it, with timeout + return header.to(_client, header.length()) == header.length(); } /** @@ -1425,112 +1284,6 @@ int HTTPClient::handleHeaderResponse() return HTTPC_ERROR_CONNECTION_LOST; } -#if 0 -/** - * write one Data Block to Stream - * @param stream Stream * - * @param size int - * @return < 0 = error >= 0 = size written - */ -int HTTPClient::writeToStreamDataBlock(Stream * stream, int size) -{ - int buff_size = HTTP_TCP_BUFFER_SIZE; - int len = size; // left size to read - int bytesWritten = 0; - - // if possible create smaller buffer then HTTP_TCP_BUFFER_SIZE - if((len > 0) && (len < HTTP_TCP_BUFFER_SIZE)) { - buff_size = len; - } - - // create buffer for read - uint8_t * buff = (uint8_t *) malloc(buff_size); - - if(!buff) { - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] not enough ram! need %d\n", HTTP_TCP_BUFFER_SIZE); - return HTTPC_ERROR_TOO_LESS_RAM; - } - - // read all data from server - while(connected() && (len > 0 || len == -1)) - { - int readBytes = len; - - // not read more the buffer can handle - if(readBytes > buff_size) { - readBytes = buff_size; - } - - // read data - int bytesRead = _client->readBytes(buff, readBytes); - if (!bytesRead) - { - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] input stream timeout\n"); - free(buff); - return HTTPC_ERROR_READ_TIMEOUT; - } - - // write it to Stream - int bytesWrite = stream->write(buff, bytesRead); - bytesWritten += bytesWrite; - - // are all Bytes a writen to stream ? - if(bytesWrite != bytesRead) { - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStream] short write asked for %d but got %d retry...\n", bytesRead, bytesWrite); - - // check for write error - if(stream->getWriteError()) { - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] stream write error %d\n", stream->getWriteError()); - - //reset write error for retry - stream->clearWriteError(); - } - - // some time for the stream - delay(1); - - int leftBytes = (bytesRead - bytesWrite); - - // retry to send the missed bytes - bytesWrite = stream->write((buff + bytesWrite), leftBytes); - bytesWritten += bytesWrite; - - if(bytesWrite != leftBytes) { - // failed again - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStream] short write asked for %d but got %d failed.\n", leftBytes, bytesWrite); - free(buff); - return HTTPC_ERROR_STREAM_WRITE; - } - } - - // check for write error - if(stream->getWriteError()) { - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] stream write error %d\n", stream->getWriteError()); - free(buff); - return HTTPC_ERROR_STREAM_WRITE; - } - - // count bytes to read left - if(len > 0) { - len -= bytesRead; - } - - delay(0); - } - - free(buff); - - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] end of chunk or data (transferred: %d).\n", bytesWritten); - - if((size > 0) && (size != bytesWritten)) { - DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] transferred size %d and request size %d mismatch!.\n", bytesWritten, size); - return HTTPC_ERROR_STREAM_WRITE; - } - - return bytesWritten; -} -#endif - /** * called to handle error return, may disconnect the connection if still exists * @param error diff --git a/tests/host/emutest-FSBrowser+ClientPoster.sh b/tests/host/emutest-FSBrowser+ClientPoster.sh deleted file mode 100755 index 31617ca3bf..0000000000 --- a/tests/host/emutest-FSBrowser+ClientPoster.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/sh - -# Purpose: debugging, timing, optimizing the core data transfers (Stream, FS, network) -# -# instructions: simply run this script -# -# what it does: -# -# Run FSbrowser official example -# use it to upload a random (big) file to emulated FS -# Stop it, copy the emulated FS file for another HTTPClientPost sketch -# Restart the Server -# Start the Poster, it will upload a copy to the server -# end - -set -e - -sizekb=16000 # FS SIZE in KBytes (file size will be the third of this value) -#DEBUG="D=1" # comment this or not -options="-f -b" - -sleep=5 -sizeb=$(($sizekb * 1024)) -fileb=$(($sizeb / 3)) - -make -j $DEBUG ../../libraries/ESP8266WebServer/examples/FSBrowser/FSBrowser -make -j $DEBUG otherexamples/HttpClientPost/HttpClientPost - -killall -9 FSBrowser HttpClientPost || true - -# remove FS -rm -f ./bin/FSBrowser/FSBrowser-littlefs${sizekb}KB -./bin/FSBrowser/FSBrowser $options -S 0 -L ${sizekb} & PIDFSBrowser=$! - -echo "------------------------------" -echo " let server start..." -echo "------------------------------" - -# make a big random file -dd if=/dev/urandom of=randfile bs=${fileb} count=1 - -sleep ${sleep} - -echo "------------------------------" -echo " Uploading $file to FS" -echo "------------------------------" -curl -F "file=@$PWD/randfile" 127.0.0.1:9080/edit -echo "------------------------------" -echo " Uploading to FS (done)" -echo " (wait&kill FSBrowser) " -echo "------------------------------" -sleep ${sleep} -kill -INT $PIDFSBrowser -sleep ${sleep} - -# FSBrowser has generated littlefs backup file, copy it for the other sketch -# This sketch will repost this file to the FSBrowser, so we can debug/tune - -# copy FS to http client sketch -cp ./bin/FSBrowser/FSBrowser-littlefs${sizekb}KB ./bin/HttpClientPost/HttpClientPost-littlefs${sizekb}KB -ls -al ./bin/FSBrowser/FSBrowser-littlefs${sizekb}KB ./bin/HttpClientPost/HttpClientPost-littlefs${sizekb}KB - -echo "------------------------------" -echo " let server start again..." -echo "------------------------------" -./bin/FSBrowser/FSBrowser $options -S 0 -L ${sizekb} & PIDFSBrowser=$! -sleep ${sleep} - -echo "------------------------------" -echo " start uploader sketch" -echo "------------------------------" -# run http client poster sketch -./bin/HttpClientPost/HttpClientPost $options -S 0 -L ${sizekb} & PIDClient=$! - -echo "------------------------------" -echo "Let run the sketches, press enter to kill" -echo "------------------------------" -echo "upload duration:" -read junk - -echo "killing everybody" -kill -INT $PIDClient $PIDFSBrowser || true -sleep 2 -echo "hardkilling everybody" -kill -9 $PIDClient $PIDFSBrowser diff --git a/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino b/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino deleted file mode 100644 index 8d4e289e1d..0000000000 --- a/tests/host/otherexamples/HttpClientPost/HttpClientPost.ino +++ /dev/null @@ -1,85 +0,0 @@ - -// used by ../../test.sh - -#include -#include -#include - -#define FILE "/randfile" -#define URL "http://127.0.0.1:9080/edit" - -#ifndef STASSID -#define STASSID "your-ssid" -#define STAPSK "your-password" -#endif - -WiFiClient client; - -void setup() { - Serial.begin(115200); - Serial.println("starting..."); - WiFi.begin(STASSID, STAPSK); - while (!WiFi.isConnected()) { - Serial.println("."); - delay(250); - } - - if (!LittleFS.begin()) - Serial.println("CLIENT: failed to start LittleFS!"); - - auto f = LittleFS.open(FILE, "r"); - const auto f_size = f.size(); - Serial.printf("CLIENT: size to upload: %d\n", (int)f_size); - - Serial.println("CLIENT: connecting to server..."); - if (!client.connect("127.0.0.1", 9080)) { - Serial.println("\nfailed to connect\n"); - return; - } - - client.println("POST /edit HTTP/1.1"); - client.println("Host: 127.0.0.1"); - client.println("Connection: close"); - client.println("Content-Type: multipart/form-data; boundary=glark"); - client.printf("Content-Length: %d\r\n", f_size); - client.println(); - client.println("--glark"); - client.printf("Content-Disposition: form-data; name=\"%s\"; filename=\"%s\"\r\n", FILE "copy", FILE "copy"); - client.println("Content-Type: application/octet-stream"); - client.println(""); - -#define TESTSTREAM 1 - - auto A = millis(); - Serial.printf("\r\n\r\nCLIENT: ----> upload duration: %lums sent: %d\r\n\r\n", - millis() - A, -#if TESTSTREAM - (int)f.to(client, -1, 0) -#else - (int)client.write(f) -#endif - ); - A = millis() - A; - - Serial.println("CLIENT waiting for server ack..."); - client.println("\r\n--glark--"); - while (!client.available()) - yield(); - - Serial.println("@@@@@@@@@@@@@ CLIENT: server response:"); -#if TESTSTREAM - client.setTimeout(10000); - client.to(Serial, 0); -#else - while (client.available()) - Serial.write(client.read()); -#endif - - Serial.println("@@@@@@@@@@@@@ CLIENT: end"); - client.stop(); - f.close(); -} - -void loop () -{ -} From e521195598f658c62c88171c3be330d33d882b3a Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 22 Dec 2019 01:58:41 +0100 Subject: [PATCH 063/207] remove debug print --- cores/esp8266/StreamDev.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index 5282688c69..6fe815c06a 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -146,11 +146,6 @@ size_t Stream::to (Print* to, _last_to = TOSTREAM_READ_ERROR; break; } - if ((size_t)r < w) - { - Serial.printf(":to read(=%zd)write(temp, r); written += w; if ((size_t)r != w) @@ -171,7 +166,7 @@ size_t Stream::to (Print* to, { if (timedOut) _last_to = TOSTREAM_TIMED_OUT; - else if (len > 0 && written != len) + else if (len > 0 && (ssize_t)written != len) _last_to = TOSTREAM_SHORT; } return written; From 8d3e51fd388687e7397928e215a849e03117ef00 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 22 Dec 2019 02:32:29 +0100 Subject: [PATCH 064/207] comments --- cores/esp8266/Print.h | 4 ++-- cores/esp8266/Stream.h | 3 ++- cores/esp8266/WString.h | 21 ++++++++++++--------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/cores/esp8266/Print.h b/cores/esp8266/Print.h index 5ba38374d1..4d801c49e7 100644 --- a/cores/esp8266/Print.h +++ b/cores/esp8266/Print.h @@ -109,8 +109,8 @@ class Print { virtual int availableForWrite(); // return type: int - // by default timeout is enabled (outgoing network,serial.. data) - // (children can override to false) + // by default write timeout is possible (outgoing data from network,serial..) + // (children can override to false (like String)) virtual bool outputTimeoutPossible () const { return true; } }; diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 817d4ed890..c6de3d7b0d 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -139,7 +139,8 @@ class Stream: public Print { // consume bytes after peekBuffer() use virtual void peekConsume (size_t consume) { (void)consume; } - // by default timeout is enabled (incoming network,serial.. data) + // by default read timeout is possible (incoming data from network,serial..) + // (children can override to false (like String)) virtual bool inputTimeoutPossible () const { return true; } //////////////////// extensions: Stream streams diff --git a/cores/esp8266/WString.h b/cores/esp8266/WString.h index 173963be06..38fc48e660 100644 --- a/cores/esp8266/WString.h +++ b/cores/esp8266/WString.h @@ -323,19 +323,22 @@ class String: public Stream { #endif ///////////////////////////////////////////// - // Stream API: + // Print/Stream/peekBuffer API: protected: // peekPointer is used with peekBufferAPI, - // on peekConsume(), string can either: - // - be really consumed -- case when peekPointer==-1 - // - marked as read -- peekPointer is increased - // (marked to not be consumed by default) + // on peekConsume(), chars can either: + // - be really consumed = disappeared + // (case when peekPointer==-1) + // - marked as read + // (peekPointer >=0 is increased) int peekPointer = 0; public: + //// Stream: + size_t write(const uint8_t *buffer, size_t size) override; size_t write(uint8_t data) override; @@ -350,7 +353,6 @@ class String: public Stream { virtual size_t availableForPeek () override { return available(); } virtual const char* peekBuffer () override; virtual void peekConsume (size_t consume) override; - virtual bool outputTimeoutPossible () const override { return false; } virtual bool inputTimeoutPossible () const override { return false; } virtual int read (char* buffer, size_t len) /*should override*/; @@ -362,9 +364,10 @@ class String: public Stream { return read(buffer, len); } - //// Print - virtual int availableForWrite() override { return 256; } - // or biggestChunk()/4 or reserved-length? + //// Print: + + virtual bool outputTimeoutPossible () const override { return false; } + virtual int availableForWrite() override { return 64; } // or biggestChunk()/4 or max(1,reserved-length)? }; class StringSumHelper: public String { From 3345ef19bda920a554e1d9d99811f6821137bc55 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 22 Dec 2019 02:53:16 +0100 Subject: [PATCH 065/207] further cleaning --- cores/esp8266/StreamDev.h | 33 +++++++++---------- .../src/ESP8266WebServer-impl.h | 9 +++-- 2 files changed, 20 insertions(+), 22 deletions(-) diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 5ec689e2b4..c9ec883394 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -5,8 +5,9 @@ #include /////////////////////////////////////////////// -// - black hole in, swallow everything, availableForWrite = infinite -// - black hole out, nothing to read, available = 0 +// /dev/null +// - black hole as output, swallow everything, availableForWrite = infinite +// - black hole as input, nothing to read, available = 0 class StreamNull: public Stream { @@ -29,8 +30,8 @@ class StreamNull: public Stream /////////////////////////////////////////////// // /dev/zero -// - black hole in, swallow everything, availableForWrite = infinite -// - white hole out, gives infinity to read, available = infinite +// - black hole as output, swallow everything, availableForWrite = infinite +// - big bang as input, gives infinity to read, available = infinite class StreamZero: public StreamNull { @@ -49,8 +50,8 @@ class StreamZero: public StreamNull /////////////////////////////////////////////// // static buffer (in flash or ram) -// - black hole in, swallow everything, availableForWrite = infinite -// - Stream buffer out +// - black hole as output, swallow everything, availableForWrite = infinite +// - Stream buffer out, resettable class StreamPtr: public StreamNull { @@ -73,17 +74,15 @@ class StreamPtr: public StreamNull virtual int peek() override { return _peekPointer < _size? _buffer[_peekPointer]: -1; } virtual size_t readBytes(char* buffer, size_t len) override { - if (_peekPointer < _size) - { - size_t cpylen = std::min(_size - _peekPointer, len); - if (_in_flash) - memcpy_P(buffer, _buffer + _peekPointer, cpylen); - else - memcpy(buffer, _buffer + _peekPointer, cpylen); - _peekPointer += cpylen; - return cpylen; - } - return 0; + if (_peekPointer >= _size) + return 0; + size_t cpylen = std::min(_size - _peekPointer, len); + if (_in_flash) + memcpy_P(buffer, _buffer + _peekPointer, cpylen); + else + memcpy(buffer, _buffer + _peekPointer, cpylen); + _peekPointer += cpylen; + return cpylen; } virtual int readNow(char* buffer, size_t len) override { return readBytes(buffer, len); } diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index 5edfb4cb1d..f39f8a51ca 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -28,6 +28,7 @@ #include "ESP8266WebServer.h" #include "FS.h" #include "detail/RequestHandlersImpl.h" +#include //#define DEBUG_ESP_HTTP_SERVER #ifdef DEBUG_ESP_PORT @@ -435,15 +436,13 @@ void ESP8266WebServerTemplate::send(int code, const char* content_ty //if(code == 200 && content.length() == 0 && _contentLength == CONTENT_LENGTH_NOT_SET) // _contentLength = CONTENT_LENGTH_UNKNOWN; _prepareHeader(header, code, content_type, content.length()); -#if 1 - size_t sent = StreamPtr(header.c_str(), header.length()).to(_currentClient); // all of it, with timeout + size_t sent = StreamPtr(header.c_str(), header.length()).to(_currentClient); // transfer all of it, with timeout + (void)sent; #ifdef DEBUG_ESP_HTTP_SERVER if (sent != header.length()) DEBUG_OUTPUT.println("HTTPServer: error: sent %zd on %zd bytes\n", sent, header.length()); #endif -#else - _currentClient.write((const uint8_t *)header.c_str(), header.length()); -#endif + if(content.length()) sendContent(content); } From 7cd36a9e586340a843f9c32006b32754d2787874 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 22 Dec 2019 02:56:05 +0100 Subject: [PATCH 066/207] fix api change --- libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index f39f8a51ca..35d88c97a9 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -436,7 +436,7 @@ void ESP8266WebServerTemplate::send(int code, const char* content_ty //if(code == 200 && content.length() == 0 && _contentLength == CONTENT_LENGTH_NOT_SET) // _contentLength = CONTENT_LENGTH_UNKNOWN; _prepareHeader(header, code, content_type, content.length()); - size_t sent = StreamPtr(header.c_str(), header.length()).to(_currentClient); // transfer all of it, with timeout + size_t sent = StreamPtr(header.c_str(), header.length()).to(&_currentClient); // transfer all of it, with timeout (void)sent; #ifdef DEBUG_ESP_HTTP_SERVER if (sent != header.length()) From a5658fe922241f0df940cba858389900819dd611 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 22 Dec 2019 09:32:53 +0100 Subject: [PATCH 067/207] fix api --- libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino index 58ea1dacfd..bf7e867cb5 100644 --- a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino @@ -90,7 +90,7 @@ void loop() { else if (t == 3) { // stream to print, possibly with only one copy - client.to(client, -1, 0); // all of it, immediate return (no timeout) + client.to(&client, -1, 0); // transfer everything possible, immediate return (no timeout) } } From 474d5a130605394513310a377041475885f828c7 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 1 Jan 2020 18:54:29 +0100 Subject: [PATCH 068/207] comments --- cores/esp8266/Stream.h | 94 +++++++++++++------ cores/esp8266/StreamDev.cpp | 26 +++-- .../src/ESP8266HTTPClient.cpp | 14 +-- 3 files changed, 86 insertions(+), 48 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index c6de3d7b0d..88494c6a07 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -109,7 +109,10 @@ class Stream: public Print { virtual String readString(); String readStringUntil(char terminator); - // ::read(buf, len): conflicting returned type: + //////////////////// extension: readNow (is ::read() with unified signature) + // (supposed to be internally used, and ephemeral) + // + // about ::read(buf, len): conflicting returned type: // - `int` in arduino's Client:: // - `size_t` in esp8266 API (HardwareSerial::, FS::) // - not existent in arduino's Stream:: @@ -117,54 +120,83 @@ class Stream: public Print { // => adding int ::readNow(buf, len) for now (following official `int Client::read(buf, len))` // // int ::readNow(buf, len) - // read at most len bytes, returns effectively transfered bytes (can be less than len) - // immediate return when no more data are available (no timeout) - virtual int readNow(char* buffer, size_t len); - virtual int readNow(uint8_t* buffer, size_t len) final { return readNow((char*)buffer, len); } + // read at most len bytes, returns effectively transfered bytes (can be less than 'len') + // with no timeout: immediate return when no more data are available + virtual int readNow (char* buffer, size_t len); + virtual int readNow (uint8_t* buffer, size_t len) final { return readNow((char*)buffer, len); } - //////////////////// extensions: direct access to input buffer + //////////////////// extension: direct access to input buffer + // for providing, when possible, a pointer to available data for read - // inform user and ::to() on effective buffered peek API implementation + // informs user and ::to() on effective buffered peek API implementation + // by default: not available virtual bool peekBufferAPI () const { return false; } - // return number of byte accessible by peekBuffer() + // returns number of byte accessible by peekBuffer() virtual size_t availableForPeek () { return 0; } - // return a pointer to available data buffer (size = availableForPeek()) - // semantic forbids any kind of read() - // after calling peekBuffer() - // and before calling peekConsume() + // returns a pointer to available data buffer (size = availableForPeek()) + // semantic forbids any kind of ::read() + // - after calling peekBuffer() + // - and before calling peekConsume() virtual const char* peekBuffer () { return nullptr; } - // consume bytes after peekBuffer() use + // consumes bytes after peekBuffer() use + // (then ::read() is allowed) virtual void peekConsume (size_t consume) { (void)consume; } // by default read timeout is possible (incoming data from network,serial..) - // (children can override to false (like String)) + // children can override to false (like String::) + // (outputTimeoutPossible() is defined in Print::) virtual bool inputTimeoutPossible () const { return true; } //////////////////// extensions: Stream streams - // Stream::to() - // transfer from `Stream::` to `Print::` at most maxlen bytes and return number of transfered bytes - // (uses 1-copy peekBuffer API when available, or transfer through a 2-copies local stack space) - // - maxLen<0 will transfer until input starvation or saturated output - // - timeout_ms==TimeoutMs::neverExpires: use getTimeout() (when 0: take what's available and immediate return) - // - readUntilChar: setting anything in 0..255 will stop transfer when this char is read *and copied too*. + // + // Stream::to() uses 1-copy transfers when peekBuffer API is + // available, or makes a regular transfer through a local temporary + // stack buffer. + // + // By default "source->to(&dest)" transfers everything until + // available (read or write) gets to 0 and a timeout occurs. + // + // "source->to(&dest, maxLen)" is like above but also returns when + // maxLen bytes are transferred. + // + // More generally ::to() will transfer as much as possible until: + // - maxLen (>=0) bytes are transferred (without timeout), + // - or readUntilChar (>=0) is reached (without timeout), + // - or available for read or write gets to zero (after timeout). + // + // Timeout value is by default thisStream->getTimeout() but it can + // be set to "alwaysExpired" (=0) or any value within oneShotMs + // allowed range. + // + // Return value: + // >0: the number of transfered bytes + // 0: nothing has been transfered, getLastTo() may contain an error reason + // + // Notes: + // - readUntilChar is copied and counted + // - for efficiency: Stream classes should implement peekAPI when possible + // - for efficiency: Stream classes should implement {input,output}TimeoutPossible() + + using oneShotMs = esp8266::polledTimeout::oneShotFastMs; + virtual size_t to (Print* to, - const ssize_t len = -1, - esp8266::polledTimeout::oneShotFastMs::timeType timeout = esp8266::polledTimeout::oneShotFastMs::neverExpires /* =>getTimeout() */, + const ssize_t maxLen = -1, + oneShotMs::timeType timeout = oneShotMs::neverExpires /* =>getTimeout() */, int readUntilChar = -1) final; - enum + typedef enum { - TOSTREAM_SUCCESS = 0, - TOSTREAM_TIMED_OUT, - TOSTREAM_READ_ERROR, - TOSTREAM_WRITE_ERROR, - TOSTREAM_SHORT, - } _last_to = TOSTREAM_SUCCESS; - - decltype(_last_to) getLastTo () const { return _last_to; } + STREAMTO_SUCCESS = 0, + STREAMTO_TIMED_OUT, + STREAMTO_READ_ERROR, + STREAMTO_WRITE_ERROR, + STREAMTO_SHORT, + } toReport_e; + + toReport_e getLastTo () /*const*/ { return (toReport_e)getWriteError(); } //////////////////// end of extensions diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index 6fe815c06a..3ce60252ff 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -10,10 +10,10 @@ size_t Stream::to (Print* to, oneShotFastMs::timeType timeout, int readUntilChar) { - _last_to = TOSTREAM_SUCCESS; + setWriteError(STREAMTO_SUCCESS); if (len == 0) - return 0; // avoid timeout for no requested data + return 0; // conveniently avoids timeout for no requested data // There are two timeouts: // - read (network, serial, ...) @@ -33,7 +33,7 @@ size_t Stream::to (Print* to, size_t written = 0; // len==-1 => maxLen=0 <=> until starvation - size_t maxLen = std::max((decltype(len))0, len); + size_t maxLen = std::max((ssize_t)0, len); if (peekBufferAPI()) @@ -105,7 +105,7 @@ size_t Stream::to (Print* to, w = to->write(c); if (w != 1) { - _last_to = TOSTREAM_WRITE_ERROR; + setWriteError(STREAMTO_WRITE_ERROR); break; } written += 1; @@ -143,14 +143,14 @@ size_t Stream::to (Print* to, ssize_t r = readNow(temp, w); if (r < 0) { - _last_to = TOSTREAM_READ_ERROR; + setWriteError(STREAMTO_READ_ERROR); break; } w = to->write(temp, r); written += w; if ((size_t)r != w) { - _last_to = TOSTREAM_WRITE_ERROR; + setWriteError(STREAMTO_WRITE_ERROR); break; } if (w) @@ -162,12 +162,18 @@ size_t Stream::to (Print* to, yield(); } - if (_last_to == TOSTREAM_SUCCESS) + if (getWriteError() == STREAMTO_SUCCESS) { if (timedOut) - _last_to = TOSTREAM_TIMED_OUT; - else if (len > 0 && (ssize_t)written != len) - _last_to = TOSTREAM_SHORT; + setWriteError(STREAMTO_TIMED_OUT); + else if ((ssize_t)written != len) + // This is happening when source cannot timeout (ex: a String) + // but has not enough data, or a dest has closed or cannot + // timeout but is too small (String, buffer...) + // + // Mark it as an error because user usually wants to get what is + // asked for. + setWriteError(STREAMTO_SHORT); } return written; } diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 7a9edb6f2e..57c109aac8 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -37,12 +37,12 @@ int TO2HTTPC (int streamToError) { switch (streamToError) { - case Stream::TOSTREAM_TIMED_OUT: return HTTPC_ERROR_READ_TIMEOUT; - case Stream::TOSTREAM_READ_ERROR: return HTTPC_ERROR_NO_STREAM; - case Stream::TOSTREAM_WRITE_ERROR: return HTTPC_ERROR_STREAM_WRITE; - case Stream::TOSTREAM_SHORT: return HTTPC_ERROR_STREAM_WRITE; + case Stream::STREAMTO_TIMED_OUT: return HTTPC_ERROR_READ_TIMEOUT; + case Stream::STREAMTO_READ_ERROR: return HTTPC_ERROR_NO_STREAM; + case Stream::STREAMTO_WRITE_ERROR: return HTTPC_ERROR_STREAM_WRITE; + case Stream::STREAMTO_SHORT: return HTTPC_ERROR_STREAM_WRITE; default: - case Stream::TOSTREAM_SUCCESS: return 0; + case Stream::STREAMTO_SUCCESS: return 0; } } @@ -849,7 +849,7 @@ int HTTPClient::writeToStream(Stream * stream) ret = _client->to(stream, len); // have we an error? - if(_client->getLastTo() != Stream::TOSTREAM_SUCCESS) { + if(_client->getLastTo() != Stream::STREAMTO_SUCCESS) { return returnError(TO2HTTPC(ret)); } } else if(_transferEncoding == HTTPC_TE_CHUNKED) { @@ -875,7 +875,7 @@ int HTTPClient::writeToStream(Stream * stream) if(len > 0) { // read len bytes with timeout int r = _client->to(stream, len); - if (_client->getLastTo() != Stream::TOSTREAM_SUCCESS) + if (_client->getLastTo() != Stream::STREAMTO_SUCCESS) // not all data transferred return returnError(TO2HTTPC(ret)); ret += r; From 5d4a507960d5bd4091bb043a72b5a6b573027555 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 1 Jan 2020 19:06:54 +0100 Subject: [PATCH 069/207] fix HTTPClient merge --- .../ESP8266HTTPClient/src/ESP8266HTTPClient.cpp | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 57c109aac8..88159c5316 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -1140,14 +1140,11 @@ bool HTTPClient::sendHeader(const char * type) } header += F(" HTTP/1."); - header += type; - header += ' '; - if (_uri.length()) - header += _uri; - else - header += '/'; - header += F(" HTTP/1."); - header += (char)('0' + !_useHTTP10); + if(_useHTTP10) { + header += '0'; + } else { + header += '1'; + } header += F("\r\nHost: "); header += _host; From d20e8676531040ca06233ef5332e0f28435a886d Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 1 Jan 2020 21:00:35 +0100 Subject: [PATCH 070/207] reduce diff --- libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 88159c5316..2d38346d90 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -828,9 +828,9 @@ WiFiClient* HTTPClient::getStreamPtr(void) * @param stream Stream * * @return bytes written ( negative values are error codes ) */ - int HTTPClient::writeToStream(Stream * stream) { + if(!stream) { return returnError(HTTPC_ERROR_NO_STREAM); } From 630b6b60c423ab7a15acc3882b212b6736c46ff7 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 1 Jan 2020 23:22:33 +0100 Subject: [PATCH 071/207] various fixes --- cores/esp8266/Print.h | 2 +- cores/esp8266/Stream.h | 2 +- cores/esp8266/StreamDev.cpp | 5 +- cores/esp8266/StreamDev.h | 4 +- cores/esp8266/WString.h | 4 +- .../examples/WiFiEcho/WiFiEcho.ino | 56 +++++++++++++------ libraries/ESP8266WiFi/src/WiFiClient.cpp | 11 ++-- libraries/ESP8266WiFi/src/WiFiClient.h | 3 + tests/host/common/mock.h | 2 +- 9 files changed, 59 insertions(+), 30 deletions(-) diff --git a/cores/esp8266/Print.h b/cores/esp8266/Print.h index 4d801c49e7..b3ac1dc3b3 100644 --- a/cores/esp8266/Print.h +++ b/cores/esp8266/Print.h @@ -111,7 +111,7 @@ class Print { // by default write timeout is possible (outgoing data from network,serial..) // (children can override to false (like String)) - virtual bool outputTimeoutPossible () const { return true; } + virtual bool outputTimeoutPossible () { return true; } }; #endif diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 88494c6a07..6caa7264f7 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -148,7 +148,7 @@ class Stream: public Print { // by default read timeout is possible (incoming data from network,serial..) // children can override to false (like String::) // (outputTimeoutPossible() is defined in Print::) - virtual bool inputTimeoutPossible () const { return true; } + virtual bool inputTimeoutPossible () { return true; } //////////////////// extensions: Stream streams // Stream::to() diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index 3ce60252ff..d616168b2b 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -135,6 +135,7 @@ size_t Stream::to (Print* to, if (w == 0 && !to->outputTimeoutPossible()) // no more data can be written, ever break; + w = std::min(w, avr); w = std::min(w, (decltype(w))64); //XXX FIXME 64 is a constant if (w) @@ -164,9 +165,9 @@ size_t Stream::to (Print* to, if (getWriteError() == STREAMTO_SUCCESS) { - if (timedOut) + if (timeout && timedOut) setWriteError(STREAMTO_TIMED_OUT); - else if ((ssize_t)written != len) + else if (len > 0 && (ssize_t)written != len) // This is happening when source cannot timeout (ex: a String) // but has not enough data, or a dest has closed or cannot // timeout but is too small (String, buffer...) diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index c9ec883394..667c6ba4ec 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -24,8 +24,8 @@ class StreamNull: public Stream virtual int peek() override { return -1; } virtual size_t readBytes(char* buffer, size_t len) override { (void)buffer; (void)len; return 0; } virtual int readNow(char* buffer, size_t len) override { (void)buffer; (void)len; return 0; } - virtual bool outputTimeoutPossible () const override { return false; } - virtual bool inputTimeoutPossible () const override { return false; } + virtual bool outputTimeoutPossible () override { return false; } + virtual bool inputTimeoutPossible () override { return false; } }; /////////////////////////////////////////////// diff --git a/cores/esp8266/WString.h b/cores/esp8266/WString.h index 38fc48e660..caf1d4b3e4 100644 --- a/cores/esp8266/WString.h +++ b/cores/esp8266/WString.h @@ -353,7 +353,7 @@ class String: public Stream { virtual size_t availableForPeek () override { return available(); } virtual const char* peekBuffer () override; virtual void peekConsume (size_t consume) override; - virtual bool inputTimeoutPossible () const override { return false; } + virtual bool inputTimeoutPossible () override { return false; } virtual int read (char* buffer, size_t len) /*should override*/; void peekPointerSetConsume () { peekPointer = -1; } @@ -366,7 +366,7 @@ class String: public Stream { //// Print: - virtual bool outputTimeoutPossible () const override { return false; } + virtual bool outputTimeoutPossible () override { return false; } virtual int availableForWrite() override { return 64; } // or biggestChunk()/4 or max(1,reserved-length)? }; diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino index bf7e867cb5..75c9cc949d 100644 --- a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino @@ -5,7 +5,7 @@ */ #include - +#include #include // std::min #ifndef STASSID @@ -16,11 +16,14 @@ #define STACK_PROTECTOR 128 const int port = 23; -int t = 3; +int t = 1; // test (1, 2 or 3, see below) WiFiServer server(port); WiFiClient client; +constexpr uint32_t breathMs = 100; +esp8266::polledTimeout::oneShotFastMs enoughMs(breathMs); + void setup() { Serial.begin(115200); @@ -44,7 +47,8 @@ void setup() { Serial.print("Ready! Use 'telnet "); Serial.print(WiFi.localIP()); - Serial.printf(" %d' to connect\n", port); + Serial.printf(" %d' to try echo, use a bandwidth meter and try typing 1, 2 or 3 on console during transfer\n", port); + } void loop() { @@ -57,25 +61,24 @@ void loop() { if (Serial.available()) switch (Serial.read()) { - case '1': t = 1; break; - case '2': t = 2; break; - case '3': t = 3; break; + case '1': t = 1; Serial.println("byte-by-byte"); break; + case '2': t = 2; Serial.println("through buffer"); break; + case '3': t = 3; Serial.println("direct access"); break; } - if (t == 1) - { + enoughMs.reset(breathMs); + + if (t == 1) { // byte by byte - while (client.available() && client.availableForWrite()) - { + while (client.available() && client.availableForWrite() && !enoughMs) { // working char by char is not efficient client.write(client.read()); } } - else if (t == 2) - { + else if (t == 2) { // block by block through a local buffer (2 copies) - while (client.available() && client.availableForWrite()) { + while (client.available() && client.availableForWrite() && !enoughMs) { size_t maxTo = std::min(client.available(), client.availableForWrite()); maxTo = std::min(maxTo, (size_t)STACK_PROTECTOR); uint8_t buf[maxTo]; @@ -87,10 +90,31 @@ void loop() { } } - else if (t == 3) - { + else if (t == 3) { // stream to print, possibly with only one copy - client.to(&client, -1, 0); // transfer everything possible, immediate return (no timeout) + +#if CORE_MOCK + + // (without limit, with a bandwitdh tester, this call would endlessly transfer data) + client.to(&client, 10000000, 0); // transfer large chunks, no timeout + if (client.getLastTo() == Stream::STREAMTO_SHORT) + // don't really care about really transfered bytes + { + client.clearWriteError(); + } + +#else + + client.to(&client, -1, 0); // on esp: no size limit, no timeout + +#endif + switch (client.getLastTo()) { + case Stream::STREAMTO_SUCCESS: break; + case Stream::STREAMTO_TIMED_OUT: Serial.println("Stream::to: timeout"); break; + case Stream::STREAMTO_READ_ERROR: Serial.println("Stream::to: read error"); break; + case Stream::STREAMTO_WRITE_ERROR: Serial.println("Stream::to: write error"); break; + case Stream::STREAMTO_SHORT: Serial.println("Stream::to: short transfer"); break; + } } } diff --git a/libraries/ESP8266WiFi/src/WiFiClient.cpp b/libraries/ESP8266WiFi/src/WiFiClient.cpp index 1a09d95e2d..96d4bc4109 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.cpp +++ b/libraries/ESP8266WiFi/src/WiFiClient.cpp @@ -208,7 +208,7 @@ bool WiFiClient::getSync() const int WiFiClient::availableForWrite () { - return _client? (int)_client->availableForWrite(): 0; + return connected()? (int)_client->availableForWrite(): 0; } size_t WiFiClient::write(uint8_t b) @@ -255,7 +255,7 @@ size_t WiFiClient::write_P(PGM_P buf, size_t size) int WiFiClient::available() { if (!_client) - return false; + return 0; int result = _client->getSize(); @@ -433,17 +433,18 @@ bool WiFiClient::peekBufferAPI () const // semantic forbids any kind of read() before calling peekConsume() const char* WiFiClient::peekBuffer () { - return _client->peekBuffer(); + return _client? _client->peekBuffer(): nullptr; } // return number of byte accessible by peekBuffer() size_t WiFiClient::availableForPeek () { - return _client->availableForPeek(); + return _client? _client->availableForPeek(): 0; } // consume bytes after use (see peekBuffer) void WiFiClient::peekConsume (size_t consume) { - return _client->peekConsume(consume); + if (_client) + _client->peekConsume(consume); } diff --git a/libraries/ESP8266WiFi/src/WiFiClient.h b/libraries/ESP8266WiFi/src/WiFiClient.h index 04deb2ad8c..3fe4f23b1c 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.h +++ b/libraries/ESP8266WiFi/src/WiFiClient.h @@ -143,6 +143,9 @@ class WiFiClient : public Client, public SList { // consume bytes after use (see peekBuffer) virtual void peekConsume (size_t consume) override; + virtual bool outputTimeoutPossible () override { return connected(); } + virtual bool inputTimeoutPossible () override { return connected(); } + protected: static int8_t _s_connected(void* arg, void* tpcb, int8_t err); diff --git a/tests/host/common/mock.h b/tests/host/common/mock.h index b4a361afda..edd392990e 100644 --- a/tests/host/common/mock.h +++ b/tests/host/common/mock.h @@ -87,7 +87,7 @@ typedef uint32_t uint32; #include #include -inline uint32_t esp_get_cycle_count() { return millis(); } +inline uint32_t esp_get_cycle_count() { return millis() * (F_CPU / 1000); } #include From 3edf028a16edf2a196c81fe1100b21e4059b8e2c Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Wed, 1 Jan 2020 23:24:33 +0100 Subject: [PATCH 072/207] fix httpclient error handling --- libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 88159c5316..9ea4eb32fd 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -850,7 +850,7 @@ int HTTPClient::writeToStream(Stream * stream) // have we an error? if(_client->getLastTo() != Stream::STREAMTO_SUCCESS) { - return returnError(TO2HTTPC(ret)); + return returnError(TO2HTTPC(_client->getLastTo())); } } else if(_transferEncoding == HTTPC_TE_CHUNKED) { int size = 0; @@ -877,7 +877,7 @@ int HTTPClient::writeToStream(Stream * stream) int r = _client->to(stream, len); if (_client->getLastTo() != Stream::STREAMTO_SUCCESS) // not all data transferred - return returnError(TO2HTTPC(ret)); + return returnError(TO2HTTPC(_client->getLastTo())); ret += r; } else { From 03519da30224fc320976826ed61f3e598c3d2fce Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 1 Jan 2020 23:45:29 +0100 Subject: [PATCH 073/207] fix --- tests/host/common/MockEsp.cpp | 5 +++++ tests/host/common/mock.h | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/host/common/MockEsp.cpp b/tests/host/common/MockEsp.cpp index e6768d83ba..d9e891d4df 100644 --- a/tests/host/common/MockEsp.cpp +++ b/tests/host/common/MockEsp.cpp @@ -221,6 +221,11 @@ void EspClass::resetFreeContStack() } uint32_t EspClass::getCycleCount() +{ + return esp_get_cycle_count(); +} + +uint32_t esp_get_cycle_count() { timeval t; gettimeofday(&t, NULL); diff --git a/tests/host/common/mock.h b/tests/host/common/mock.h index edd392990e..1cf5821efd 100644 --- a/tests/host/common/mock.h +++ b/tests/host/common/mock.h @@ -87,7 +87,7 @@ typedef uint32_t uint32; #include #include -inline uint32_t esp_get_cycle_count() { return millis() * (F_CPU / 1000); } +uint32_t esp_get_cycle_count(); #include From bc2b59a99b7483efcf7263c7761845930d3f44f3 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Thu, 2 Jan 2020 22:01:29 +0100 Subject: [PATCH 074/207] Stream::to(): changed argument order --- cores/esp8266/Stream.h | 4 +- cores/esp8266/StreamDev.cpp | 44 +++++++++++++++---- .../src/ESP8266HTTPClient.cpp | 2 +- .../examples/WiFiEcho/WiFiEcho.ino | 16 +------ tests/host/Makefile | 2 +- 5 files changed, 40 insertions(+), 28 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 6caa7264f7..d15ed07382 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -185,8 +185,8 @@ class Stream: public Print { virtual size_t to (Print* to, const ssize_t maxLen = -1, - oneShotMs::timeType timeout = oneShotMs::neverExpires /* =>getTimeout() */, - int readUntilChar = -1) final; + int readUntilChar = -1, + oneShotMs::timeType timeout = oneShotMs::neverExpires /* =>getTimeout() */) final; typedef enum { STREAMTO_SUCCESS = 0, diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index d616168b2b..4c81717325 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -7,8 +7,8 @@ using esp8266::polledTimeout::periodicFastMs; size_t Stream::to (Print* to, const ssize_t len, - oneShotFastMs::timeType timeout, - int readUntilChar) + int readUntilChar, + oneShotFastMs::timeType timeout) { setWriteError(STREAMTO_SUCCESS); @@ -71,13 +71,22 @@ size_t Stream::to (Print* to, { peekConsume(w); written += w; - timedOut.reset(); + if (maxLen) + timedOut.reset(); if (foundChar) break; } } - else if (timedOut) + + if (!maxLen && !w) + // nothing has been transfered and no specific size requested + break; + + if (timedOut) + // either (maxLen>0) nothing has been transfered for too long + // or (maxLen=0) too much time has been spent here break; + if (yieldNow) yield(); } @@ -109,12 +118,21 @@ size_t Stream::to (Print* to, break; } written += 1; - timedOut.reset(); + if (maxLen) + timedOut.reset(); if (c == readUntilChar) break; } - else if (timedOut) + + if (!maxLen && !w) + // nothing has been transfered and no specific size requested break; + + if (timedOut) + // either (maxLen>0) nothing has been transfered for too long + // or (maxLen=0) too much time has been spent here + break; + if (yieldNow) yield(); } @@ -154,20 +172,28 @@ size_t Stream::to (Print* to, setWriteError(STREAMTO_WRITE_ERROR); break; } - if (w) + if (maxLen && w) timedOut.reset(); } + + if (!maxLen && !w) + // nothing has been transfered and no specific size requested + break; + if (timedOut) + // either (maxLen>0) nothing has been transfered for too long + // or (maxLen=0) too much time has been spent here break; + if (yieldNow) yield(); } - if (getWriteError() == STREAMTO_SUCCESS) + if (getWriteError() == STREAMTO_SUCCESS && maxLen > 0) { if (timeout && timedOut) setWriteError(STREAMTO_TIMED_OUT); - else if (len > 0 && (ssize_t)written != len) + else if ((ssize_t)written != len) // This is happening when source cannot timeout (ex: a String) // but has not enough data, or a dest has closed or cannot // timeout but is too small (String, buffer...) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 963db4c5be..c1e9cc5dec 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -1072,7 +1072,7 @@ bool HTTPClient::connect(void) DEBUG_HTTPCLIENT("[HTTP-Client] connect: already connected, try reuse!\n"); } StreamNull devnull; - _client->to(&devnull, -1, 0); // clear _client's output (all of it, no timeout) + _client->to(&devnull, -1, -1, 0); // clear _client's output (all of it, no timeout) return true; } diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino index 75c9cc949d..9509b5bbdb 100644 --- a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino @@ -92,22 +92,8 @@ void loop() { else if (t == 3) { // stream to print, possibly with only one copy + client.to(&client); -#if CORE_MOCK - - // (without limit, with a bandwitdh tester, this call would endlessly transfer data) - client.to(&client, 10000000, 0); // transfer large chunks, no timeout - if (client.getLastTo() == Stream::STREAMTO_SHORT) - // don't really care about really transfered bytes - { - client.clearWriteError(); - } - -#else - - client.to(&client, -1, 0); // on esp: no size limit, no timeout - -#endif switch (client.getLastTo()) { case Stream::STREAMTO_SUCCESS: break; case Stream::STREAMTO_TIMED_OUT: Serial.println("Stream::to: timeout"); break; diff --git a/tests/host/Makefile b/tests/host/Makefile index 0996944b92..78c3acb007 100644 --- a/tests/host/Makefile +++ b/tests/host/Makefile @@ -22,7 +22,7 @@ GENHTML ?= genhtml ifeq ($(FORCE32),1) SIZEOFLONG = $(shell echo 'int main(){return sizeof(long);}'|$(CXX) -m32 -x c++ - -o sizeoflong 2>/dev/null && ./sizeoflong; echo $$?; rm -f sizeoflong;) ifneq ($(SIZEOFLONG),4) -$(warning Cannot compile in 32 bit mode, switching to native mode) +$(warning Cannot compile in 32 bit mode (g++-multilib is missing?), switching to native mode) else N32 = 32 M32 = -m32 From 72c3237d85c9582a4e6bd1503914cff060e266c3 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Thu, 2 Jan 2020 22:32:36 +0100 Subject: [PATCH 075/207] update echo example --- .../ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino index 9509b5bbdb..6370dbdf6e 100644 --- a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino @@ -13,16 +13,15 @@ #define STAPSK "your-password" #endif -#define STACK_PROTECTOR 128 - -const int port = 23; -int t = 1; // test (1, 2 or 3, see below) +constexpr int port = 23; WiFiServer server(port); WiFiClient client; -constexpr uint32_t breathMs = 100; +constexpr uint32_t stackProtector = 128; +constexpr uint32_t breathMs = 200; esp8266::polledTimeout::oneShotFastMs enoughMs(breathMs); +int t = 1; // test (1, 2 or 3, see below) void setup() { @@ -80,7 +79,7 @@ void loop() { // block by block through a local buffer (2 copies) while (client.available() && client.availableForWrite() && !enoughMs) { size_t maxTo = std::min(client.available(), client.availableForWrite()); - maxTo = std::min(maxTo, (size_t)STACK_PROTECTOR); + maxTo = std::min(maxTo, stackProtector); uint8_t buf[maxTo]; size_t tcp_got = client.read(buf, maxTo); size_t tcp_sent = client.write(buf, tcp_got); @@ -92,7 +91,8 @@ void loop() { else if (t == 3) { // stream to print, possibly with only one copy - client.to(&client); + //client.to(&client); // <=> client.to(&client, -1, -1, client->getTimeout()) + client.to(&client, -1, -1, breathMs); switch (client.getLastTo()) { case Stream::STREAMTO_SUCCESS: break; From 7db503a124787022389d52addb440121cf10bfc0 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Thu, 2 Jan 2020 23:00:02 +0100 Subject: [PATCH 076/207] fix example --- libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino index 6370dbdf6e..e0ed6f6bfa 100644 --- a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino @@ -18,7 +18,7 @@ constexpr int port = 23; WiFiServer server(port); WiFiClient client; -constexpr uint32_t stackProtector = 128; +constexpr size_t stackProtector = 128; constexpr uint32_t breathMs = 200; esp8266::polledTimeout::oneShotFastMs enoughMs(breathMs); int t = 1; // test (1, 2 or 3, see below) From 048aecf2c965a6ce9294b9a621a7255b536aa09b Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 3 Jan 2020 03:05:44 +0100 Subject: [PATCH 077/207] fix so device on host test works again --- libraries/ESP8266WiFi/src/WiFiClient.cpp | 2 +- tests/device/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/ESP8266WiFi/src/WiFiClient.cpp b/libraries/ESP8266WiFi/src/WiFiClient.cpp index 96d4bc4109..97659548cb 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.cpp +++ b/libraries/ESP8266WiFi/src/WiFiClient.cpp @@ -208,7 +208,7 @@ bool WiFiClient::getSync() const int WiFiClient::availableForWrite () { - return connected()? (int)_client->availableForWrite(): 0; + return _client? (int)_client->availableForWrite(): 0; } size_t WiFiClient::write(uint8_t b) diff --git a/tests/device/Makefile b/tests/device/Makefile index c60135c81e..8bc145a532 100644 --- a/tests/device/Makefile +++ b/tests/device/Makefile @@ -64,7 +64,7 @@ $(TEST_LIST): $(SILENT)mkdir -p $(LOCAL_BUILD_DIR) ifeq ("$(MOCK)", "1") @echo Compiling $(notdir $@) - (cd ../host; make ULIBDIRS=../device/libraries/BSTest ../device/$(@:%.ino=%)) + (cd ../host; make D=$(V) ULIBDIRS=../device/libraries/BSTest ../device/$(@:%.ino=%)) $(SILENT)source $(BS_DIR)/virtualenv/bin/activate && \ $(PYTHON) $(BS_DIR)/runner.py \ $(RUNNER_DEBUG_FLAG) \ From 58a625728526481732d779932c21db4034b7f824 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 3 Jan 2020 10:59:20 +0100 Subject: [PATCH 078/207] fix comments and readUntilChar behaviour --- cores/esp8266/Stream.h | 43 ++++++++++++++++++++++++------------- cores/esp8266/StreamDev.cpp | 27 ++++++++++++----------- 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index d15ed07382..2260d9e1fb 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -120,7 +120,7 @@ class Stream: public Print { // => adding int ::readNow(buf, len) for now (following official `int Client::read(buf, len))` // // int ::readNow(buf, len) - // read at most len bytes, returns effectively transfered bytes (can be less than 'len') + // read at most len bytes, returns effectively transferred bytes (can be less than 'len') // with no timeout: immediate return when no more data are available virtual int readNow (char* buffer, size_t len); virtual int readNow (uint8_t* buffer, size_t len) final { return readNow((char*)buffer, len); } @@ -155,38 +155,51 @@ class Stream: public Print { // // Stream::to() uses 1-copy transfers when peekBuffer API is // available, or makes a regular transfer through a local temporary - // stack buffer. + // stack 2-copies buffer. // // By default "source->to(&dest)" transfers everything until - // available (read or write) gets to 0 and a timeout occurs. + // available (read or write) gets to 0, then immediately returns. // // "source->to(&dest, maxLen)" is like above but also returns when - // maxLen bytes are transferred. + // maxLen bytes are transferred, using the default Stream timeout. // - // More generally ::to() will transfer as much as possible until: - // - maxLen (>=0) bytes are transferred (without timeout), - // - or readUntilChar (>=0) is reached (without timeout), - // - or available for read or write gets to zero (after timeout). + // "source->to(&string, -1, '\n')" transfers source to string until + // and including a specific character, with the default Stream + // timeout. // - // Timeout value is by default thisStream->getTimeout() but it can - // be set to "alwaysExpired" (=0) or any value within oneShotMs + // More generally ::to() will transfer as much as possible with the + // following constraints: + // - at most maxLen bytes (-1 by default is no length + // constraint) + // - readUntilChar as last transferred byte (-1 by default is + // no last char contraint) + // - timeoutMs as maximum wait time (only if maxLen>=0 or + // readUntilChar>=0, immediate otherwise) + // + // timeoutMs value is by default "oneShotMs::neverExpires" which is + // internally converted to this->getTimeout() but it can be set to + // "oneShotMs::alwaysExpired" (=0) or any value within oneShotMs // allowed range. // // Return value: - // >0: the number of transfered bytes - // 0: nothing has been transfered, getLastTo() may contain an error reason + // >0: the number of transferred bytes + // 0: nothing has been transferred + // When result is 0 or less than requested maxLen, this->getLastTo() + // may contain an error reason. // // Notes: // - readUntilChar is copied and counted - // - for efficiency: Stream classes should implement peekAPI when possible - // - for efficiency: Stream classes should implement {input,output}TimeoutPossible() + // - for efficiency, Stream classes should implement peekAPI when + // possible + // - for an efficient timeout management, Print/Stream classes + // should implement {output,input}TimeoutPossible() using oneShotMs = esp8266::polledTimeout::oneShotFastMs; virtual size_t to (Print* to, const ssize_t maxLen = -1, int readUntilChar = -1, - oneShotMs::timeType timeout = oneShotMs::neverExpires /* =>getTimeout() */) final; + oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* =>getTimeout() */) final; typedef enum { STREAMTO_SUCCESS = 0, diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index 4c81717325..d19b56afd7 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -8,7 +8,7 @@ using esp8266::polledTimeout::periodicFastMs; size_t Stream::to (Print* to, const ssize_t len, int readUntilChar, - oneShotFastMs::timeType timeout) + oneShotFastMs::timeType timeoutMs) { setWriteError(STREAMTO_SUCCESS); @@ -25,7 +25,7 @@ size_t Stream::to (Print* to, // (also when inputTimeoutPossible() is false) // "neverExpires (default, impossible)" is translated to default timeout - oneShotFastMs timedOut(timeout == oneShotFastMs::neverExpires? getTimeout(): timeout); + oneShotFastMs timedOut(timeoutMs >= oneShotFastMs::neverExpires? getTimeout(): timeoutMs); // yield about every 5ms (XXX SHOULD BE A SYSTEM-WIDE CONSTANT?) periodicFastMs yieldNow(5); @@ -78,12 +78,13 @@ size_t Stream::to (Print* to, } } - if (!maxLen && !w) - // nothing has been transfered and no specific size requested + if (!w && !maxLen && readUntilChar < 0) + // nothing has been transferred and no specific condition is requested break; if (timedOut) - // either (maxLen>0) nothing has been transfered for too long + // either (maxLen>0) nothing has been transferred for too long + // or readUntilChar >= 0 but char is not encountered for too long // or (maxLen=0) too much time has been spent here break; @@ -124,12 +125,13 @@ size_t Stream::to (Print* to, break; } - if (!maxLen && !w) - // nothing has been transfered and no specific size requested + if (!w && !maxLen && readUntilChar < 0) + // nothing has been transferred and no specific condition is requested break; if (timedOut) - // either (maxLen>0) nothing has been transfered for too long + // either (maxLen>0) nothing has been transferred for too long + // or readUntilChar >= 0 but char is not encountered for too long // or (maxLen=0) too much time has been spent here break; @@ -176,12 +178,13 @@ size_t Stream::to (Print* to, timedOut.reset(); } - if (!maxLen && !w) - // nothing has been transfered and no specific size requested + if (!w && !maxLen && readUntilChar < 0) + // nothing has been transferred and no specific condition is requested break; if (timedOut) - // either (maxLen>0) nothing has been transfered for too long + // either (maxLen>0) nothing has been transferred for too long + // or readUntilChar >= 0 but char is not encountered for too long // or (maxLen=0) too much time has been spent here break; @@ -191,7 +194,7 @@ size_t Stream::to (Print* to, if (getWriteError() == STREAMTO_SUCCESS && maxLen > 0) { - if (timeout && timedOut) + if (timeoutMs && timedOut) setWriteError(STREAMTO_TIMED_OUT); else if ((ssize_t)written != len) // This is happening when source cannot timeout (ex: a String) From fd32e65ae5451d9c15ffd83120d6eb09c5d55d90 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 3 Jan 2020 11:52:31 +0100 Subject: [PATCH 079/207] overload with ::to(ref), missing license, minor details --- cores/esp8266/Stream.h | 17 +++++++++++---- cores/esp8266/StreamDev.cpp | 21 +++++++++++++++++++ cores/esp8266/StreamDev.h | 20 ++++++++++++++++++ cores/esp8266/StreamString.h | 2 +- .../src/ESP8266HTTPClient.cpp | 8 +++---- .../examples/WiFiEcho/WiFiEcho.ino | 6 +++--- tests/host/Makefile | 2 ++ tests/host/README.txt | 2 +- 8 files changed, 65 insertions(+), 13 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 2260d9e1fb..c5f1e94233 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -196,10 +196,19 @@ class Stream: public Print { using oneShotMs = esp8266::polledTimeout::oneShotFastMs; - virtual size_t to (Print* to, - const ssize_t maxLen = -1, - int readUntilChar = -1, - oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* =>getTimeout() */) final; + size_t to (Print* to, + const ssize_t maxLen = -1, + int readUntilChar = -1, + oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* =>getTimeout() */); + + size_t to (Print& to, + const ssize_t maxLen = -1, + int readUntilChar = -1, + oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* =>getTimeout() */) + { + return this->to(&to, maxLen, readUntilChar, timeoutMs); + } + typedef enum { STREAMTO_SUCCESS = 0, diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index d19b56afd7..77547d0e3f 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -1,3 +1,24 @@ +/* + StreamDev.cpp - 1-copy transfer function + Copyright (c) 2019 David Gauchard. All right reserved. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + parsing functions based on TextFinder library by Michael Margolis +*/ + #include #include diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 667c6ba4ec..4c43aebce8 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -1,3 +1,23 @@ +/* + StreamDev.h - Stream helpers + Copyright (c) 2019 David Gauchard. All right reserved. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + parsing functions based on TextFinder library by Michael Margolis +*/ #ifndef __STREAMDEV_H #define __STREAMDEV_H diff --git a/cores/esp8266/StreamString.h b/cores/esp8266/StreamString.h index 6aa72a96cb..28fba85e07 100644 --- a/cores/esp8266/StreamString.h +++ b/cores/esp8266/StreamString.h @@ -25,7 +25,7 @@ #include "WString.h" -// StreamString has been integrated insite String +// StreamString has been integrated into String using StreamString = String; diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index c1e9cc5dec..9ea72e9881 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -33,7 +33,7 @@ #include #include -int TO2HTTPC (int streamToError) +static int TO2HTTPC (Stream::toReport_e streamToError) { switch (streamToError) { @@ -41,9 +41,9 @@ int TO2HTTPC (int streamToError) case Stream::STREAMTO_READ_ERROR: return HTTPC_ERROR_NO_STREAM; case Stream::STREAMTO_WRITE_ERROR: return HTTPC_ERROR_STREAM_WRITE; case Stream::STREAMTO_SHORT: return HTTPC_ERROR_STREAM_WRITE; - default: case Stream::STREAMTO_SUCCESS: return 0; } + return 0; // never reached, keep gcc quiet } #if HTTPCLIENT_1_1_COMPATIBLE @@ -1071,8 +1071,8 @@ bool HTTPClient::connect(void) } else { DEBUG_HTTPCLIENT("[HTTP-Client] connect: already connected, try reuse!\n"); } - StreamNull devnull; - _client->to(&devnull, -1, -1, 0); // clear _client's output (all of it, no timeout) + StreamNull devNull; // should we have a global one? + _client->to(devNull); // clear _client's output (all of it, no timeout) return true; } diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino index e0ed6f6bfa..2b12d4bbaa 100644 --- a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino @@ -1,5 +1,5 @@ /* - WiFiEcho - Echo tester + WiFiEcho - Echo server released to public domain */ @@ -91,8 +91,8 @@ void loop() { else if (t == 3) { // stream to print, possibly with only one copy - //client.to(&client); // <=> client.to(&client, -1, -1, client->getTimeout()) - client.to(&client, -1, -1, breathMs); + //client.to(client); // <=> client.to(&client, -1, -1, client->getTimeout()) + client.to(client, -1, -1, breathMs); switch (client.getLastTo()) { case Stream::STREAMTO_SUCCESS: break; diff --git a/tests/host/Makefile b/tests/host/Makefile index 78c3acb007..33be11084d 100644 --- a/tests/host/Makefile +++ b/tests/host/Makefile @@ -6,6 +6,7 @@ LIBRARIES_PATH := ../../libraries FORCE32 ?= 1 OPTZ ?= -Os V ?= 0 +R ?= noexec MAKEFILE = $(word 1, $(MAKEFILE_LIST)) @@ -330,6 +331,7 @@ ifeq ($(INO),) %: %.ino @# recursive 'make' with paths $(MAKE) -f $(MAKEFILE) MKFLAGS=-Wextra INODIR=$(dir $@) INO=$(notdir $@) $(BINDIR)/$(notdir $@)/$(notdir $@) + test "$(R)" = noexec || $(BINDIR)/$(notdir $@)/$(notdir $@) $(R) @# see below the new build rule with fixed output path outside from core location ##################### diff --git a/tests/host/README.txt b/tests/host/README.txt index 4e947a954e..f9765987be 100644 --- a/tests/host/README.txt +++ b/tests/host/README.txt @@ -45,7 +45,7 @@ Optional 'V=1' enables makefile verbosity Optional 'D=1' enables core debug (same as IDE's tools menu) Optional 'OPTZ=-O2' will update gcc -O option (default is -Os, D=1 implies -O0) Optional 'FORCE32=0' will use native/default gcc (default is FORCE32=1 unless gcc-multilib is not detected) - +Optional 'R=""' (ex: R="-b -v") runs the executable with given options after build Non exhaustive list of working examples: make D=1 ../../libraries/ESP8266WiFi/examples/udp/udp From e480ea524d883bea206f4bf378ea805e9098f779 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 3 Jan 2020 14:34:14 +0100 Subject: [PATCH 080/207] fix debug print --- libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index 0f3c6c9589..9943466f9b 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -440,7 +440,7 @@ void ESP8266WebServerTemplate::send(int code, const char* content_ty (void)sent; #ifdef DEBUG_ESP_HTTP_SERVER if (sent != header.length()) - DEBUG_OUTPUT.println("HTTPServer: error: sent %zd on %zd bytes\n", sent, header.length()); + DEBUG_OUTPUT.printf("HTTPServer: error: sent %zd on %zd bytes\n", sent, header.length()); #endif if(content.length()) From 15ab6cfef551138ed3873de8fdb1a28a99af12b4 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 3 Jan 2020 15:01:15 +0100 Subject: [PATCH 081/207] fix host test --- cores/esp8266/WString.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cores/esp8266/WString.h b/cores/esp8266/WString.h index caf1d4b3e4..d13073930d 100644 --- a/cores/esp8266/WString.h +++ b/cores/esp8266/WString.h @@ -70,7 +70,7 @@ class String: public Stream { explicit String(unsigned long, unsigned char base = 10); explicit String(float, unsigned char decimalPlaces = 2); explicit String(double, unsigned char decimalPlaces = 2); - ~String(void); + virtual ~String(void); // memory management // return true on success, false on failure (in which case, the string From 203674923d0857e505dff0183e322fe540dbecfa Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Wed, 22 Jan 2020 12:58:39 +0100 Subject: [PATCH 082/207] + Stream serializer --- cores/esp8266/FS.cpp | 2 +- cores/esp8266/FS.h | 2 +- cores/esp8266/Stream.h | 7 + cores/esp8266/StreamDev.h | 146 +++++++++++++++++- cores/esp8266/WString.h | 2 + .../ESP8266WebServer/src/ESP8266WebServer.h | 24 +++ 6 files changed, 180 insertions(+), 3 deletions(-) diff --git a/cores/esp8266/FS.cpp b/cores/esp8266/FS.cpp index fa841c64fa..03cca5aac6 100644 --- a/cores/esp8266/FS.cpp +++ b/cores/esp8266/FS.cpp @@ -96,7 +96,7 @@ size_t File::position() const { return _p->position(); } -size_t File::size() const { +ssize_t File::size() { if (!_p) return 0; diff --git a/cores/esp8266/FS.h b/cores/esp8266/FS.h index 4985793dd1..2ac4354cd3 100644 --- a/cores/esp8266/FS.h +++ b/cores/esp8266/FS.h @@ -72,7 +72,7 @@ class File : public Stream return seek(pos, SeekSet); } size_t position() const; - size_t size() const; + virtual ssize_t size() override; void close(); operator bool() const; const char* name() const; diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index c5f1e94233..67281959fa 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -220,6 +220,13 @@ class Stream: public Print { toReport_e getLastTo () /*const*/ { return (toReport_e)getWriteError(); } + //////////////////// + // size of input + // -1 by default is unknown + // may be used by http streamer (using a SerialStream as a file) + + virtual ssize_t size () { return -1; } + //////////////////// end of extensions protected: diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 4c43aebce8..e15a6c9dc7 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -35,7 +35,7 @@ class StreamNull: public Stream // Print virtual size_t write(uint8_t) override { return 1; } - virtual size_t write(const uint8_t* buffer, size_t size) override { (void)buffer; return size; } + virtual size_t write(const uint8_t* buffer, size_t size) override { (void)buffer; (void)size; return size; } virtual int availableForWrite() override { return 32767; } // Stream @@ -46,6 +46,8 @@ class StreamNull: public Stream virtual int readNow(char* buffer, size_t len) override { (void)buffer; (void)len; return 0; } virtual bool outputTimeoutPossible () override { return false; } virtual bool inputTimeoutPossible () override { return false; } + + virtual ssize_t size () override { return 0; } }; /////////////////////////////////////////////// @@ -66,6 +68,8 @@ class StreamZero: public StreamNull virtual int peek() override { return _x; } virtual size_t readBytes(char* buffer, size_t len) override { memset(buffer, _x, len); return len; } virtual int readNow(char* buffer, size_t len) override { memset(buffer, _x, len); return len; } + + virtual ssize_t size () override { return 32767; } }; /////////////////////////////////////////////// @@ -85,6 +89,7 @@ class StreamPtr: public StreamNull StreamPtr (const char* buffer, size_t size, bool in_flash = false): _buffer(buffer), _size(size), _in_flash(in_flash) { } StreamPtr (const uint8_t* buffer, size_t size, bool in_flash = false): _buffer((const char*)buffer), _size(size), _in_flash(in_flash) { } StreamPtr (const __FlashStringHelper* buffer, size_t size): _buffer(reinterpret_cast(buffer)), _size(size), _in_flash(true) { } + StreamPtr (const __FlashStringHelper* buffer): _buffer(reinterpret_cast(buffer)), _size(strlen_P(_buffer)), _in_flash(true) { } void peekPointerReset (int pointer = 0) { _peekPointer = pointer; } @@ -106,6 +111,8 @@ class StreamPtr: public StreamNull } virtual int readNow(char* buffer, size_t len) override { return readBytes(buffer, len); } + virtual ssize_t size () override { return _size; } + // peekBuffer virtual bool peekBufferAPI () const override { return !_in_flash; } virtual size_t availableForPeek () override { return _peekPointer < _size? _size - _peekPointer: 0; } @@ -113,5 +120,142 @@ class StreamPtr: public StreamNull virtual void peekConsume (size_t consume) { _peekPointer += consume; } }; +/////////////////////////////////////////////// +// serialization: +// combine multiple input Stream into one +// useful when sending HTML content (strings, files. ...) + +template +class SerialStreamArray: public Stream +{ +protected: + + Stream* m_segments [MaxSegments]; + int m_size = 0; + int m_current = 0; + +public: + + // not writable + virtual size_t write(uint8_t) override { return 0; } + virtual size_t write(const uint8_t* buffer, size_t size) override { (void)buffer; (void)size; return 0; } + virtual int availableForWrite() override { return 0; } + virtual bool outputTimeoutPossible () override { return false; } + + // not offering peekBuffer because one streamed element can be not compatible + // (Stream:: is by default not peekBuffer-enabled) + // input timeout may be possible: + virtual bool inputTimeoutPossible () override { return true; } + + SerialStreamArray () {} + + bool add (Stream* s) + { + if (m_size >= MaxSegments) + return false; + + m_segments[m_size++] = s; + return true; + } + + // Stream + virtual int available() override + { + while (true) + { + if (m_current >= m_size) + // end of all + return 0; + + int ret = m_segments[m_current]->available(); + if (ret > 0) + return ret; + + m_current++; + } + } + + virtual int read() override + { + while (true) + { + if (m_current >= m_size) + // end of all + return 0; + + int ret = m_segments[m_current]->read(); + if (ret > 0) + return ret; + + m_current++; + } + } + + virtual int peek() override + { + while (true) + { + if (m_current >= m_size) + // end of all + return 0; + + int ret = m_segments[m_current]->peek(); + if (ret > 0) + return ret; + + m_current++; + } + } + + virtual size_t readBytes(char* buffer, size_t len) override + { + while (true) + { + if (m_current >= m_size) + // end of all + return 0; + + size_t ret = m_segments[m_current]->readBytes(buffer, len); + if (ret > 0) + return ret; + + m_current++; + } + } + + virtual int readNow(char* buffer, size_t len) override + { + while (true) + { + if (m_current >= m_size) + // end of all + return 0; + + int ret = m_segments[m_current]->readNow(buffer, len); + if (ret > 0) + return ret; + + m_current++; + } + } + + virtual ssize_t size () override + { + ssize_t ret = 0; + for (int i = 0; i < m_size; i++) + { + ssize_t s = m_segments[i]->size(); + if (s == -1) + return -1; + ret += s; + } + return ret; + } +}; + +class SerialStream: public SerialStreamArray<> +{ +}; + /////////////////////////////////////////////// #endif // __STREAMDEV_H diff --git a/cores/esp8266/WString.h b/cores/esp8266/WString.h index d13073930d..cabcc5e0c1 100644 --- a/cores/esp8266/WString.h +++ b/cores/esp8266/WString.h @@ -364,6 +364,8 @@ class String: public Stream { return read(buffer, len); } + virtual ssize_t size () override { return length(); } + //// Print: virtual bool outputTimeoutPossible () override { return false; } diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer.h b/libraries/ESP8266WebServer/src/ESP8266WebServer.h index 2366ea93af..5a94e00c4d 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer.h @@ -169,6 +169,30 @@ class ESP8266WebServerTemplate return contentLength; } + // Implement GET and HEAD requests for stream + // Stream body on HTTP_GET but not on HTTP_HEAD requests. + template + size_t stream(T &aStream, const String& contentType, HTTPMethod requestMethod, ssize_t size) { + setContentLength(size); + send(200, contentType, emptyString); + if (requestMethod == HTTP_GET) + size = aStream.to(_currentClient, size); + return size; + } + + // Implement GET and HEAD requests for stream + // Stream body on HTTP_GET but not on HTTP_HEAD requests. + template + size_t stream(T& aStream, const String& contentType, HTTPMethod requestMethod = HTTP_GET) { + ssize_t size = aStream.size(); + if (size < 0) + { + send(500, F("text/html"), F("input stream: undetermined size")); + return 0; + } + return stream(aStream, contentType, requestMethod, size); + } + static String responseCodeToString(const int code); protected: From 5e7c44888409744b05a5900a395d110177b2ba4e Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Sat, 25 Jan 2020 00:52:49 +0100 Subject: [PATCH 083/207] wip --- cores/esp8266/StreamDev.h | 10 +++++++++- libraries/ESP8266WebServer/src/ESP8266WebServer.h | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index e15a6c9dc7..3380a36025 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -149,10 +149,18 @@ class SerialStreamArray: public Stream SerialStreamArray () {} - bool add (Stream* s) + bool addref (Stream& s) + { + return addref(&s); + } + + bool addref (Stream* s) { if (m_size >= MaxSegments) + { + //XXXDEBUG: print exhausted message return false; + } m_segments[m_size++] = s; return true; diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer.h b/libraries/ESP8266WebServer/src/ESP8266WebServer.h index 5a94e00c4d..c8136df48c 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer.h @@ -151,14 +151,14 @@ class ESP8266WebServerTemplate static String urlDecode(const String& text); - // Handle a GET request by sending a response header and stream file content to response body + // Handle a GET request by sending a response header and stream file content to response body template size_t streamFile(T &file, const String& contentType) { return streamFile(file, contentType, HTTP_GET); } // Implement GET and HEAD requests for files. - // Stream body on HTTP_GET but not on HTTP_HEAD requests. + // Stream body on HTTP_GET but not on HTTP_HEAD requests. template size_t streamFile(T &file, const String& contentType, HTTPMethod requestMethod) { size_t contentLength = 0; From afc24940de7dec1fdfe36d8e0a0d5c79efe44b11 Mon Sep 17 00:00:00 2001 From: Szabolcs Szekelyi Date: Sat, 25 Jan 2020 22:27:28 -0500 Subject: [PATCH 084/207] Fix/enable UDP packet reassembly UdpContext didn't care about pbuf chaining when receiving datagrams, leading to fragments delivered to the application as individual packets. --- .../ESP8266WiFi/src/include/UdpContext.h | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/libraries/ESP8266WiFi/src/include/UdpContext.h b/libraries/ESP8266WiFi/src/include/UdpContext.h index 31aa5619d1..3bcb8e97c0 100644 --- a/libraries/ESP8266WiFi/src/include/UdpContext.h +++ b/libraries/ESP8266WiFi/src/include/UdpContext.h @@ -187,7 +187,7 @@ class UdpContext if (!_rx_buf) return 0; - return _rx_buf->len - _rx_buf_offset; + return _rx_buf->tot_len - _rx_buf_offset; } size_t tell() const @@ -202,7 +202,7 @@ class UdpContext } bool isValidOffset(const size_t pos) const { - return (pos <= _rx_buf->len); + return (pos <= _rx_buf->tot_len); } CONST IPAddress& getRemoteAddress() CONST @@ -238,6 +238,8 @@ class UdpContext } auto deleteme = _rx_buf; + + while(_rx_buf->len != _rx_buf->tot_len) _rx_buf = _rx_buf->next; _rx_buf = _rx_buf->next; if (_rx_buf) @@ -274,10 +276,10 @@ class UdpContext int read() { - if (!_rx_buf || _rx_buf_offset >= _rx_buf->len) + if (!_rx_buf || _rx_buf_offset >= _rx_buf->tot_len) return -1; - char c = reinterpret_cast(_rx_buf->payload)[_rx_buf_offset]; + char c = pbuf_get_at(_rx_buf, _rx_buf_offset); _consume(1); return c; } @@ -287,11 +289,15 @@ class UdpContext if (!_rx_buf) return 0; - size_t max_size = _rx_buf->len - _rx_buf_offset; + size_t max_size = _rx_buf->tot_len - _rx_buf_offset; size = (size < max_size) ? size : max_size; - DEBUGV(":urd %d, %d, %d\r\n", size, _rx_buf->len, _rx_buf_offset); + DEBUGV(":urd %d, %d, %d\r\n", size, _rx_buf->tot_len, _rx_buf_offset); + + void* buf = pbuf_get_contiguous(_rx_buf, dst, size, size, _rx_buf_offset); + if(!buf) return 0; + + if(buf != dst) memcpy(dst, buf, size); - memcpy(dst, reinterpret_cast(_rx_buf->payload) + _rx_buf_offset, size); _consume(size); return size; @@ -299,10 +305,10 @@ class UdpContext int peek() const { - if (!_rx_buf || _rx_buf_offset == _rx_buf->len) + if (!_rx_buf || _rx_buf_offset == _rx_buf->tot_len) return -1; - return reinterpret_cast(_rx_buf->payload)[_rx_buf_offset]; + return pbuf_get_at(_rx_buf, _rx_buf_offset); } void flush() @@ -311,7 +317,7 @@ class UdpContext if (!_rx_buf) return; - _consume(_rx_buf->len - _rx_buf_offset); + _consume(_rx_buf->tot_len - _rx_buf_offset); } size_t append(const char* data, size_t size) @@ -432,8 +438,8 @@ class UdpContext void _consume(size_t size) { _rx_buf_offset += size; - if (_rx_buf_offset > _rx_buf->len) { - _rx_buf_offset = _rx_buf->len; + if (_rx_buf_offset > _rx_buf->tot_len) { + _rx_buf_offset = _rx_buf->tot_len; } } From 4196458da39f3b2262d0106c5af01b8209b0c2a2 Mon Sep 17 00:00:00 2001 From: Szabolcs Szekelyi Date: Sun, 26 Jan 2020 15:40:57 -0500 Subject: [PATCH 085/207] Provide pbuf_get_contiguous for backwards compatibility with LwIP 1.4 Implementation copied verbatim from LwIP 2.1.2 --- .../ESP8266WiFi/src/include/UdpContext.h | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/libraries/ESP8266WiFi/src/include/UdpContext.h b/libraries/ESP8266WiFi/src/include/UdpContext.h index 3bcb8e97c0..5aeba893a2 100644 --- a/libraries/ESP8266WiFi/src/include/UdpContext.h +++ b/libraries/ESP8266WiFi/src/include/UdpContext.h @@ -528,6 +528,90 @@ class UdpContext reinterpret_cast(arg)->_recv(upcb, p, srcaddr, srcport); } +#if LWIP_VERSION_MAJOR == 1 + /* + * Code in this conditional block is copied/backported verbatim from + * LwIP 2.1.2 to provide pbuf_get_contiguous. + */ + + static const struct pbuf * + pbuf_skip_const(const struct pbuf *in, u16_t in_offset, u16_t *out_offset) + { + u16_t offset_left = in_offset; + const struct pbuf *q = in; + + /* get the correct pbuf */ + while ((q != NULL) && (q->len <= offset_left)) { + offset_left = (u16_t)(offset_left - q->len); + q = q->next; + } + if (out_offset != NULL) { + *out_offset = offset_left; + } + return q; + } + + u16_t + pbuf_copy_partial(const struct pbuf *buf, void *dataptr, u16_t len, u16_t offset) + { + const struct pbuf *p; + u16_t left = 0; + u16_t buf_copy_len; + u16_t copied_total = 0; + + LWIP_ERROR("pbuf_copy_partial: invalid buf", (buf != NULL), return 0;); + LWIP_ERROR("pbuf_copy_partial: invalid dataptr", (dataptr != NULL), return 0;); + + /* Note some systems use byte copy if dataptr or one of the pbuf payload pointers are unaligned. */ + for (p = buf; len != 0 && p != NULL; p = p->next) { + if ((offset != 0) && (offset >= p->len)) { + /* don't copy from this buffer -> on to the next */ + offset = (u16_t)(offset - p->len); + } else { + /* copy from this buffer. maybe only partially. */ + buf_copy_len = (u16_t)(p->len - offset); + if (buf_copy_len > len) { + buf_copy_len = len; + } + /* copy the necessary parts of the buffer */ + MEMCPY(&((char *)dataptr)[left], &((char *)p->payload)[offset], buf_copy_len); + copied_total = (u16_t)(copied_total + buf_copy_len); + left = (u16_t)(left + buf_copy_len); + len = (u16_t)(len - buf_copy_len); + offset = 0; + } + } + return copied_total; + } + + void * + pbuf_get_contiguous(const struct pbuf *p, void *buffer, size_t bufsize, u16_t len, u16_t offset) + { + const struct pbuf *q; + u16_t out_offset; + + LWIP_ERROR("pbuf_get_contiguous: invalid buf", (p != NULL), return NULL;); + LWIP_ERROR("pbuf_get_contiguous: invalid dataptr", (buffer != NULL), return NULL;); + LWIP_ERROR("pbuf_get_contiguous: invalid dataptr", (bufsize >= len), return NULL;); + + q = pbuf_skip_const(p, offset, &out_offset); + if (q != NULL) { + if (q->len >= (out_offset + len)) { + /* all data in this pbuf, return zero-copy */ + return (u8_t *)q->payload + out_offset; + } + /* need to copy */ + if (pbuf_copy_partial(q, buffer, len, out_offset) != len) { + /* copying failed: pbuf is too short */ + return NULL; + } + return buffer; + } + /* pbuf is too short (offset does not fit in) */ + return NULL; + } +#endif + private: udp_pcb* _pcb; pbuf* _rx_buf; From 34a6d006010daedc7112479731348fdb1dc02f71 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 27 Jan 2020 23:28:40 +0100 Subject: [PATCH 086/207] httpserver: sendContent(Stream) (sub-streams as http-chunks) --- .../src/ESP8266WebServer-impl.h | 25 +++++++++++++++++++ .../ESP8266WebServer/src/ESP8266WebServer.h | 2 ++ 2 files changed, 27 insertions(+) diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index 9943466f9b..ac40ab4611 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -487,6 +487,31 @@ void ESP8266WebServerTemplate::send(int code, const String& content_ send(code, (const char*)content_type.c_str(), content); } +template +void ESP8266WebServerTemplate::sendContent(Stream& content) { + sendContent(&content); +} + +template +void ESP8266WebServerTemplate::sendContent(Stream* content) { + if (_currentMethod == HTTP_HEAD) return; + const char * footer = "\r\n"; + size_t len = content->size(); + if(_chunked) { + char chunkSize[11]; + sprintf(chunkSize, "%zx\r\n", len); + _currentClient.write((const uint8_t *)chunkSize, strlen(chunkSize)); + } + size_t sent = content->to(_currentClient); + (void)sent; /// if (sent != len) print-error-on-console-and-return-false + if(_chunked){ + _currentClient.write((const uint8_t *)footer, 2); + if (len == 0) { + _chunked = false; + } + } +} + template void ESP8266WebServerTemplate::sendContent(const String& content) { if (_currentMethod == HTTP_HEAD) return; diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer.h b/libraries/ESP8266WebServer/src/ESP8266WebServer.h index c8136df48c..d67efe3586 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer.h @@ -141,6 +141,8 @@ class ESP8266WebServerTemplate void setContentLength(const size_t contentLength); void sendHeader(const String& name, const String& value, bool first = false); + void sendContent(Stream& content); + void sendContent(Stream* content); void sendContent(const String& content); void sendContent_P(PGM_P content); void sendContent_P(PGM_P content, size_t size); From 40713986ab935ea1330d7bc607ede627160744bc Mon Sep 17 00:00:00 2001 From: david gauchard Date: Thu, 30 Jan 2020 01:59:07 +0100 Subject: [PATCH 087/207] sdfat incompatible with `ssize_t size()`, rename to streamSize --- cores/esp8266/FS.cpp | 2 +- cores/esp8266/FS.h | 3 ++- cores/esp8266/Stream.h | 2 +- cores/esp8266/StreamDev.h | 8 ++++---- cores/esp8266/WString.h | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/cores/esp8266/FS.cpp b/cores/esp8266/FS.cpp index 03cca5aac6..fa841c64fa 100644 --- a/cores/esp8266/FS.cpp +++ b/cores/esp8266/FS.cpp @@ -96,7 +96,7 @@ size_t File::position() const { return _p->position(); } -ssize_t File::size() { +size_t File::size() const { if (!_p) return 0; diff --git a/cores/esp8266/FS.h b/cores/esp8266/FS.h index 2ac4354cd3..93b5158bfa 100644 --- a/cores/esp8266/FS.h +++ b/cores/esp8266/FS.h @@ -72,7 +72,8 @@ class File : public Stream return seek(pos, SeekSet); } size_t position() const; - virtual ssize_t size() override; + size_t size() const; + virtual ssize_t streamSize() override { return size(); } void close(); operator bool() const; const char* name() const; diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 67281959fa..f96fa62c29 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -225,7 +225,7 @@ class Stream: public Print { // -1 by default is unknown // may be used by http streamer (using a SerialStream as a file) - virtual ssize_t size () { return -1; } + virtual ssize_t streamSize () { return -1; } //////////////////// end of extensions diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 3380a36025..6ec109f4da 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -47,7 +47,7 @@ class StreamNull: public Stream virtual bool outputTimeoutPossible () override { return false; } virtual bool inputTimeoutPossible () override { return false; } - virtual ssize_t size () override { return 0; } + virtual ssize_t streamSize () override { return 0; } }; /////////////////////////////////////////////// @@ -69,7 +69,7 @@ class StreamZero: public StreamNull virtual size_t readBytes(char* buffer, size_t len) override { memset(buffer, _x, len); return len; } virtual int readNow(char* buffer, size_t len) override { memset(buffer, _x, len); return len; } - virtual ssize_t size () override { return 32767; } + virtual ssize_t streamSize () override { return 32767; } }; /////////////////////////////////////////////// @@ -111,7 +111,7 @@ class StreamPtr: public StreamNull } virtual int readNow(char* buffer, size_t len) override { return readBytes(buffer, len); } - virtual ssize_t size () override { return _size; } + virtual ssize_t streamSize () override { return _size; } // peekBuffer virtual bool peekBufferAPI () const override { return !_in_flash; } @@ -247,7 +247,7 @@ class SerialStreamArray: public Stream } } - virtual ssize_t size () override + virtual ssize_t streamSize () override { ssize_t ret = 0; for (int i = 0; i < m_size; i++) diff --git a/cores/esp8266/WString.h b/cores/esp8266/WString.h index cabcc5e0c1..49fbc2004e 100644 --- a/cores/esp8266/WString.h +++ b/cores/esp8266/WString.h @@ -364,7 +364,7 @@ class String: public Stream { return read(buffer, len); } - virtual ssize_t size () override { return length(); } + virtual ssize_t streamSize () override { return length(); } //// Print: From 820acf001b75d9a004d386f95d71a5edd5246d8d Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 31 Jan 2020 21:21:28 +0100 Subject: [PATCH 088/207] fix stream api use --- libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index ac40ab4611..5472b1b3dc 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -496,7 +496,7 @@ template void ESP8266WebServerTemplate::sendContent(Stream* content) { if (_currentMethod == HTTP_HEAD) return; const char * footer = "\r\n"; - size_t len = content->size(); + size_t len = content->streamSize(); if(_chunked) { char chunkSize[11]; sprintf(chunkSize, "%zx\r\n", len); From 99740f86d7ffc29cba0b7c772a3664bfdc5819b1 Mon Sep 17 00:00:00 2001 From: David Gauchard Date: Wed, 4 Mar 2020 21:20:51 +0100 Subject: [PATCH 089/207] expose hidden WebServer's chunked API --- .../examples/FSBrowser/FSBrowser.ino | 22 ++++++++++++++----- .../ESP8266WebServer/src/ESP8266WebServer.h | 20 +++++++++++++++-- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/libraries/ESP8266WebServer/examples/FSBrowser/FSBrowser.ino b/libraries/ESP8266WebServer/examples/FSBrowser/FSBrowser.ino index fb5847d4b0..645d46b137 100644 --- a/libraries/ESP8266WebServer/examples/FSBrowser/FSBrowser.ino +++ b/libraries/ESP8266WebServer/examples/FSBrowser/FSBrowser.ino @@ -186,12 +186,22 @@ void handleFileList() { Dir dir = filesystem->openDir(path); path.clear(); - String output = "["; + // use chunked response to avoid building a huge temporary string + server.chunkedResponseModeStart(200, "text/json"); + bool first = true; + // use the same string for every line + String output; + output.reserve(64); while (dir.next()) { + + if (output.length()) + // send string from previous iteration + // as an HTTP chunk + server.sendContent(output); + File entry = dir.openFile("r"); - if (output != "[") { - output += ','; - } + output = first? '[': ','; + first = false; bool isDir = false; output += "{\"type\":\""; output += (isDir) ? "dir" : "file"; @@ -205,8 +215,10 @@ void handleFileList() { entry.close(); } + // send last string output += "]"; - server.send(200, "text/json", output); + server.sendContent(output); + server.chunkedResponseFinalize(); } void setup(void) { diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer.h b/libraries/ESP8266WebServer/src/ESP8266WebServer.h index eefcba2ba3..b3e88a5029 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer.h @@ -149,18 +149,34 @@ class ESP8266WebServerTemplate void sendContent(const char *content) { sendContent_P(content); } void sendContent(const char *content, size_t size) { sendContent_P(content, size); } + bool chunkedResponseModeStart (int code, const char* content_type) { + setContentLength(CONTENT_LENGTH_UNKNOWN); + send(code, content_type, emptyString); + } + bool chunkedResponseModeStart (int code, const String& content_type) { + setContentLength(CONTENT_LENGTH_UNKNOWN); + send(code, content_type, emptyString); + } + bool chunkedResponseModeStart (int code, PGM_P content_type) { + setContentLength(CONTENT_LENGTH_UNKNOWN); + send(code, content_type, emptyString); + } + void chunkedResponseFinalize () { + sendContent(emptyString); + } + static String credentialHash(const String& username, const String& realm, const String& password); static String urlDecode(const String& text); - // Handle a GET request by sending a response header and stream file content to response body + // Handle a GET request by sending a response header and stream file content to response body template size_t streamFile(T &file, const String& contentType) { return streamFile(file, contentType, HTTP_GET); } // Implement GET and HEAD requests for files. - // Stream body on HTTP_GET but not on HTTP_HEAD requests. + // Stream body on HTTP_GET but not on HTTP_HEAD requests. template size_t streamFile(T &file, const String& contentType, HTTPMethod requestMethod) { size_t contentLength = 0; From fee13076e59711ec8c75b66cd7a45c2455d2d8d6 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 4 Mar 2020 22:17:42 +0100 Subject: [PATCH 090/207] expose hidden WebServer's chunked API --- .../examples/FSBrowser/FSBrowser.ino | 8 ++++++-- .../ESP8266WebServer/src/ESP8266WebServer.h | 18 ++++++++++-------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/libraries/ESP8266WebServer/examples/FSBrowser/FSBrowser.ino b/libraries/ESP8266WebServer/examples/FSBrowser/FSBrowser.ino index 645d46b137..5503cb001f 100644 --- a/libraries/ESP8266WebServer/examples/FSBrowser/FSBrowser.ino +++ b/libraries/ESP8266WebServer/examples/FSBrowser/FSBrowser.ino @@ -186,8 +186,12 @@ void handleFileList() { Dir dir = filesystem->openDir(path); path.clear(); - // use chunked response to avoid building a huge temporary string - server.chunkedResponseModeStart(200, "text/json"); + // use HTTP/1.1 Chunked response to avoid building a huge temporary string + if (!server.chunkedResponseModeStart(200, "text/json")) { + server.send(505, FPSTR("text/html"), FPSTR("HTTP1.1 required")); + return; + } + bool first = true; // use the same string for every line String output; diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer.h b/libraries/ESP8266WebServer/src/ESP8266WebServer.h index b3e88a5029..9e999acadc 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer.h @@ -149,17 +149,19 @@ class ESP8266WebServerTemplate void sendContent(const char *content) { sendContent_P(content); } void sendContent(const char *content, size_t size) { sendContent_P(content, size); } - bool chunkedResponseModeStart (int code, const char* content_type) { + bool chunkedResponseModeStart_P (int code, PGM_P content_type) { + if (_currentVersion == 0) + // no chunk mode in HTTP/1.0 + return false; setContentLength(CONTENT_LENGTH_UNKNOWN); - send(code, content_type, emptyString); + send_P(code, content_type, ""); + return true; } - bool chunkedResponseModeStart (int code, const String& content_type) { - setContentLength(CONTENT_LENGTH_UNKNOWN); - send(code, content_type, emptyString); + bool chunkedResponseModeStart (int code, const char* content_type) { + return chunkedResponseModeStart_P(code, content_type); } - bool chunkedResponseModeStart (int code, PGM_P content_type) { - setContentLength(CONTENT_LENGTH_UNKNOWN); - send(code, content_type, emptyString); + bool chunkedResponseModeStart (int code, const String& content_type) { + return chunkedResponseModeStart_P(code, content_type.c_str()); } void chunkedResponseFinalize () { sendContent(emptyString); From cc3e01cc180bb86687089af50b449671f34581d6 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 4 Mar 2020 22:29:41 +0100 Subject: [PATCH 091/207] expose hidden WebServer's chunked API --- .../examples/FSBrowser/FSBrowser.ino | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/libraries/ESP8266WebServer/examples/FSBrowser/FSBrowser.ino b/libraries/ESP8266WebServer/examples/FSBrowser/FSBrowser.ino index 5503cb001f..bd25fad582 100644 --- a/libraries/ESP8266WebServer/examples/FSBrowser/FSBrowser.ino +++ b/libraries/ESP8266WebServer/examples/FSBrowser/FSBrowser.ino @@ -192,20 +192,21 @@ void handleFileList() { return; } - bool first = true; // use the same string for every line String output; output.reserve(64); while (dir.next()) { - if (output.length()) - // send string from previous iteration - // as an HTTP chunk - server.sendContent(output); + if (output.length()) { + // send string from previous iteration + // as an HTTP chunk + server.sendContent(output); + output = ','; + } else { + output = '['; + } File entry = dir.openFile("r"); - output = first? '[': ','; - first = false; bool isDir = false; output += "{\"type\":\""; output += (isDir) ? "dir" : "file"; From 9387a256d946d014576df0610dc93ef618e7a954 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 17 Apr 2020 23:59:21 +0200 Subject: [PATCH 092/207] move ESP.getCpuFreqMHz to features --- cores/esp8266/Esp.cpp | 7 ------- cores/esp8266/Esp.h | 7 +++++-- cores/esp8266/PolledTimeout.h | 2 +- cores/esp8266/core_esp8266_features.h | 18 ++++++++++++++++++ 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/cores/esp8266/Esp.cpp b/cores/esp8266/Esp.cpp index db29ec599c..e15fe8a3c5 100644 --- a/cores/esp8266/Esp.cpp +++ b/cores/esp8266/Esp.cpp @@ -264,13 +264,6 @@ uint8_t EspClass::getBootMode(void) return system_get_boot_mode(); } -#ifndef F_CPU -uint8_t EspClass::getCpuFreqMHz(void) -{ - return system_get_cpu_freq(); -} -#endif - uint32_t EspClass::getFlashChipId(void) { static uint32_t flash_chip_id = 0; diff --git a/cores/esp8266/Esp.h b/cores/esp8266/Esp.h index 01327747ef..93cd937ab5 100644 --- a/cores/esp8266/Esp.h +++ b/cores/esp8266/Esp.h @@ -160,10 +160,13 @@ class EspClass { #if defined(F_CPU) || defined(CORE_MOCK) constexpr uint8_t getCpuFreqMHz() const { - return clockCyclesPerMicrosecond(); + return esp_get_cpu_freq_mhz(); } #else - uint8_t getCpuFreqMHz(); + uint8_t getCpuFreqMHz() const + { + return esp_get_cpu_freq_mhz(); + } #endif uint32_t getFlashChipId(); diff --git a/cores/esp8266/PolledTimeout.h b/cores/esp8266/PolledTimeout.h index 919a110c4d..8256b68140 100644 --- a/cores/esp8266/PolledTimeout.h +++ b/cores/esp8266/PolledTimeout.h @@ -77,7 +77,7 @@ struct TimeSourceCycles using timeType = decltype(esp_get_cycle_count()); static timeType time() {return esp_get_cycle_count();} - static constexpr timeType ticksPerSecond = ESP.getCpuFreqMHz() * 1000000UL; // 80'000'000 or 160'000'000 Hz + static constexpr timeType ticksPerSecond = esp_get_cpu_freq_mhz() * 1000000UL; // 80'000'000 or 160'000'000 Hz static constexpr timeType ticksPerSecondMax = 160000000; // 160MHz }; diff --git a/cores/esp8266/core_esp8266_features.h b/cores/esp8266/core_esp8266_features.h index 1154ea1f80..e5d572d318 100644 --- a/cores/esp8266/core_esp8266_features.h +++ b/cores/esp8266/core_esp8266_features.h @@ -117,6 +117,24 @@ uint64_t micros64(void); void delay(unsigned long); void delayMicroseconds(unsigned int us); +#if defined(F_CPU) || defined(CORE_MOCK) +#ifdef __cplusplus +constexpr +#else +inline +#endif +int esp_get_cpu_freq_mhz() +{ + return F_CPU / 1000000L; +} +#else +inline int esp_get_cpu_freq_mhz() +{ + return system_get_cpu_freq(); +} +#endif + + #ifdef __cplusplus } #endif From 0ee1c969b1e65482f6373a8d6df6391df5bfc9d2 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 26 May 2020 18:12:49 +0200 Subject: [PATCH 093/207] fix last merge --- .../src/ESP8266HTTPClient.cpp | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index abcef67ed3..220598e1c6 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -1105,22 +1105,10 @@ bool HTTPClient::hasHeader(const char* name) */ bool HTTPClient::connect(void) { -<<<<<<< HEAD - if(connected()) { - if(_reuse) { - DEBUG_HTTPCLIENT("[HTTP-Client] connect: already connected, reusing connection\n"); - } else { - DEBUG_HTTPCLIENT("[HTTP-Client] connect: already connected, try reuse!\n"); - } - StreamNull devNull; // should we have a global one? - _client->to(devNull); // clear _client's output (all of it, no timeout) -======= if(_reuse && _canReuse && connected()) { DEBUG_HTTPCLIENT("[HTTP-Client] connect: already connected, reusing connection\n"); - while(_client->available() > 0) { - _client->read(); - } ->>>>>>> master + StreamNull devNull; // should we have a global one? + _client->to(devNull); // clear _client's output (all of it, no timeout) return true; } @@ -1157,10 +1145,6 @@ bool HTTPClient::connect(void) } #endif - -#ifdef ESP8266 - _client->setNoDelay(true); -#endif return connected(); } From 679134c7d7e7e8372c04a9cb9bef88fc44aa0aee Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 18 Jul 2020 12:49:17 +0200 Subject: [PATCH 094/207] wip api --- cores/esp8266/Stream.h | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index f96fa62c29..21b26bdbf7 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -196,18 +196,15 @@ class Stream: public Print { using oneShotMs = esp8266::polledTimeout::oneShotFastMs; - size_t to (Print* to, - const ssize_t maxLen = -1, - int readUntilChar = -1, - oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* =>getTimeout() */); - - size_t to (Print& to, - const ssize_t maxLen = -1, - int readUntilChar = -1, - oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* =>getTimeout() */) - { - return this->to(&to, maxLen, readUntilChar, timeoutMs); - } + size_t toNow (Print* to) { return toFull(to, -1, -1, oneShotMs::alwaysExpired); } + size_t toUntil (Print* to, int readUntilChar, oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toFull(to, -1, readUntilChar, timeout); } + size_t toSize (Print* to, const ssize_t maxLen, oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toFull(to, maxLen, -1, timeout); } + + + size_t toFull (Print* to, + const ssize_t maxLen = -1, + int readUntilChar = -1, + oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* =>getTimeout() */); typedef enum { From dc30f677fb06c5baa04df7a4f16a8de90cc0398f Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 19 Jul 2020 17:35:07 +0200 Subject: [PATCH 095/207] api wip --- cores/esp8266/Stream.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 21b26bdbf7..7f2e84fa77 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -196,6 +196,7 @@ class Stream: public Print { using oneShotMs = esp8266::polledTimeout::oneShotFastMs; +#if 0 size_t toNow (Print* to) { return toFull(to, -1, -1, oneShotMs::alwaysExpired); } size_t toUntil (Print* to, int readUntilChar, oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toFull(to, -1, readUntilChar, timeout); } size_t toSize (Print* to, const ssize_t maxLen, oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toFull(to, maxLen, -1, timeout); } @@ -205,6 +206,19 @@ class Stream: public Print { const ssize_t maxLen = -1, int readUntilChar = -1, oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* =>getTimeout() */); +#else + size_t to (Print* to, + const ssize_t maxLen = -1, + int readUntilChar = -1, + oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* =>getTimeout() */); + size_t to (Print& to, + const ssize_t maxLen = -1, + int readUntilChar = -1, + oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* =>getTimeout() */) + { + return to(&to, maxLenm readUntilChar, timeoutMs); + } +#endif typedef enum { From 13e85121b84940313bb76db2fee587358e777df0 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 19 Jul 2020 18:07:30 +0200 Subject: [PATCH 096/207] wip --- cores/esp8266/Stream.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 7f2e84fa77..b89a2e5ca4 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -216,7 +216,7 @@ class Stream: public Print { int readUntilChar = -1, oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* =>getTimeout() */) { - return to(&to, maxLenm readUntilChar, timeoutMs); + return to(&to, maxLen, readUntilChar, timeoutMs); } #endif From b5ced49f06ec5059bb0a47602e79ee2621fbb2f4 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 24 Jul 2020 18:00:31 +0200 Subject: [PATCH 097/207] size_t read() -> int read() / to() -> toNow/toUntil/toSize --- cores/esp8266/Client.h | 23 ++++++-------- cores/esp8266/FS.cpp | 2 +- cores/esp8266/FS.h | 8 ++--- cores/esp8266/FSImpl.h | 2 +- cores/esp8266/HardwareSerial.h | 10 ++---- cores/esp8266/Stream.cpp | 4 +-- cores/esp8266/Stream.h | 31 +++++++++++++++++-- cores/esp8266/StreamDev.cpp | 10 +++--- cores/esp8266/StreamDev.h | 10 +++--- cores/esp8266/WString.h | 5 --- cores/esp8266/spiffs_api.h | 4 +-- .../src/ESP8266HTTPClient.cpp | 16 +++++----- .../src/ESP8266WebServer-impl.h | 6 ++-- .../ESP8266WebServer/src/ESP8266WebServer.h | 2 +- .../examples/WiFiEcho/WiFiEcho.ino | 3 +- libraries/ESP8266WiFi/src/WiFiClient.cpp | 4 +++ libraries/ESP8266WiFi/src/WiFiClient.h | 12 ++----- .../ESP8266WiFi/src/WiFiClientSecureBearSSL.h | 3 +- libraries/LittleFS/src/LittleFS.h | 2 +- libraries/SDFS/src/SDFS.h | 2 +- 20 files changed, 80 insertions(+), 79 deletions(-) diff --git a/cores/esp8266/Client.h b/cores/esp8266/Client.h index 14aef7bdbc..7f8f810458 100644 --- a/cores/esp8266/Client.h +++ b/cores/esp8266/Client.h @@ -26,15 +26,15 @@ class Client: public Stream { public: - virtual int connect(IPAddress ip, uint16_t port) =0; - virtual int connect(const char *host, uint16_t port) =0; - virtual size_t write(uint8_t) =0; - virtual size_t write(const uint8_t *buf, size_t size) =0; - virtual int available() = 0; - virtual int read() = 0; - virtual int read(uint8_t *buf, size_t size) = 0; - virtual int peek() = 0; - virtual void flush() = 0; + virtual int connect(IPAddress ip, uint16_t port) = 0; + virtual int connect(const char *host, uint16_t port) = 0; + virtual size_t write(uint8_t) override = 0; + virtual size_t write(const uint8_t *buf, size_t size) override = 0; + virtual int available() override = 0; + virtual int read() override = 0; + virtual int read(uint8_t *buf, size_t size) override = 0; + virtual int peek() override = 0; + virtual void flush() override = 0; virtual void stop() = 0; virtual uint8_t connected() = 0; virtual operator bool() = 0; @@ -45,11 +45,6 @@ class Client: public Stream { const uint8_t* rawIPAddress(const IPAddress& addr) { return addr.raw_address(); } - - // substitute for `virtual int ::read(buf, len)` in `Stream::` - virtual int readNow (char* buffer, size_t len) override { - return read((uint8_t*)buffer, len); - } }; #endif diff --git a/cores/esp8266/FS.cpp b/cores/esp8266/FS.cpp index 392955d492..98671f1476 100644 --- a/cores/esp8266/FS.cpp +++ b/cores/esp8266/FS.cpp @@ -58,7 +58,7 @@ int File::read() { return result; } -size_t File::read(uint8_t* buf, size_t size) { +int File::read(uint8_t* buf, size_t size) { if (!_p) return -1; diff --git a/cores/esp8266/FS.h b/cores/esp8266/FS.h index cf4017acbc..083729f3d2 100644 --- a/cores/esp8266/FS.h +++ b/cores/esp8266/FS.h @@ -66,7 +66,7 @@ class File : public Stream size_t readBytes(char *buffer, size_t length) override { return read((uint8_t*)buffer, length); } - size_t read(uint8_t* buf, size_t size); + int read(uint8_t* buf, size_t size) override; bool seek(uint32_t pos, SeekMode mode); bool seek(uint32_t pos) { return seek(pos, SeekSet); @@ -84,6 +84,7 @@ class File : public Stream bool isDirectory() const; // Arduino "class SD" methods for compatibility + //XXX use stream::to / check read(buf,size) result template size_t write(T &src){ uint8_t obuf[256]; size_t doneLen = 0; @@ -116,11 +117,6 @@ class File : public Stream time_t getCreationTime(); void setTimeCallback(time_t (*cb)(void)); - // substitute for `virtual int ::read(buf, len)` in `Stream::` - virtual int readNow (char* buffer, size_t len) override { - return read((uint8_t*)buffer, len); - } - protected: FileImplPtr _p; diff --git a/cores/esp8266/FSImpl.h b/cores/esp8266/FSImpl.h index 1a3566f2cb..c1c8575cf9 100644 --- a/cores/esp8266/FSImpl.h +++ b/cores/esp8266/FSImpl.h @@ -30,7 +30,7 @@ class FileImpl { public: virtual ~FileImpl() { } virtual size_t write(const uint8_t *buf, size_t size) = 0; - virtual size_t read(uint8_t* buf, size_t size) = 0; + virtual int read(uint8_t* buf, size_t size) = 0; virtual void flush() = 0; virtual bool seek(uint32_t pos, SeekMode mode) = 0; virtual size_t position() const = 0; diff --git a/cores/esp8266/HardwareSerial.h b/cores/esp8266/HardwareSerial.h index 83185eedfe..bed3800c31 100644 --- a/cores/esp8266/HardwareSerial.h +++ b/cores/esp8266/HardwareSerial.h @@ -141,12 +141,6 @@ class HardwareSerial: public Stream return true; } - // substitute for virtual int ::read(buf, len) - virtual int readNow (char* buffer, size_t len) override - { - return HardwareSerial::read(buffer, len); - } - // return a pointer to available data buffer (size = available()) // semantic forbids any kind of read() before calling peekConsume() const char* peekBuffer () override @@ -172,11 +166,11 @@ class HardwareSerial: public Stream return uart_read_char(_uart); } // ::read(buffer, size): same as readBytes without timeout - size_t read(char* buffer, size_t size) // should override, see Stream.h + int read(char* buffer, size_t size) { return uart_read(_uart, buffer, size); } - size_t read(uint8_t* buffer, size_t size) // should override, see Stream.h + int read(uint8_t* buffer, size_t size) override { return uart_read(_uart, (char*)buffer, size); } diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index 50771b97a3..211cd40e02 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -263,8 +263,8 @@ String Stream::readStringUntil(char terminator) { } // read what can be read, immediate exit on unavailable data -// prototype should be `int Stream::read(char* buffer, size_t maxLen)` like Arduino `int Client::read(buf, len)` -int Stream::readNow (char* buffer, size_t maxLen) +// prototype similar to Arduino's `int Client::read(buf, len)` +int Stream::read (uint8_t* buffer, size_t maxLen) { IAMSLOW(); diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index b89a2e5ca4..c64a3edff2 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -109,6 +109,7 @@ class Stream: public Print { virtual String readString(); String readStringUntil(char terminator); +#if 0 //////////////////// extension: readNow (is ::read() with unified signature) // (supposed to be internally used, and ephemeral) // @@ -124,6 +125,10 @@ class Stream: public Print { // with no timeout: immediate return when no more data are available virtual int readNow (char* buffer, size_t len); virtual int readNow (uint8_t* buffer, size_t len) final { return readNow((char*)buffer, len); } +#else + virtual int read (uint8_t* buffer, size_t len); + int read (char* buffer, size_t len) { return read((uint8_t*)buffer, len); } +#endif //////////////////// extension: direct access to input buffer // for providing, when possible, a pointer to available data for read @@ -196,16 +201,36 @@ class Stream: public Print { using oneShotMs = esp8266::polledTimeout::oneShotFastMs; -#if 0 +#if 1 + // ::to*() methods: + // - always stop before timeout when "no-more-input-possible-data" or "no-more-output-possible-data" condition is met + // - always return number of transfered bytes + + // transfers already buffered / immediately available data + // returns number of transfered bytes size_t toNow (Print* to) { return toFull(to, -1, -1, oneShotMs::alwaysExpired); } - size_t toUntil (Print* to, int readUntilChar, oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toFull(to, -1, readUntilChar, timeout); } - size_t toSize (Print* to, const ssize_t maxLen, oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toFull(to, maxLen, -1, timeout); } + // transfers data until standard or specified timeout + // returns number of transfered bytes + size_t toAll (Print* to, oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* =>getTimeout() */) { return toFull(to, -1, -1, timeoutMs); } + + // transfers data until a char is encountered (the char is also transfered) with standard timeout + // returns number of transfered bytes + size_t toUntil (Print* to, int readUntilChar, oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toFull(to, -1, readUntilChar, timeoutMs); } + + // transfers data until requested size or standard timeout + // returns number of transfered bytes + size_t toSize (Print* to, const ssize_t maxLen, oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toFull(to, maxLen, -1, timeoutMs); } + +protected: size_t toFull (Print* to, const ssize_t maxLen = -1, int readUntilChar = -1, oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* =>getTimeout() */); + +public: + #else size_t to (Print* to, const ssize_t maxLen = -1, diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index 77547d0e3f..2291b54763 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -26,10 +26,10 @@ using esp8266::polledTimeout::oneShotFastMs; using esp8266::polledTimeout::periodicFastMs; -size_t Stream::to (Print* to, - const ssize_t len, - int readUntilChar, - oneShotFastMs::timeType timeoutMs) +size_t Stream::toFull (Print* to, + const ssize_t len, + int readUntilChar, + oneShotFastMs::timeType timeoutMs) { setWriteError(STREAMTO_SUCCESS); @@ -182,7 +182,7 @@ size_t Stream::to (Print* to, if (w) { char temp[w]; - ssize_t r = readNow(temp, w); + ssize_t r = read(temp, w); if (r < 0) { setWriteError(STREAMTO_READ_ERROR); diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 6ec109f4da..5ec6ad2743 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -43,7 +43,7 @@ class StreamNull: public Stream virtual int read() override { return -1; } virtual int peek() override { return -1; } virtual size_t readBytes(char* buffer, size_t len) override { (void)buffer; (void)len; return 0; } - virtual int readNow(char* buffer, size_t len) override { (void)buffer; (void)len; return 0; } + virtual int read(uint8_t* buffer, size_t len) override { (void)buffer; (void)len; return 0; } virtual bool outputTimeoutPossible () override { return false; } virtual bool inputTimeoutPossible () override { return false; } @@ -67,7 +67,7 @@ class StreamZero: public StreamNull virtual int read() override { return _x; } virtual int peek() override { return _x; } virtual size_t readBytes(char* buffer, size_t len) override { memset(buffer, _x, len); return len; } - virtual int readNow(char* buffer, size_t len) override { memset(buffer, _x, len); return len; } + virtual int read(uint8_t* buffer, size_t len) override { memset((char*)buffer, _x, len); return len; } virtual ssize_t streamSize () override { return 32767; } }; @@ -109,7 +109,7 @@ class StreamPtr: public StreamNull _peekPointer += cpylen; return cpylen; } - virtual int readNow(char* buffer, size_t len) override { return readBytes(buffer, len); } + virtual int read(uint8_t* buffer, size_t len) override { return readBytes((char*)buffer, len); } virtual ssize_t streamSize () override { return _size; } @@ -231,7 +231,7 @@ class SerialStreamArray: public Stream } } - virtual int readNow(char* buffer, size_t len) override + virtual int read(uint8_t* buffer, size_t len) override { while (true) { @@ -239,7 +239,7 @@ class SerialStreamArray: public Stream // end of all return 0; - int ret = m_segments[m_current]->readNow(buffer, len); + int ret = m_segments[m_current]->read(buffer, len); if (ret > 0) return ret; diff --git a/cores/esp8266/WString.h b/cores/esp8266/WString.h index 49fbc2004e..958b02b976 100644 --- a/cores/esp8266/WString.h +++ b/cores/esp8266/WString.h @@ -359,11 +359,6 @@ class String: public Stream { void peekPointerSetConsume () { peekPointer = -1; } void peekPointerReset (int pointer = 0) { peekPointer = pointer; } - // substitute for `virtual int ::read(buf, len)` in `Stream::` - virtual int readNow (char* buffer, size_t len) override { - return read(buffer, len); - } - virtual ssize_t streamSize () override { return length(); } //// Print: diff --git a/cores/esp8266/spiffs_api.h b/cores/esp8266/spiffs_api.h index 1ea04849e1..333fab6b89 100644 --- a/cores/esp8266/spiffs_api.h +++ b/cores/esp8266/spiffs_api.h @@ -390,10 +390,10 @@ class SPIFFSFileImpl : public FileImpl return result; } - size_t read(uint8_t* buf, size_t size) override + int read(uint8_t* buf, size_t size) override { CHECKFD(); - auto result = SPIFFS_read(_fs->getFs(), _fd, (void*) buf, size); + int result = SPIFFS_read(_fs->getFs(), _fd, (void*) buf, size); if (result < 0) { DEBUGV("SPIFFS_read rc=%d\r\n", result); return 0; diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 69d042309e..63976e0cc9 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -604,8 +604,8 @@ int HTTPClient::sendRequest(const char * type, const uint8_t * payload, size_t s return returnError(HTTPC_ERROR_SEND_HEADER_FAILED); } - // transfer all of it, with timeout - if (size && StreamPtr(payload, size).to(_client) != size) + // transfer all of it, with send-timeout + if (size && StreamPtr(payload, size).toAll(_client) != size) return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); // handle Server Response (Header) @@ -706,7 +706,7 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) } // transfer all of it, with timeout - size_t transferred = stream->to(_client, size); + size_t transferred = stream->toSize(_client, size); if (transferred != size) { DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] short write, asked for %d but got %d failed.\n", size, transferred); @@ -788,7 +788,7 @@ int HTTPClient::writeToStream(Stream * stream) if(_transferEncoding == HTTPC_TE_IDENTITY) { // len < 0: transfer all of it, with timeout // len >= 0: max:len, with timeout - ret = _client->to(stream, len); + ret = _client->toSize(stream, len); // have we an error? if(_client->getLastTo() != Stream::STREAMTO_SUCCESS) { @@ -816,7 +816,7 @@ int HTTPClient::writeToStream(Stream * stream) // data left? if(len > 0) { // read len bytes with timeout - int r = _client->to(stream, len); + int r = _client->toSize(stream, len); if (_client->getLastTo() != Stream::STREAMTO_SUCCESS) // not all data transferred return returnError(TO2HTTPC(_client->getLastTo())); @@ -1009,8 +1009,8 @@ bool HTTPClient::connect(void) { if(_reuse && _canReuse && connected()) { DEBUG_HTTPCLIENT("[HTTP-Client] connect: already connected, reusing connection\n"); - StreamNull devNull; // should we have a global one? - _client->to(devNull); // clear _client's output (all of it, no timeout) + StreamNull devnull; //XXXFIXME a global one would be nice + _client->toNow(&devnull); // clear _client's output (all of it, no timeout) return true; } @@ -1105,7 +1105,7 @@ bool HTTPClient::sendHeader(const char * type) DEBUG_HTTPCLIENT("[HTTP-Client] sending request header\n-----\n%s-----\n", header.c_str()); // transfer all of it, with timeout - return header.to(_client, header.length()) == header.length(); + return header.toSize(_client, header.length()) == header.length(); } /** diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index 1834c79275..6610e75ace 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -458,7 +458,7 @@ void ESP8266WebServerTemplate::send(int code, const char* content_ty //if(code == 200 && content.length() == 0 && _contentLength == CONTENT_LENGTH_NOT_SET) // _contentLength = CONTENT_LENGTH_UNKNOWN; _prepareHeader(header, code, content_type, content.length()); - size_t sent = StreamPtr(header.c_str(), header.length()).to(&_currentClient); // transfer all of it, with timeout + size_t sent = StreamPtr(header.c_str(), header.length()).toAll(&_currentClient); // transfer all of it, with timeout (void)sent; #ifdef DEBUG_ESP_HTTP_SERVER if (sent != header.length()) @@ -524,7 +524,7 @@ void ESP8266WebServerTemplate::sendContent(Stream* content) { sprintf(chunkSize, "%zx\r\n", len); _currentClient.write((const uint8_t *)chunkSize, strlen(chunkSize)); } - size_t sent = content->to(_currentClient); + size_t sent = content->toAll(_currentClient); (void)sent; /// if (sent != len) print-error-on-console-and-return-false if(_chunked){ _currentClient.write((const uint8_t *)footer, 2); @@ -534,6 +534,7 @@ void ESP8266WebServerTemplate::sendContent(Stream* content) { } } +#if 0 //XXX removeme (String is Stream) template void ESP8266WebServerTemplate::sendContent(const String& content) { if (_currentMethod == HTTP_HEAD) return; @@ -552,6 +553,7 @@ void ESP8266WebServerTemplate::sendContent(const String& content) { } } } +#endif template void ESP8266WebServerTemplate::sendContent_P(PGM_P content) { diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer.h b/libraries/ESP8266WebServer/src/ESP8266WebServer.h index 2d531adb36..f6d35abfd7 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer.h @@ -207,7 +207,7 @@ class ESP8266WebServerTemplate setContentLength(size); send(200, contentType, emptyString); if (requestMethod == HTTP_GET) - size = aStream.to(_currentClient, size); + size = aStream.toSize(_currentClient, size); return size; } diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino index 2b12d4bbaa..eb66c49d46 100644 --- a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino @@ -91,8 +91,7 @@ void loop() { else if (t == 3) { // stream to print, possibly with only one copy - //client.to(client); // <=> client.to(&client, -1, -1, client->getTimeout()) - client.to(client, -1, -1, breathMs); + client.toNow(&client); switch (client.getLastTo()) { case Stream::STREAMTO_SUCCESS: break; diff --git a/libraries/ESP8266WiFi/src/WiFiClient.cpp b/libraries/ESP8266WiFi/src/WiFiClient.cpp index 48f9561ffd..c6fbc9e94c 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.cpp +++ b/libraries/ESP8266WiFi/src/WiFiClient.cpp @@ -262,6 +262,10 @@ int WiFiClient::read() return _client->read(); } +int WiFiClient::read(uint8_t* buf, size_t size) +{ + return (int)_client->read((char*)buf, size); +} int WiFiClient::read(char* buf, size_t size) { diff --git a/libraries/ESP8266WiFi/src/WiFiClient.h b/libraries/ESP8266WiFi/src/WiFiClient.h index 3fe4f23b1c..05e4459cdc 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.h +++ b/libraries/ESP8266WiFi/src/WiFiClient.h @@ -66,11 +66,8 @@ class WiFiClient : public Client, public SList { virtual int available() override; virtual int read() override; - virtual int read(char *buf, size_t size); // should override, see Stream.h - virtual int read(uint8_t *buf, size_t size) // should override, see Stream.h - { - return WiFiClient::read((char*)buf, size); - } + virtual int read(uint8_t* buf, size_t size) override; + int read(char* buf, size_t size); virtual int peek() override; virtual size_t peekBytes(uint8_t *buffer, size_t length); @@ -125,11 +122,6 @@ class WiFiClient : public Client, public SList { bool getSync() const; void setSync(bool sync); - // substitute for virtual int ::read(buf, len) - virtual int readNow (char* buffer, size_t len) override { - return WiFiClient::read(buffer, len); - } - // peek buffer API is present virtual bool peekBufferAPI () const override; diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h index 0312857c4a..8b57ce0de8 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h @@ -54,7 +54,7 @@ class WiFiClientSecure : public WiFiClient { } size_t write(Stream& stream); // Note this is not virtual int read(uint8_t *buf, size_t size) override; - int read(char *buf, size_t size) override { return read((uint8_t*)buf, size); } + int read(char *buf, size_t size) { return read((uint8_t*)buf, size); } int available() override; int read() override; int peek() override; @@ -134,7 +134,6 @@ class WiFiClientSecure : public WiFiClient { // peek buffer API is present virtual bool peekBufferAPI () const override { return true; } - virtual int readNow (char* buffer, size_t len) override { return WiFiClientSecure::read((uint8_t*)buffer, len); } // return number of byte accessible by peekBuffer() virtual size_t availableForPeek () override { return WiFiClientSecure::available(); } diff --git a/libraries/LittleFS/src/LittleFS.h b/libraries/LittleFS/src/LittleFS.h index f78680f662..2e484e0ec4 100644 --- a/libraries/LittleFS/src/LittleFS.h +++ b/libraries/LittleFS/src/LittleFS.h @@ -346,7 +346,7 @@ class LittleFSFileImpl : public FileImpl return result; } - size_t read(uint8_t* buf, size_t size) override { + int read(uint8_t* buf, size_t size) override { if (!_opened || !_fd | !buf) { return 0; } diff --git a/libraries/SDFS/src/SDFS.h b/libraries/SDFS/src/SDFS.h index a884552ffc..ab225efa6f 100644 --- a/libraries/SDFS/src/SDFS.h +++ b/libraries/SDFS/src/SDFS.h @@ -273,7 +273,7 @@ class SDFSFileImpl : public FileImpl return _opened ? _fd->write(buf, size) : -1; } - size_t read(uint8_t* buf, size_t size) override + int read(uint8_t* buf, size_t size) override { return _opened ? _fd->read(buf, size) : -1; } From 3f9d4ccb59e742224cbab174728c49ddd730ce33 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 24 Jul 2020 18:17:52 +0200 Subject: [PATCH 098/207] streamstring helper --- cores/esp8266/WString.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cores/esp8266/WString.h b/cores/esp8266/WString.h index 958b02b976..2966cd0936 100644 --- a/cores/esp8266/WString.h +++ b/cores/esp8266/WString.h @@ -40,6 +40,8 @@ class __FlashStringHelper; #define FPSTR(pstr_pointer) (reinterpret_cast(pstr_pointer)) #define F(string_literal) (FPSTR(PSTR(string_literal))) +#define STRING_IS_STREAM 1 // helper to avoid overloading functions using String:: with StreamString:: + // The string class class String: public Stream { // use a function pointer to allow for "if (s)" without the From b80bfff166a6c71a08eb458762fddabdf0ef5a07 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 24 Jul 2020 18:25:30 +0200 Subject: [PATCH 099/207] fixes --- .../src/ESP8266WebServer-impl.h | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index 6610e75ace..4441dbd146 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -524,7 +524,7 @@ void ESP8266WebServerTemplate::sendContent(Stream* content) { sprintf(chunkSize, "%zx\r\n", len); _currentClient.write((const uint8_t *)chunkSize, strlen(chunkSize)); } - size_t sent = content->toAll(_currentClient); + size_t sent = content->toAll(&_currentClient); (void)sent; /// if (sent != len) print-error-on-console-and-return-false if(_chunked){ _currentClient.write((const uint8_t *)footer, 2); @@ -534,7 +534,20 @@ void ESP8266WebServerTemplate::sendContent(Stream* content) { } } -#if 0 //XXX removeme (String is Stream) +#if 1 + +template +void ESP8266WebServerTemplate::sendContent(const String& content) { + StreamPtr ref(content.c_str(), content.length()); + return sendContent(&ref); +} + +//template +//void ESP8266WebServerTemplate::sendContent(String& content) { +// return sendContent(&content); +//} + +#else template void ESP8266WebServerTemplate::sendContent(const String& content) { if (_currentMethod == HTTP_HEAD) return; From d62c7193a83d00b2aa59fe87b9683d2492d13c1a Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 28 Jul 2020 00:52:00 +0200 Subject: [PATCH 100/207] make string_is_Stream optional --- cores/esp8266/StreamString.h | 18 +++++++++++++++++- cores/esp8266/WString.cpp | 10 ++++------ cores/esp8266/WString.h | 19 +++++++++++-------- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/cores/esp8266/StreamString.h b/cores/esp8266/StreamString.h index 28fba85e07..e5e2d9bb92 100644 --- a/cores/esp8266/StreamString.h +++ b/cores/esp8266/StreamString.h @@ -25,8 +25,24 @@ #include "WString.h" -// StreamString has been integrated into String +#if STRING_IS_STREAM +// StreamString has been integrated into String using StreamString = String; +#else // !STRING_IS_STREAM + +class StreamString: public Stream, public String { +public: + size_t write(const uint8_t *buffer, size_t size) override; + size_t write(uint8_t data) override; + + int available() override; + int read() override; + int peek() override; + void flush() override; +}; + +#endif // !STRING_IS_STREAM + #endif /* STREAMSTRING_H_ */ diff --git a/cores/esp8266/WString.cpp b/cores/esp8266/WString.cpp index adf348894e..135249ffc0 100644 --- a/cores/esp8266/WString.cpp +++ b/cores/esp8266/WString.cpp @@ -48,7 +48,6 @@ String::String(const __FlashStringHelper *pstr) { *this = pstr; // see operator = } -#ifdef __GXX_EXPERIMENTAL_CXX0X__ String::String(String &&rval) { init(); move(rval); @@ -58,7 +57,6 @@ String::String(StringSumHelper &&rval) { init(); move(rval); } -#endif String::String(char c) { init(); @@ -226,7 +224,6 @@ String & String::copy(const __FlashStringHelper *pstr, unsigned int length) { return *this; } -#ifdef __GXX_EXPERIMENTAL_CXX0X__ void String::move(String &rhs) { if (buffer()) { if (capacity() >= rhs.len()) { @@ -255,7 +252,6 @@ void String::move(String &rhs) { rhs.setLen(0); rhs.setBuffer(nullptr); } -#endif String & String::operator =(const String &rhs) { if (this == &rhs) @@ -269,7 +265,6 @@ String & String::operator =(const String &rhs) { return *this; } -#ifdef __GXX_EXPERIMENTAL_CXX0X__ String & String::operator =(String &&rval) { if (this != &rval) move(rval); @@ -281,7 +276,6 @@ String & String::operator =(StringSumHelper &&rval) { move(rval); return *this; } -#endif String & String::operator =(const char *cstr) { if (cstr) @@ -858,6 +852,8 @@ double String::toDouble(void) const const String emptyString; +#if STRING_IS_STREAM + ///////////////////////////////////////////// // Stream API: @@ -965,3 +961,5 @@ int String::read (char* buffer, size_t len) peekPointer += l; return l; } + +#endif // STRING_IS_STREAM diff --git a/cores/esp8266/WString.h b/cores/esp8266/WString.h index 2966cd0936..2a8f5053fe 100644 --- a/cores/esp8266/WString.h +++ b/cores/esp8266/WString.h @@ -40,10 +40,14 @@ class __FlashStringHelper; #define FPSTR(pstr_pointer) (reinterpret_cast(pstr_pointer)) #define F(string_literal) (FPSTR(PSTR(string_literal))) -#define STRING_IS_STREAM 1 // helper to avoid overloading functions using String:: with StreamString:: +#define STRING_IS_STREAM 0 // helper to avoid overloading functions using String:: with StreamString:: // The string class -class String: public Stream { +class String +#if STRING_IS_STREAM + : public Stream +#endif +{ // use a function pointer to allow for "if (s)" without the // complications of an operator bool(). for more information, see: // http://www.artima.com/cppsource/safebool.html @@ -60,10 +64,9 @@ class String: public Stream { String(const char *cstr = nullptr); String(const String &str); String(const __FlashStringHelper *str); -#ifdef __GXX_EXPERIMENTAL_CXX0X__ String(String &&rval); String(StringSumHelper &&rval); -#endif + explicit String(char c); explicit String(unsigned char, unsigned char base = 10); explicit String(int, unsigned char base = 10); @@ -99,10 +102,8 @@ class String: public Stream { String & operator =(const String &rhs); String & operator =(const char *cstr); String & operator = (const __FlashStringHelper *str); -#ifdef __GXX_EXPERIMENTAL_CXX0X__ String & operator =(String &&rval); String & operator =(StringSumHelper &&rval); -#endif // concatenate (works w/ built-in types) @@ -320,10 +321,9 @@ class String: public Stream { // copy and move String & copy(const char *cstr, unsigned int length); String & copy(const __FlashStringHelper *pstr, unsigned int length); -#ifdef __GXX_EXPERIMENTAL_CXX0X__ void move(String &rhs); -#endif +#if STRING_IS_STREAM ///////////////////////////////////////////// // Print/Stream/peekBuffer API: @@ -367,6 +367,9 @@ class String: public Stream { virtual bool outputTimeoutPossible () override { return false; } virtual int availableForWrite() override { return 64; } // or biggestChunk()/4 or max(1,reserved-length)? + +#endif // STRING_IS_STREAM + }; class StringSumHelper: public String { From 192faab18d54c06001de58a1f4f0d7cd2a98e872 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 28 Jul 2020 01:40:19 +0200 Subject: [PATCH 101/207] enable string is stream --- cores/esp8266/WString.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cores/esp8266/WString.h b/cores/esp8266/WString.h index 2a8f5053fe..358426d281 100644 --- a/cores/esp8266/WString.h +++ b/cores/esp8266/WString.h @@ -40,7 +40,7 @@ class __FlashStringHelper; #define FPSTR(pstr_pointer) (reinterpret_cast(pstr_pointer)) #define F(string_literal) (FPSTR(PSTR(string_literal))) -#define STRING_IS_STREAM 0 // helper to avoid overloading functions using String:: with StreamString:: +#define STRING_IS_STREAM 1 // helper to avoid overloading functions using String:: with StreamString:: // The string class class String From 951737c0308483b9933f10c53446cc036ed190b4 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 28 Jul 2020 18:31:27 +0200 Subject: [PATCH 102/207] stream::to(): templates allowing passing reference, fixes --- cores/esp8266/Stream.cpp | 8 +++ cores/esp8266/Stream.h | 57 ++++++------------- cores/esp8266/StreamDev.cpp | 10 +++- cores/esp8266/StreamDev.h | 7 ++- cores/esp8266/WString.cpp | 2 +- cores/esp8266/WString.h | 3 +- .../src/ESP8266HTTPClient.cpp | 3 +- .../ESP8266WiFi/src/CertStoreBearSSL.cpp | 4 +- 8 files changed, 43 insertions(+), 51 deletions(-) diff --git a/cores/esp8266/Stream.cpp b/cores/esp8266/Stream.cpp index 211cd40e02..76f83992b0 100644 --- a/cores/esp8266/Stream.cpp +++ b/cores/esp8266/Stream.cpp @@ -244,21 +244,29 @@ size_t Stream::readBytesUntil(char terminator, char *buffer, size_t length) { String Stream::readString() { String ret; +#if STRING_IS_STREAM + toAll(ret); +#else int c = timedRead(); while(c >= 0) { ret += (char) c; c = timedRead(); } +#endif return ret; } String Stream::readStringUntil(char terminator) { String ret; +#if STRING_IS_STREAM + toUntil(ret, terminator); +#else int c = timedRead(); while(c >= 0 && c != terminator) { ret += (char) c; c = timedRead(); } +#endif return ret; } diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index c64a3edff2..ae833d153b 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -58,7 +58,7 @@ class Stream: public Print { // parsing methods void setTimeout(unsigned long timeout); // sets maximum milliseconds to wait for stream data, default is 1 second - unsigned long getTimeout () { return _timeout; } + unsigned long getTimeout () const { return _timeout; } bool find(const char *target); // reads data from the stream until the target string is found bool find(uint8_t *target) { @@ -109,26 +109,8 @@ class Stream: public Print { virtual String readString(); String readStringUntil(char terminator); -#if 0 - //////////////////// extension: readNow (is ::read() with unified signature) - // (supposed to be internally used, and ephemeral) - // - // about ::read(buf, len): conflicting returned type: - // - `int` in arduino's Client:: - // - `size_t` in esp8266 API (HardwareSerial::, FS::) - // - not existent in arduino's Stream:: - // changing every read()/write() `size_t` return type to `int` will be a breaking change - // => adding int ::readNow(buf, len) for now (following official `int Client::read(buf, len))` - // - // int ::readNow(buf, len) - // read at most len bytes, returns effectively transferred bytes (can be less than 'len') - // with no timeout: immediate return when no more data are available - virtual int readNow (char* buffer, size_t len); - virtual int readNow (uint8_t* buffer, size_t len) final { return readNow((char*)buffer, len); } -#else virtual int read (uint8_t* buffer, size_t len); int read (char* buffer, size_t len) { return read((uint8_t*)buffer, len); } -#endif //////////////////// extension: direct access to input buffer // for providing, when possible, a pointer to available data for read @@ -201,7 +183,6 @@ class Stream: public Print { using oneShotMs = esp8266::polledTimeout::oneShotFastMs; -#if 1 // ::to*() methods: // - always stop before timeout when "no-more-input-possible-data" or "no-more-output-possible-data" condition is met // - always return number of transfered bytes @@ -209,42 +190,36 @@ class Stream: public Print { // transfers already buffered / immediately available data // returns number of transfered bytes size_t toNow (Print* to) { return toFull(to, -1, -1, oneShotMs::alwaysExpired); } + template + size_t toNow (T& to) { return toNow((Print*)&to); } - // transfers data until standard or specified timeout + // transfers data until timeout // returns number of transfered bytes - size_t toAll (Print* to, oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* =>getTimeout() */) { return toFull(to, -1, -1, timeoutMs); } + size_t toAll (Print* to, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toFull(to, -1, -1, timeoutMs); } + template + size_t toAll (T& to, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toAll((Print*)&to, timeoutMs); } - // transfers data until a char is encountered (the char is also transfered) with standard timeout + // transfers data until a char is encountered (the char is also transfered) with timeout // returns number of transfered bytes - size_t toUntil (Print* to, int readUntilChar, oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toFull(to, -1, readUntilChar, timeoutMs); } + size_t toUntil (Print* to, const int readUntilChar, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toFull(to, -1, readUntilChar, timeoutMs); } + template + size_t toUntil (T& to, const int readUntilChar, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toUntil((Print*)&to, readUntilChar, timeoutMs); } - // transfers data until requested size or standard timeout + // transfers data until requested size or timeout // returns number of transfered bytes - size_t toSize (Print* to, const ssize_t maxLen, oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toFull(to, maxLen, -1, timeoutMs); } + size_t toSize (Print* to, const ssize_t maxLen, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toFull(to, maxLen, -1, timeoutMs); } + template + size_t toSize (T& to, const ssize_t maxLen, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toSize((Print*)&to, maxLen, timeoutMs); } protected: size_t toFull (Print* to, const ssize_t maxLen = -1, - int readUntilChar = -1, + const int readUntilChar = -1, oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* =>getTimeout() */); public: -#else - size_t to (Print* to, - const ssize_t maxLen = -1, - int readUntilChar = -1, - oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* =>getTimeout() */); - size_t to (Print& to, - const ssize_t maxLen = -1, - int readUntilChar = -1, - oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* =>getTimeout() */) - { - return to(&to, maxLen, readUntilChar, timeoutMs); - } -#endif - typedef enum { STREAMTO_SUCCESS = 0, diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index 2291b54763..8cf1ae7a4c 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -21,15 +21,15 @@ #include -#include +#include using esp8266::polledTimeout::oneShotFastMs; using esp8266::polledTimeout::periodicFastMs; size_t Stream::toFull (Print* to, const ssize_t len, - int readUntilChar, - oneShotFastMs::timeType timeoutMs) + const int readUntilChar, + const oneShotFastMs::timeType timeoutMs) { setWriteError(STREAMTO_SUCCESS); @@ -228,3 +228,7 @@ size_t Stream::toFull (Print* to, } return written; } + +#if !defined(NO_GLOBAL_INSTANCES) && !defined(NO_GLOBAL_STREAMDEV) +StreamNull devnull; +#endif diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 5ec6ad2743..6ea0427f82 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -75,7 +75,7 @@ class StreamZero: public StreamNull /////////////////////////////////////////////// // static buffer (in flash or ram) // - black hole as output, swallow everything, availableForWrite = infinite -// - Stream buffer out, resettable +// - Stream buffer out as input, resettable class StreamPtr: public StreamNull { @@ -266,4 +266,9 @@ class SerialStream: public SerialStreamArray<> }; /////////////////////////////////////////////// + +#if !defined(NO_GLOBAL_INSTANCES) && !defined(NO_GLOBAL_STREAMDEV) +extern StreamNull devnull; +#endif + #endif // __STREAMDEV_H diff --git a/cores/esp8266/WString.cpp b/cores/esp8266/WString.cpp index 135249ffc0..43ac69df18 100644 --- a/cores/esp8266/WString.cpp +++ b/cores/esp8266/WString.cpp @@ -941,7 +941,7 @@ void String::peekConsume (size_t consume) peekPointer = std::min((size_t)length(), peekPointer + consume); } -int String::read (char* buffer, size_t len) +int String::read (uint8_t* buffer, size_t len) { if (peekPointer < 0) { diff --git a/cores/esp8266/WString.h b/cores/esp8266/WString.h index 358426d281..ebb2c84c07 100644 --- a/cores/esp8266/WString.h +++ b/cores/esp8266/WString.h @@ -356,8 +356,9 @@ class String virtual const char* peekBuffer () override; virtual void peekConsume (size_t consume) override; virtual bool inputTimeoutPossible () override { return false; } - virtual int read (char* buffer, size_t len) /*should override*/; + virtual int read (uint8_t* buffer, size_t len) override; + // calling peekPointerSetConsume() will consume bytes as they are stream-read void peekPointerSetConsume () { peekPointer = -1; } void peekPointerReset (int pointer = 0) { peekPointer = pointer; } diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 63976e0cc9..471ad60f8b 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -1009,8 +1009,7 @@ bool HTTPClient::connect(void) { if(_reuse && _canReuse && connected()) { DEBUG_HTTPCLIENT("[HTTP-Client] connect: already connected, reusing connection\n"); - StreamNull devnull; //XXXFIXME a global one would be nice - _client->toNow(&devnull); // clear _client's output (all of it, no timeout) + _client->toNow(devnull); // clear _client's output (all of it, no timeout) return true; } diff --git a/libraries/ESP8266WiFi/src/CertStoreBearSSL.cpp b/libraries/ESP8266WiFi/src/CertStoreBearSSL.cpp index 905efde2ba..bcd042d265 100644 --- a/libraries/ESP8266WiFi/src/CertStoreBearSSL.cpp +++ b/libraries/ESP8266WiFi/src/CertStoreBearSSL.cpp @@ -121,7 +121,7 @@ int CertStore::initCertStore(FS &fs, const char *indexFileName, const char *data uint8_t fileHeader[60]; // 0..15 = filename in ASCII // 48...57 = length in decimal ASCII - uint32_t length; + int32_t length; if (data.read(fileHeader, sizeof(fileHeader)) != sizeof(fileHeader)) { break; } @@ -197,7 +197,7 @@ const br_x509_trust_anchor *CertStore::findHashedTA(void *ctx, void *hashed_dn, free(der); return nullptr; } - if (data.read((uint8_t *)der, ci.length) != ci.length) { + if (data.read((uint8_t *)der, ci.length) != (int)ci.length) { free(der); return nullptr; } From ca27dd7465aa30e0e7a846c585aeaba5c0b5abf5 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 28 Jul 2020 18:44:39 +0200 Subject: [PATCH 103/207] wwwserver: parsing: use stream::to --- libraries/ESP8266WebServer/src/Parsing-impl.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/libraries/ESP8266WebServer/src/Parsing-impl.h b/libraries/ESP8266WebServer/src/Parsing-impl.h index 8d3fda2782..4652b2a506 100644 --- a/libraries/ESP8266WebServer/src/Parsing-impl.h +++ b/libraries/ESP8266WebServer/src/Parsing-impl.h @@ -39,6 +39,16 @@ static const char Content_Type[] PROGMEM = "Content-Type"; static const char filename[] PROGMEM = "filename"; +#if STRING_IS_STREAM + +template +static bool readBytesWithTimeout(typename ServerType::ClientType& client, size_t maxLength, String& data, int timeout_ms) +{ + return client.toSize(data, maxLength, timeout_ms) == maxLength; +} + +#else + template static bool readBytesWithTimeout(typename ServerType::ClientType& client, size_t maxLength, String& data, int timeout_ms) { @@ -60,6 +70,8 @@ static bool readBytesWithTimeout(typename ServerType::ClientType& client, size_t return data.length() == maxLength; } +#endif + template bool ESP8266WebServerTemplate::_parseRequest(ClientType& client) { // Read the first line of HTTP request From 1ce2351dc2d25a263eb0fef8e97b8956d9e934e9 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 28 Jul 2020 23:01:36 +0200 Subject: [PATCH 104/207] replace datasource by stream (less fragmentation) --- .../ESP8266WiFi/src/include/ClientContext.h | 33 ++-- .../ESP8266WiFi/src/include/DataSource.h | 154 ------------------ 2 files changed, 17 insertions(+), 170 deletions(-) delete mode 100644 libraries/ESP8266WiFi/src/include/DataSource.h diff --git a/libraries/ESP8266WiFi/src/include/ClientContext.h b/libraries/ESP8266WiFi/src/include/ClientContext.h index 4d9546b8e5..092f3d923a 100644 --- a/libraries/ESP8266WiFi/src/include/ClientContext.h +++ b/libraries/ESP8266WiFi/src/include/ClientContext.h @@ -29,7 +29,8 @@ typedef void (*discard_cb_t)(void*, ClientContext*); extern "C" void esp_yield(); extern "C" void esp_schedule(); -#include "DataSource.h" +#include +#include bool getDefaultPrivateGlobalSyncValue (); @@ -374,7 +375,8 @@ class ClientContext if (!_pcb) { return 0; } - return _write_from_source(new BufferDataSource(data, size)); + StreamPtr ptr(data, size); + return _write_from_source(&ptr); } size_t write(Stream& stream) @@ -382,7 +384,7 @@ class ClientContext if (!_pcb) { return 0; } - return _write_from_source(new BufferedStreamDataSource(stream, stream.available())); + return _write_from_source(&stream); } size_t write_P(PGM_P buf, size_t size) @@ -390,8 +392,8 @@ class ClientContext if (!_pcb) { return 0; } - ProgmemStream stream(buf, size); - return _write_from_source(new BufferedStreamDataSource(stream, size)); + StreamPtr ptr(buf, size); + return _write_from_source(&ptr); } void keepAlive (uint16_t idle_sec = TCP_DEFAULT_KEEPALIVE_IDLE_SEC, uint16_t intv_sec = TCP_DEFAULT_KEEPALIVE_INTERVAL_SEC, uint8_t count = TCP_DEFAULT_KEEPALIVE_COUNT) @@ -475,7 +477,7 @@ class ClientContext } } - size_t _write_from_source(DataSource* ds) + size_t _write_from_source(Stream* ds) { assert(_datasource == nullptr); assert(!_send_waiting); @@ -491,7 +493,6 @@ class ClientContext if (_is_timeout()) { DEBUGV(":wtmo\r\n"); } - delete _datasource; _datasource = nullptr; break; } @@ -518,20 +519,20 @@ class ClientContext return false; } - DEBUGV(":wr %d %d\r\n", _datasource->available(), _written); + DEBUGV(":wr %d %d\r\n", _datasource->availableForPeek(), _written); bool has_written = false; while (_datasource) { if (state() == CLOSED) return false; - size_t next_chunk_size = std::min((size_t)tcp_sndbuf(_pcb), _datasource->available()); + size_t next_chunk_size = std::min((size_t)tcp_sndbuf(_pcb), _datasource->availableForPeek()); if (!next_chunk_size) break; - const uint8_t* buf = _datasource->get_buffer(next_chunk_size); + const char* buf = _datasource->peekBuffer(); uint8_t flags = 0; - if (next_chunk_size < _datasource->available()) + if (next_chunk_size < _datasource->availableForPeek()) // PUSH is meant for peer, telling to give data to user app as soon as received // PUSH "may be set" when sender has finished sending a "meaningful" data block // PUSH does not break Nagle @@ -545,15 +546,15 @@ class ClientContext err_t err = tcp_write(_pcb, buf, next_chunk_size, flags); - DEBUGV(":wrc %d %d %d\r\n", next_chunk_size, _datasource->available(), (int)err); + DEBUGV(":wrc %d %d %d\r\n", next_chunk_size, _datasource->availableForPeek(), (int)err); if (err == ERR_OK) { - _datasource->release_buffer(buf, next_chunk_size); + _datasource->peekConsume(next_chunk_size); _written += next_chunk_size; has_written = true; } else { - // ERR_MEM(-1) is a valid error meaning - // "come back later". It leaves state() opened + // ERR_MEM(-1) is a valid error meaning + // "come back later". It leaves state() opened break; } } @@ -706,7 +707,7 @@ class ClientContext discard_cb_t _discard_cb; void* _discard_cb_arg; - DataSource* _datasource = nullptr; + Stream* _datasource = nullptr; size_t _written = 0; uint32_t _timeout_ms = 5000; uint32_t _op_start_time = 0; diff --git a/libraries/ESP8266WiFi/src/include/DataSource.h b/libraries/ESP8266WiFi/src/include/DataSource.h deleted file mode 100644 index 2a0bfed260..0000000000 --- a/libraries/ESP8266WiFi/src/include/DataSource.h +++ /dev/null @@ -1,154 +0,0 @@ -/* DataSource.h - a read-only object similar to Stream, but with less methods - * Copyright (c) 2016 Ivan Grokhotkov. All rights reserved. - * This file is distributed under MIT license. - */ -#ifndef DATASOURCE_H -#define DATASOURCE_H - -#include - -class DataSource { -public: - virtual ~DataSource() {} - virtual size_t available() = 0; - virtual const uint8_t* get_buffer(size_t size) = 0; - virtual void release_buffer(const uint8_t* buffer, size_t size) = 0; - -}; - -class BufferDataSource : public DataSource { -public: - BufferDataSource(const uint8_t* data, size_t size) : - _data(data), - _size(size) - { - } - - size_t available() override - { - return _size - _pos; - } - - const uint8_t* get_buffer(size_t size) override - { - (void)size; - assert(_pos + size <= _size); - return _data + _pos; - } - - void release_buffer(const uint8_t* buffer, size_t size) override - { - (void)buffer; - assert(buffer == _data + _pos); - _pos += size; - } - -protected: - const uint8_t* _data; - const size_t _size; - size_t _pos = 0; -}; - -template -class BufferedStreamDataSource : public DataSource { -public: - BufferedStreamDataSource(TStream& stream, size_t size) : - _stream(stream), - _size(size) - { - } - - size_t available() override - { - return _size - _pos; - } - - const uint8_t* get_buffer(size_t size) override - { - assert(_pos + size <= _size); - - //Data that was already read from the stream but not released (e.g. if tcp_write error occured). Otherwise this should be 0. - const size_t stream_read = _streamPos - _pos; - - //Min required buffer size: max(requested size, previous stream data already in buffer) - const size_t min_buffer_size = size > stream_read ? size : stream_read; - - //Buffer too small? - if (_bufferSize < min_buffer_size) { - uint8_t *new_buffer = new uint8_t[min_buffer_size]; - //If stream reading is ahead, than some data is already in the old buffer and needs to be copied to new resized buffer - if (_buffer && stream_read > 0) { - memcpy(new_buffer, _buffer.get(), stream_read); - } - _buffer.reset(new_buffer); - _bufferSize = min_buffer_size; - } - - //Fetch remaining data from stream - //If error in tcp_write in ClientContext::_write_some() occured earlier and therefore release_buffer was not called last time, than the requested stream data is already in the buffer. - if (size > stream_read) { - //Remaining bytes to read from stream - const size_t stream_rem = size - stream_read; - const size_t cb = _stream.readBytes(reinterpret_cast(_buffer.get() + stream_read), stream_rem); - assert(cb == stream_rem); - (void)cb; - _streamPos += stream_rem; - } - return _buffer.get(); - - } - - void release_buffer(const uint8_t* buffer, size_t size) override - { - if (size == 0) { - return; - } - - (void)buffer; - _pos += size; - - //Cannot release more than acquired through get_buffer - assert(_pos <= _streamPos); - - //Release less than requested with get_buffer? - if (_pos < _streamPos) { - // Move unreleased stream data in buffer to front - assert(_buffer); - memmove(_buffer.get(), _buffer.get() + size, _streamPos - _pos); - } - } - -protected: - TStream & _stream; - std::unique_ptr _buffer; - size_t _size; - size_t _pos = 0; - size_t _bufferSize = 0; - size_t _streamPos = 0; -}; - -class ProgmemStream -{ -public: - ProgmemStream(PGM_P buf, size_t size) : - _buf(buf), - _left(size) - { - } - - size_t readBytes(char* dst, size_t size) - { - size_t will_read = (_left < size) ? _left : size; - memcpy_P((void*)dst, (PGM_VOID_P)_buf, will_read); - _left -= will_read; - _buf += will_read; - return will_read; - } - -protected: - PGM_P _buf; - size_t _left; -}; - - -#endif //DATASOURCE_H From 84a313787272f0874a9c65301a8f5e12cec8af8a Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 29 Jul 2020 23:58:31 +0200 Subject: [PATCH 105/207] fix comments, fix mock --- cores/esp8266/Stream.h | 79 +++++++---------------- tests/host/common/include/ClientContext.h | 2 +- 2 files changed, 23 insertions(+), 58 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index ae833d153b..128b0351fb 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -113,9 +113,9 @@ class Stream: public Print { int read (char* buffer, size_t len) { return read((uint8_t*)buffer, len); } //////////////////// extension: direct access to input buffer - // for providing, when possible, a pointer to available data for read + // to provide when possible a pointer to available data for read - // informs user and ::to() on effective buffered peek API implementation + // informs user and ::to*() on effective buffered peek API implementation // by default: not available virtual bool peekBufferAPI () const { return false; } @@ -134,48 +134,16 @@ class Stream: public Print { // by default read timeout is possible (incoming data from network,serial..) // children can override to false (like String::) - // (outputTimeoutPossible() is defined in Print::) virtual bool inputTimeoutPossible () { return true; } - //////////////////// extensions: Stream streams - // Stream::to() - // - // Stream::to() uses 1-copy transfers when peekBuffer API is - // available, or makes a regular transfer through a local temporary - // stack 2-copies buffer. - // - // By default "source->to(&dest)" transfers everything until - // available (read or write) gets to 0, then immediately returns. - // - // "source->to(&dest, maxLen)" is like above but also returns when - // maxLen bytes are transferred, using the default Stream timeout. - // - // "source->to(&string, -1, '\n')" transfers source to string until - // and including a specific character, with the default Stream - // timeout. - // - // More generally ::to() will transfer as much as possible with the - // following constraints: - // - at most maxLen bytes (-1 by default is no length - // constraint) - // - readUntilChar as last transferred byte (-1 by default is - // no last char contraint) - // - timeoutMs as maximum wait time (only if maxLen>=0 or - // readUntilChar>=0, immediate otherwise) + // (outputTimeoutPossible() is defined in Print::) + + //////////////////// extensions: Streaming streams to streams + // Stream::to*() // - // timeoutMs value is by default "oneShotMs::neverExpires" which is - // internally converted to this->getTimeout() but it can be set to - // "oneShotMs::alwaysExpired" (=0) or any value within oneShotMs - // allowed range. + // Stream::to*() uses 1-copy transfers when peekBuffer API is + // available, or makes a regular transfer through a temporary buffer. // - // Return value: - // >0: the number of transferred bytes - // 0: nothing has been transferred - // When result is 0 or less than requested maxLen, this->getLastTo() - // may contain an error reason. - // - // Notes: - // - readUntilChar is copied and counted // - for efficiency, Stream classes should implement peekAPI when // possible // - for an efficient timeout management, Print/Stream classes @@ -184,10 +152,13 @@ class Stream: public Print { using oneShotMs = esp8266::polledTimeout::oneShotFastMs; // ::to*() methods: - // - always stop before timeout when "no-more-input-possible-data" or "no-more-output-possible-data" condition is met + // - always stop before timeout when "no-more-input-possible-data" + // or "no-more-output-possible-data" condition is met // - always return number of transfered bytes + // When result is 0 or less than requested maxLen, this->getLastTo() + // contains an error reason. - // transfers already buffered / immediately available data + // transfers already buffered / immediately available data, not timeout // returns number of transfered bytes size_t toNow (Print* to) { return toFull(to, -1, -1, oneShotMs::alwaysExpired); } template @@ -211,14 +182,8 @@ class Stream: public Print { template size_t toSize (T& to, const ssize_t maxLen, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toSize((Print*)&to, maxLen, timeoutMs); } -protected: - - size_t toFull (Print* to, - const ssize_t maxLen = -1, - const int readUntilChar = -1, - oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* =>getTimeout() */); - -public: + // size of input (-1 by default = unknown) + virtual ssize_t streamSize () { return -1; } typedef enum { @@ -232,20 +197,20 @@ class Stream: public Print { toReport_e getLastTo () /*const*/ { return (toReport_e)getWriteError(); } //////////////////// - // size of input - // -1 by default is unknown - // may be used by http streamer (using a SerialStream as a file) - virtual ssize_t streamSize () { return -1; } + protected: + size_t toFull (Print* to, + const ssize_t maxLen = -1, + const int readUntilChar = -1, + oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* neverExpires=>getTimeout() */); //////////////////// end of extensions protected: - long parseInt(char skipChar); // as above but the given skipChar is ignored - // as above but the given skipChar is ignored + long parseInt(char skipChar); // as parseInt() but the given skipChar is ignored // this allows format characters (typically commas) in values to be ignored - float parseFloat(char skipChar); // as above but the given skipChar is ignored + float parseFloat(char skipChar); // as parseFloat() but the given skipChar is ignored }; #endif diff --git a/tests/host/common/include/ClientContext.h b/tests/host/common/include/ClientContext.h index 2dcbeb185d..8d295a9a45 100644 --- a/tests/host/common/include/ClientContext.h +++ b/tests/host/common/include/ClientContext.h @@ -27,7 +27,7 @@ class WiFiClient; extern "C" void esp_yield(); extern "C" void esp_schedule(); -#include +#include bool getDefaultPrivateGlobalSyncValue (); From dfb39cab9baed96645dd0dc1da9d08fd94458869 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 29 Jul 2020 23:59:00 +0200 Subject: [PATCH 106/207] simplify webserver --- .../src/ESP8266WebServer-impl.h | 121 +++++++++++++----- .../ESP8266WebServer/src/ESP8266WebServer.h | 11 +- 2 files changed, 100 insertions(+), 32 deletions(-) diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index 4441dbd146..a40d2b53f0 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -451,6 +451,95 @@ void ESP8266WebServerTemplate::_prepareHeader(String& response, int _responseHeaders = ""; } +#if 1 ////////////////////////////////////////////////////////////////////////////////////////////////// +// new stream::to + +template +void ESP8266WebServerTemplate::send(int code, char* content_type, const String& content) { + return send(code, (const char*)content_type, content); +} + +template +void ESP8266WebServerTemplate::send(int code, const char* content_type, const String& content) { + return send(code, content_type, content.c_str(), content.length()); +} + +template +void ESP8266WebServerTemplate::send(int code, const String& content_type, const String& content) { + return send(code, (const char*)content_type.c_str(), content); +} + +template +void ESP8266WebServerTemplate::sendContent(const String& content) { + StreamPtr ref(content.c_str(), content.length()); + sendContent(&ref); +} + +template +void ESP8266WebServerTemplate::send(int code, const char* content_type, Stream* stream, size_t content_length /*= 0*/) { + String header; + if (content_length == 0) + content_length = std::max(0, stream->streamSize()); + _prepareHeader(header, code, content_type, content_length); +#if STRING_IS_STREAM + size_t sent = header.toAll(&_currentClient); // with timeout +#else + size_t sent = StreamPtr(header.c_str(), header.length()).toAll(&_currentClient); // with timeout +#endif +#ifdef DEBUG_ESP_HTTP_SERVER + if (sent != content_length) + DEBUG_OUTPUT.printf("HTTPServer: error: sent %zd on %zd bytes\n", sent, content_length); +#else + (void)sent; +#endif + if(content_length) + return sendContent(stream, content_length); +} + +template +void ESP8266WebServerTemplate::send_P(int code, PGM_P content_type, PGM_P content) { + StreamPtr ref(content, strlen_P(content)); + return send(code, String(content_type).c_str(), &ref); +} + +template +void ESP8266WebServerTemplate::send_P(int code, PGM_P content_type, PGM_P content, size_t contentLength) { + StreamPtr ref(content, contentLength); + return send(code, String(content_type).c_str(), &ref); +} + +template +void ESP8266WebServerTemplate::sendContent(Stream* content, ssize_t content_length /* = 0*/) { + if (_currentMethod == HTTP_HEAD) + return; + if (content_length == 0) + content_length = std::max(0, content->streamSize()); + if(_chunked) { + _currentClient.printf("%zx\r\n", content_length); + } + size_t sent = content->toSize(&_currentClient, content_length); + (void)sent; //XXXFIXME if (sent != content_length) print-error-on-console-and-return-false + if(_chunked) { + _currentClient.printf_P(PSTR("\r\n")); + if (content_length == 0) { + _chunked = false; + } + } +} + +template +void ESP8266WebServerTemplate::sendContent_P(PGM_P content) { + sendContent_P(content, strlen_P(content)); +} + +template +void ESP8266WebServerTemplate::sendContent_P(PGM_P content, size_t size) { + StreamPtr ptr(content, size); + return sendContent(&ptr, size); +} + +#else ////////////////////////////////////////////////////////////////////////////////////////////////// + template void ESP8266WebServerTemplate::send(int code, const char* content_type, const String& content) { String header; @@ -534,40 +623,12 @@ void ESP8266WebServerTemplate::sendContent(Stream* content) { } } -#if 1 - template void ESP8266WebServerTemplate::sendContent(const String& content) { StreamPtr ref(content.c_str(), content.length()); return sendContent(&ref); } -//template -//void ESP8266WebServerTemplate::sendContent(String& content) { -// return sendContent(&content); -//} - -#else -template -void ESP8266WebServerTemplate::sendContent(const String& content) { - if (_currentMethod == HTTP_HEAD) return; - const char * footer = "\r\n"; - size_t len = content.length(); - if(_chunked) { - char chunkSize[11]; - sprintf(chunkSize, "%zx\r\n", len); - _currentClient.write((const uint8_t *)chunkSize, strlen(chunkSize)); - } - _currentClient.write((const uint8_t *)content.c_str(), len); - if(_chunked){ - _currentClient.write((const uint8_t *)footer, 2); - if (len == 0) { - _chunked = false; - } - } -} -#endif - template void ESP8266WebServerTemplate::sendContent_P(PGM_P content) { sendContent_P(content, strlen_P(content)); @@ -590,6 +651,8 @@ void ESP8266WebServerTemplate::sendContent_P(PGM_P content, size_t s } } +#endif ////////////////////////////////////////////////////////////////////////////////////////////////// + template String ESP8266WebServerTemplate::credentialHash(const String& username, const String& realm, const String& password) { @@ -752,7 +815,7 @@ void ESP8266WebServerTemplate::_handleRequest() { } if (!handled) { using namespace mime; - send(404, String(FPSTR(mimeTable[html].mimeType)), String(F("Not found: ")) + _currentUri); + send(404, FPSTR(mimeTable[html].mimeType), String(F("Not found: ")) + _currentUri); handled = true; } if (handled) { diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer.h b/libraries/ESP8266WebServer/src/ESP8266WebServer.h index f6d35abfd7..5fa0846367 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer.h @@ -127,7 +127,7 @@ class ESP8266WebServerTemplate // code - HTTP response code, can be 200 or 404 // content_type - HTTP content type, like "text/plain" or "image/png" // content - actual content body - void send(int code, const char* content_type = NULL, const String& content = String("")); + void send(int code, const char* content_type = NULL, const String& content = emptyString); void send(int code, char* content_type, const String& content); void send(int code, const String& content_type, const String& content); void send(int code, const char *content_type, const char *content) { @@ -142,16 +142,21 @@ class ESP8266WebServerTemplate void send_P(int code, PGM_P content_type, PGM_P content); void send_P(int code, PGM_P content_type, PGM_P content, size_t contentLength); + void send(int code, const char* content_type, Stream* stream, size_t content_length = 0); + void send(int code, const char* content_type, Stream& stream, size_t content_length = 0); + void setContentLength(const size_t contentLength); void sendHeader(const String& name, const String& value, bool first = false); - void sendContent(Stream& content); - void sendContent(Stream* content); void sendContent(const String& content); void sendContent_P(PGM_P content); void sendContent_P(PGM_P content, size_t size); void sendContent(const char *content) { sendContent_P(content); } void sendContent(const char *content, size_t size) { sendContent_P(content, size); } + void sendContent(Stream* content, ssize_t content_length = 0); + template + void sendContent(T& content, ssize_t content_length = 0) { sendContent(&content, content_length); } + bool chunkedResponseModeStart_P (int code, PGM_P content_type) { if (_currentVersion == 0) // no chunk mode in HTTP/1.0 From ceaaf1871e82ff367854816d0a5aa312eb31dd96 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Thu, 30 Jul 2020 00:58:55 +0200 Subject: [PATCH 107/207] fix debug message --- libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index a40d2b53f0..911a481611 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -487,8 +487,8 @@ void ESP8266WebServerTemplate::send(int code, const char* content_ty size_t sent = StreamPtr(header.c_str(), header.length()).toAll(&_currentClient); // with timeout #endif #ifdef DEBUG_ESP_HTTP_SERVER - if (sent != content_length) - DEBUG_OUTPUT.printf("HTTPServer: error: sent %zd on %zd bytes\n", sent, content_length); + if (sent != header.length()) + DEBUG_OUTPUT.printf("HTTPServer: error: sent %zd on %zd bytes\n", sent, header.length()); #else (void)sent; #endif From 42ffd6f731ee39dccb529ccf6407b77b43c50132 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Thu, 30 Jul 2020 01:03:40 +0200 Subject: [PATCH 108/207] fix debug msg --- .../ESP8266WebServer/src/ESP8266WebServer-impl.h | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index c10848ed9c..efdee12865 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -494,17 +494,13 @@ void ESP8266WebServerTemplate::send(int code, const char* content_ty content_length = std::max(0, stream->streamSize()); _prepareHeader(header, code, content_type, content_length); #if STRING_IS_STREAM - size_t sent = header.toAll(&_currentClient); // with timeout + size_t sent = header.toAll(&_currentClient); #else - size_t sent = StreamPtr(header.c_str(), header.length()).toAll(&_currentClient); // with timeout + size_t sent = StreamPtr(header.c_str(), header.length()).toAll(&_currentClient); #endif -#ifdef DEBUG_ESP_HTTP_SERVER if (sent != header.length()) - DEBUG_OUTPUT.printf("HTTPServer: error: sent %zd on %zd bytes\n", sent, header.length()); -#else - (void)sent; -#endif - if(content_length) + DBGWS("HTTPServer: error: sent %zd on %zd bytes\n", sent, header.length()); + if (content_length) return sendContent(stream, content_length); } From feaccaa62c9a36ccad3a8718e925ad672e4e133a Mon Sep 17 00:00:00 2001 From: david gauchard Date: Thu, 30 Jul 2020 01:15:17 +0200 Subject: [PATCH 109/207] fix webserver bug --- libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index efdee12865..a89e5b3668 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -367,7 +367,7 @@ void ESP8266WebServerTemplate::handleClient() { break; case HC_WAIT_CLOSE: // Wait for client to close the connection - if (!_server.available() && (millis() - _statusChange <= HTTP_MAX_CLOSE_WAIT)) { + if (!_server.hasClient() && (millis() - _statusChange <= HTTP_MAX_CLOSE_WAIT)) { keepCurrentClient = true; callYield = true; if (_currentClient.available()) From bf9ad754ea0ab57133ac1147fce4d5d6bd2c95f4 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 31 Jul 2020 01:24:14 +0200 Subject: [PATCH 110/207] remove comment --- .../src/ESP8266WebServer-impl.h | 118 ------------------ 1 file changed, 118 deletions(-) diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index a89e5b3668..ccd7c7e047 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -463,9 +463,6 @@ void ESP8266WebServerTemplate::_prepareHeader(String& response, int _responseHeaders = ""; } -#if 1 ////////////////////////////////////////////////////////////////////////////////////////////////// -// new stream::to - template void ESP8266WebServerTemplate::send(int code, char* content_type, const String& content) { return send(code, (const char*)content_type, content); @@ -546,121 +543,6 @@ void ESP8266WebServerTemplate::sendContent_P(PGM_P content, size_t s return sendContent(&ptr, size); } -#else ////////////////////////////////////////////////////////////////////////////////////////////////// - -template -void ESP8266WebServerTemplate::send(int code, const char* content_type, const String& content) { - String header; - // Can we asume the following? - //if(code == 200 && content.length() == 0 && _contentLength == CONTENT_LENGTH_NOT_SET) - // _contentLength = CONTENT_LENGTH_UNKNOWN; - _prepareHeader(header, code, content_type, content.length()); - size_t sent = StreamPtr(header.c_str(), header.length()).toAll(&_currentClient); // transfer all of it, with timeout - (void)sent; -#ifdef DEBUG_ESP_HTTP_SERVER - if (sent != header.length()) - DEBUG_OUTPUT.printf("HTTPServer: error: sent %zd on %zd bytes\n", sent, header.length()); -#endif - - if(content.length()) - sendContent(content); -} - -template -void ESP8266WebServerTemplate::send_P(int code, PGM_P content_type, PGM_P content) { - size_t contentLength = 0; - - if (content != NULL) { - contentLength = strlen_P(content); - } - - String header; - char type[64]; - memccpy_P((void*)type, (PGM_VOID_P)content_type, 0, sizeof(type)); - _prepareHeader(header, code, (const char* )type, contentLength); - _currentClient.write((const uint8_t *)header.c_str(), header.length()); - if (contentLength) { - sendContent_P(content); - } -} - -template -void ESP8266WebServerTemplate::send_P(int code, PGM_P content_type, PGM_P content, size_t contentLength) { - String header; - char type[64]; - memccpy_P((void*)type, (PGM_VOID_P)content_type, 0, sizeof(type)); - _prepareHeader(header, code, (const char* )type, contentLength); - _currentClient.write((const uint8_t *)header.c_str(), header.length()); - if (contentLength) { - sendContent_P(content, contentLength); - } -} - -template -void ESP8266WebServerTemplate::send(int code, char* content_type, const String& content) { - send(code, (const char*)content_type, content); -} - -template -void ESP8266WebServerTemplate::send(int code, const String& content_type, const String& content) { - send(code, (const char*)content_type.c_str(), content); -} - -template -void ESP8266WebServerTemplate::sendContent(Stream& content) { - sendContent(&content); -} - -template -void ESP8266WebServerTemplate::sendContent(Stream* content) { - if (_currentMethod == HTTP_HEAD) return; - const char * footer = "\r\n"; - size_t len = content->streamSize(); - if(_chunked) { - char chunkSize[11]; - sprintf(chunkSize, "%zx\r\n", len); - _currentClient.write((const uint8_t *)chunkSize, strlen(chunkSize)); - } - size_t sent = content->toAll(&_currentClient); - (void)sent; /// if (sent != len) print-error-on-console-and-return-false - if(_chunked){ - _currentClient.write((const uint8_t *)footer, 2); - if (len == 0) { - _chunked = false; - } - } -} - -template -void ESP8266WebServerTemplate::sendContent(const String& content) { - StreamPtr ref(content.c_str(), content.length()); - return sendContent(&ref); -} - -template -void ESP8266WebServerTemplate::sendContent_P(PGM_P content) { - sendContent_P(content, strlen_P(content)); -} - -template -void ESP8266WebServerTemplate::sendContent_P(PGM_P content, size_t size) { - const char * footer = "\r\n"; - if(_chunked) { - char chunkSize[11]; - sprintf(chunkSize, "%zx\r\n", size); - _currentClient.write((const uint8_t *)chunkSize, strlen(chunkSize)); - } - _currentClient.write_P(content, size); - if(_chunked){ - _currentClient.write((const uint8_t *)footer, 2); - if (size == 0) { - _chunked = false; - } - } -} - -#endif ////////////////////////////////////////////////////////////////////////////////////////////////// - template String ESP8266WebServerTemplate::credentialHash(const String& username, const String& realm, const String& password) { From cdd72f9a2443d9ada9efb295f429a20c74e42206 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 2 Aug 2020 23:45:19 +0200 Subject: [PATCH 111/207] define indicating a breaking change --- cores/esp8266/Stream.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 128b0351fb..f8549d85ae 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -38,6 +38,12 @@ readBytesBetween( pre_string, terminator, buffer, length) */ +// Arduino `Client: public Stream` class defines `virtual int read(uint8_t *buf, size_t size) = 0;` +// This function is now imported into `Stream::` for `Stream::to*()`. +// Other classes inheriting from `Stream::` and implementing `read(uint8_t *buf, size_t size)` +// must consequently use `int` as return type, namely Hardware/SoftwareSerial, FileSystems... +#define STREAM_READ_RETURNS_INT 1 + class Stream: public Print { protected: unsigned long _timeout; // number of milliseconds to wait for the next char before aborting timed read From 3350923e3712dc212eecc280b086f9e8ea816b4f Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 7 Aug 2020 16:23:20 +0200 Subject: [PATCH 112/207] temporarily use an updated softwareserial --- libraries/SoftwareSerial | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/SoftwareSerial b/libraries/SoftwareSerial index e51872ca20..dbdb5883a3 160000 --- a/libraries/SoftwareSerial +++ b/libraries/SoftwareSerial @@ -1 +1 @@ -Subproject commit e51872ca201d307a57ca03cbce4d18497c9daef3 +Subproject commit dbdb5883a33528c3cffd7121bad4c18eb2519f35 From f0483e102c1b6c90e101870578bdee33af5e42ff Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 7 Aug 2020 17:08:06 +0200 Subject: [PATCH 113/207] CI: on host: force 64 bit mode like `make -j2 CI` implicitly does It allows to run CI tests on computers having gcc-multilib installed --- tests/ci/host_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/host_test.sh b/tests/ci/host_test.sh index 9c9be68761..ef143f90e1 100755 --- a/tests/ci/host_test.sh +++ b/tests/ci/host_test.sh @@ -7,14 +7,14 @@ set -ev cd $TRAVIS_BUILD_DIR/tests/host -make -j2 ssl +make -j2 FORCE32=0 ssl for i in ../../libraries/ESP8266WiFi/examples/WiFiClient/WiFiClient \ ../../libraries/ESP8266WiFi/examples/BearSSL_Validation/BearSSL_Validation \ ../../libraries/ESP8266WebServer/examples/HelloServer/HelloServer \ ../../libraries/SD/examples/Files/Files \ ../../libraries/LittleFS/examples/LittleFS_Timestamp/LittleFS_Timestamp \ ../../libraries/LittleFS/examples/SpeedTest/SpeedTest ; do - make -j2 D=1 $i + make -j2 D=1 FORCE32=0 $i valgrind --leak-check=full --track-origins=yes --error-limit=no --show-leak-kinds=all --error-exitcode=999 bin/$(basename $i)/$(basename $i) -1 done From f290fd585748fc1a50fa58a26d4e9a947ee50a93 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 8 Aug 2020 08:32:11 +0200 Subject: [PATCH 114/207] fix CI --- libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h | 6 +++--- tests/host/Makefile | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index ccd7c7e047..d312b9e09e 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -488,7 +488,7 @@ template void ESP8266WebServerTemplate::send(int code, const char* content_type, Stream* stream, size_t content_length /*= 0*/) { String header; if (content_length == 0) - content_length = std::max(0, stream->streamSize()); + content_length = std::max((ssize_t)0, stream->streamSize()); _prepareHeader(header, code, content_type, content_length); #if STRING_IS_STREAM size_t sent = header.toAll(&_currentClient); @@ -496,7 +496,7 @@ void ESP8266WebServerTemplate::send(int code, const char* content_ty size_t sent = StreamPtr(header.c_str(), header.length()).toAll(&_currentClient); #endif if (sent != header.length()) - DBGWS("HTTPServer: error: sent %zd on %zd bytes\n", sent, header.length()); + DBGWS("HTTPServer: error: sent %zd on %u bytes\n", sent, header.length()); if (content_length) return sendContent(stream, content_length); } @@ -518,7 +518,7 @@ void ESP8266WebServerTemplate::sendContent(Stream* content, ssize_t if (_currentMethod == HTTP_HEAD) return; if (content_length == 0) - content_length = std::max(0, content->streamSize()); + content_length = std::max((ssize_t)0, content->streamSize()); if(_chunked) { _currentClient.printf("%zx\r\n", content_length); } diff --git a/tests/host/Makefile b/tests/host/Makefile index e3312e6b35..cf0ed4fe9f 100644 --- a/tests/host/Makefile +++ b/tests/host/Makefile @@ -53,6 +53,7 @@ endif $(shell mkdir -p $(BINDIR)) CORE_CPP_FILES := $(addprefix $(CORE_PATH)/,\ + debug.cpp \ StreamDev.cpp \ Stream.cpp \ WString.cpp \ @@ -236,7 +237,7 @@ build-info: # show toolchain version $(BINDIR)/core.a: $(C_OBJECTS) $(CPP_OBJECTS_CORE) ar -rcu $@ $^ - ranlib -c $@ + ranlib $@ $(OUTPUT_BINARY): $(CPP_OBJECTS_TESTS) $(BINDIR)/core.a $(VERBLD) $(CXX) $(DEFSYM_FS) $(LDFLAGS) $^ -o $@ @@ -249,7 +250,6 @@ ARDUINO_LIBS := \ IPAddress.cpp \ Updater.cpp \ base64.cpp \ - debug.cpp \ ) \ $(addprefix ../../libraries/ESP8266WiFi/src/,\ ESP8266WiFi.cpp \ @@ -323,7 +323,7 @@ CPP_OBJECTS_CORE_EMU = $(CPP_SOURCES_CORE_EMU:.cpp=.cpp$(E32).o) $(USERLIBSRCS:. bin/fullcore.a: $(C_OBJECTS) $(CPP_OBJECTS_CORE_EMU) $(VERBAR) ar -rcu $@ $^ - $(VERBAR) ranlib -c $@ + $(VERBAR) ranlib $@ %: %.ino.cpp$(E32).o bin/fullcore.a $(VERBLD) $(CXX) $(LDFLAGS) $< bin/fullcore.a $(LIBSSL) -o $@ From 55bb160a8bdb050145bf88b7224b458616c43eb5 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 8 Aug 2020 23:22:32 +0200 Subject: [PATCH 115/207] stick to arduino, per review --- cores/esp8266/Print.cpp | 5 ----- cores/esp8266/Print.h | 5 +++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/cores/esp8266/Print.cpp b/cores/esp8266/Print.cpp index 1c2bc7a94f..c8e0214195 100644 --- a/cores/esp8266/Print.cpp +++ b/cores/esp8266/Print.cpp @@ -263,8 +263,3 @@ size_t Print::printFloat(double number, uint8_t digits) { char buf[40]; return write(dtostrf(number, 0, digits, buf)); } - -int Print::availableForWrite() { - IAMSLOW(); - return 1; -} diff --git a/cores/esp8266/Print.h b/cores/esp8266/Print.h index 536ecdc944..36e0245848 100644 --- a/cores/esp8266/Print.h +++ b/cores/esp8266/Print.h @@ -107,8 +107,9 @@ class Print { virtual void flush() { /* Empty implementation for backward compatibility */ } - virtual int availableForWrite(); - // return type: int + // default to zero, meaning "a single write may block" + // should be overriden by subclasses with buffering + virtual int availableForWrite() { return 0; } // by default write timeout is possible (outgoing data from network,serial..) // (children can override to false (like String)) From ce3c92250ed6d1366df3ba47402b66881f1afd36 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 8 Aug 2020 23:23:29 +0200 Subject: [PATCH 116/207] python echo client --- .../examples/WiFiEcho/echo-client.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100755 libraries/ESP8266WiFi/examples/WiFiEcho/echo-client.py diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/echo-client.py b/libraries/ESP8266WiFi/examples/WiFiEcho/echo-client.py new file mode 100755 index 0000000000..b6ab5c9cd3 --- /dev/null +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/echo-client.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +import os +import asyncio + +# 512 bytes +message = 'abcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnop'.encode('utf8') #bytearray(os.urandom(bufsize)) +bufsize=len(message) +print('message len=', bufsize) + +global recv +recv = 0 + +async def tcp_echo_open (ip, port, loop): + return await asyncio.open_connection(ip, port, loop=loop) + +async def tcp_echo_sender(message, writer, loop): + print('Writer started') + while True: + writer.write(message) + await writer.drain() + +async def tcp_echo_receiver(message, reader, loop): + global recv + print('Reader started') + while True: + data = ''.encode('utf8') + while len(data) < bufsize: + data += await reader.read(bufsize - len(data)) + recv += len(data); + if data != message: + print('error') + +async def tcp_stat(loop): + global recv + while True: + last = recv + await asyncio.sleep(2) + print('BW=', (recv - last) * 8 / 1024, 'Kibits/s') + +loop = asyncio.get_event_loop() +reader, writer = loop.run_until_complete(tcp_echo_open('echo23.local', 23, loop)) +loop.create_task(tcp_echo_receiver(message, reader,loop)) +loop.create_task(tcp_echo_sender(message, writer, loop)) +loop.create_task(tcp_stat(loop)) +loop.run_forever() From bc29fe6a62c1281486df6135db87da8dcd768f0f Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 8 Aug 2020 23:24:01 +0200 Subject: [PATCH 117/207] improve echo server --- .../examples/WiFiEcho/WiFiEcho.ino | 53 +++++++++++++++---- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino index eb66c49d46..05bc31f9e3 100644 --- a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino @@ -5,6 +5,7 @@ */ #include +#include #include #include // std::min @@ -18,10 +19,12 @@ constexpr int port = 23; WiFiServer server(port); WiFiClient client; -constexpr size_t stackProtector = 128; +constexpr size_t sizes [] = { 0, 512, 384, 256, 128, 64, 16, 8, 4 }; constexpr uint32_t breathMs = 200; esp8266::polledTimeout::oneShotFastMs enoughMs(breathMs); +esp8266::polledTimeout::periodicFastMs test(2000); int t = 1; // test (1, 2 or 3, see below) +int s = 0; // sizes[] index void setup() { @@ -40,18 +43,35 @@ void setup() { Serial.print("connected, address="); Serial.println(WiFi.localIP()); - //start server server.begin(); - //server.setNoDelay(true); - Serial.print("Ready! Use 'telnet "); + MDNS.begin("echo23"); + + Serial.print("Ready! Use 'telnet/nc "); Serial.print(WiFi.localIP()); Serial.printf(" %d' to try echo, use a bandwidth meter and try typing 1, 2 or 3 on console during transfer\n", port); - + Serial.printf("Use 'python3 echo-client.py' to measure bandwidth and compare algorithms\n"); } + void loop() { + MDNS.update(); + + static uint32_t tot = 0; + static uint32_t cnt = 0; + if (test && cnt) + { + Serial.printf("measured-block-size=%u min-free-stack=%u", tot/cnt, ESP.getFreeContStack()); + if (t == 2 && sizes[s]) + Serial.printf(" (blocks: at most %d bytes)", sizes[s]); + if (t == 3 && sizes[s]) + Serial.printf(" (blocks: exactly %d bytes)", sizes[s]); + if (t == 3 && !sizes[s]) + Serial.printf(" (blocks: any size)"); + Serial.printf("\n"); + } + //check if there are any new clients if (server.hasClient()) { client = server.available(); @@ -59,11 +79,16 @@ void loop() { } if (Serial.available()) + { + s = (s + 1) % (sizeof(sizes) / sizeof(sizes[0])); switch (Serial.read()) { - case '1': t = 1; Serial.println("byte-by-byte"); break; - case '2': t = 2; Serial.println("through buffer"); break; - case '3': t = 3; Serial.println("direct access"); break; + case '1': if (t != 1) s = 0; t = 1; Serial.println("byte-by-byte"); break; + case '2': if (t != 2) s = 1; t = 2; Serial.printf("through buffer\n"); break; + case '3': if (t != 3) s = 0; t = 3; Serial.printf("direct access\n"); break; } + tot = cnt = 0; + ESP.resetFreeContStack(); + } enoughMs.reset(breathMs); @@ -72,6 +97,8 @@ void loop() { while (client.available() && client.availableForWrite() && !enoughMs) { // working char by char is not efficient client.write(client.read()); + cnt++; + tot+=1; } } @@ -79,19 +106,25 @@ void loop() { // block by block through a local buffer (2 copies) while (client.available() && client.availableForWrite() && !enoughMs) { size_t maxTo = std::min(client.available(), client.availableForWrite()); - maxTo = std::min(maxTo, stackProtector); + maxTo = std::min(maxTo, sizes[s]); uint8_t buf[maxTo]; size_t tcp_got = client.read(buf, maxTo); size_t tcp_sent = client.write(buf, tcp_got); if (tcp_sent != maxTo) { Serial.printf("len mismatch: available:%zd tcp-read:%zd serial-write:%zd\n", maxTo, tcp_got, tcp_sent); } + tot += tcp_sent; + cnt++; } } else if (t == 3) { // stream to print, possibly with only one copy - client.toNow(&client); + if (sizes[s]) + tot += client.toSize(&client, sizes[s]); + else + tot += client.toAll(&client); + cnt++; switch (client.getLastTo()) { case Stream::STREAMTO_SUCCESS: break; From dd947c8abd2fddbe046db788164dab9604af4aaf Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 8 Aug 2020 23:24:24 +0200 Subject: [PATCH 118/207] mock: hide debug message --- tests/host/common/ClientContextSocket.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/host/common/ClientContextSocket.cpp b/tests/host/common/ClientContextSocket.cpp index b76c9a2a91..3ff7293540 100644 --- a/tests/host/common/ClientContextSocket.cpp +++ b/tests/host/common/ClientContextSocket.cpp @@ -200,7 +200,7 @@ ssize_t mockWrite (int sock, const uint8_t* data, size_t size, int timeout_ms) } } #ifdef DEBUG_ESP_WIFI - fprintf(stderr, MOCK "ClientContext::write: total sent %zd bytes\n", sent); + mockverbose(MOCK "ClientContext::write: total sent %zd bytes\n", sent); #endif return sent; } From 81f2d6f8f283545093831007a66a2a5a18058ccc Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 8 Aug 2020 23:26:59 +0200 Subject: [PATCH 119/207] style --- .../examples/WiFiEcho/WiFiEcho.ino | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino index 05bc31f9e3..4953ea0cad 100644 --- a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino @@ -60,15 +60,17 @@ void loop() { static uint32_t tot = 0; static uint32_t cnt = 0; - if (test && cnt) - { - Serial.printf("measured-block-size=%u min-free-stack=%u", tot/cnt, ESP.getFreeContStack()); - if (t == 2 && sizes[s]) - Serial.printf(" (blocks: at most %d bytes)", sizes[s]); - if (t == 3 && sizes[s]) - Serial.printf(" (blocks: exactly %d bytes)", sizes[s]); - if (t == 3 && !sizes[s]) - Serial.printf(" (blocks: any size)"); + if (test && cnt) { + Serial.printf("measured-block-size=%u min-free-stack=%u", tot / cnt, ESP.getFreeContStack()); + if (t == 2 && sizes[s]) { + Serial.printf(" (blocks: at most %d bytes)", sizes[s]); + } + if (t == 3 && sizes[s]) { + Serial.printf(" (blocks: exactly %d bytes)", sizes[s]); + } + if (t == 3 && !sizes[s]) { + Serial.printf(" (blocks: any size)"); + } Serial.printf("\n"); } @@ -78,8 +80,7 @@ void loop() { Serial.println("New client"); } - if (Serial.available()) - { + if (Serial.available()) { s = (s + 1) % (sizeof(sizes) / sizeof(sizes[0])); switch (Serial.read()) { case '1': if (t != 1) s = 0; t = 1; Serial.println("byte-by-byte"); break; @@ -98,7 +99,7 @@ void loop() { // working char by char is not efficient client.write(client.read()); cnt++; - tot+=1; + tot += 1; } } @@ -120,10 +121,11 @@ void loop() { else if (t == 3) { // stream to print, possibly with only one copy - if (sizes[s]) + if (sizes[s]) { tot += client.toSize(&client, sizes[s]); - else + } else { tot += client.toAll(&client); + } cnt++; switch (client.getLastTo()) { From f85ddbb30ca485133bd67204064f72e2a0d6de64 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 9 Aug 2020 12:24:05 +0200 Subject: [PATCH 120/207] improve python echo client --- .../ESP8266WiFi/examples/WiFiEcho/echo-client.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/echo-client.py b/libraries/ESP8266WiFi/examples/WiFiEcho/echo-client.py index b6ab5c9cd3..3e78b6af37 100755 --- a/libraries/ESP8266WiFi/examples/WiFiEcho/echo-client.py +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/echo-client.py @@ -4,7 +4,7 @@ import asyncio # 512 bytes -message = 'abcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnopabcdefghijklmnop'.encode('utf8') #bytearray(os.urandom(bufsize)) +message = bytearray(512); bufsize=len(message) print('message len=', bufsize) @@ -33,14 +33,17 @@ async def tcp_echo_receiver(message, reader, loop): async def tcp_stat(loop): global recv + dur = 0 + loopsec = 2 while True: last = recv - await asyncio.sleep(2) - print('BW=', (recv - last) * 8 / 1024, 'Kibits/s') + await asyncio.sleep(loopsec) # drifting + dur += loopsec + print('BW=', (recv - last) * 2 * 8 / 1024 / loopsec, 'Kibits/s avg=', recv * 2 * 8 / 1024 / dur) loop = asyncio.get_event_loop() reader, writer = loop.run_until_complete(tcp_echo_open('echo23.local', 23, loop)) -loop.create_task(tcp_echo_receiver(message, reader,loop)) +loop.create_task(tcp_echo_receiver(message, reader, loop)) loop.create_task(tcp_echo_sender(message, writer, loop)) loop.create_task(tcp_stat(loop)) loop.run_forever() From 1fd80d1150ee63fa6f25eb5aac7a9f4e29d5d844 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 9 Aug 2020 12:55:09 +0200 Subject: [PATCH 121/207] httpclient: fix when string is not stream --- libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 471ad60f8b..45aef4b8af 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -1104,7 +1104,11 @@ bool HTTPClient::sendHeader(const char * type) DEBUG_HTTPCLIENT("[HTTP-Client] sending request header\n-----\n%s-----\n", header.c_str()); // transfer all of it, with timeout +#if STRING_IS_STREAM return header.toSize(_client, header.length()) == header.length(); +#else + return (_client->write((const uint8_t *) header.c_str(), header.length()) == header.length()); +#endif } /** From e96f80d1797f04064176b4566e44b7908b37bf10 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 9 Aug 2020 13:02:17 +0200 Subject: [PATCH 122/207] restore streamstring.cpp when string is not stream --- cores/esp8266/StreamString.cpp | 71 ++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 cores/esp8266/StreamString.cpp diff --git a/cores/esp8266/StreamString.cpp b/cores/esp8266/StreamString.cpp new file mode 100644 index 0000000000..0f79eb427c --- /dev/null +++ b/cores/esp8266/StreamString.cpp @@ -0,0 +1,71 @@ +/** + StreamString.cpp + + Copyright (c) 2015 Markus Sattler. All rights reserved. + This file is part of the esp8266 core for Arduino environment. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + */ + + +#include "StreamString.h" + +#if !STRING_IS_STREAM + +size_t StreamString::write(const uint8_t *data, size_t size) { + if(size && data) { + const unsigned int newlen = length() + size; + if(reserve(newlen + 1)) { + memcpy((void *) (wbuffer() + len()), (const void *) data, size); + setLen(newlen); + *(wbuffer() + newlen) = 0x00; // add null for string end + return size; + } + DEBUGV(":stream2string: OOM (%d->%d)\n", length(), newlen+1); + } + return 0; +} + +size_t StreamString::write(uint8_t data) { + return concat((char) data); +} + +int StreamString::available() { + return length(); +} + +int StreamString::read() { + if(length()) { + char c = charAt(0); + remove(0, 1); + return c; + + } + return -1; +} + +int StreamString::peek() { + if(length()) { + char c = charAt(0); + return c; + } + return -1; +} + +void StreamString::flush() { +} + +#endif // !STRING_IS_STREAM From 10e86ad258820f287ab5ff4dadf539ecaca38d36 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 9 Aug 2020 13:25:07 +0200 Subject: [PATCH 123/207] missing overrides --- cores/esp8266/Stream.h | 2 +- cores/esp8266/StreamDev.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index f8549d85ae..d2e962bbc8 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -164,7 +164,7 @@ class Stream: public Print { // When result is 0 or less than requested maxLen, this->getLastTo() // contains an error reason. - // transfers already buffered / immediately available data, not timeout + // transfers already buffered / immediately available data (no timeout) // returns number of transfered bytes size_t toNow (Print* to) { return toFull(to, -1, -1, oneShotMs::alwaysExpired); } template diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 6ea0427f82..d7da1e5355 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -116,8 +116,8 @@ class StreamPtr: public StreamNull // peekBuffer virtual bool peekBufferAPI () const override { return !_in_flash; } virtual size_t availableForPeek () override { return _peekPointer < _size? _size - _peekPointer: 0; } - virtual const char* peekBuffer () { return _peekPointer < _size? _buffer + _peekPointer: nullptr; } - virtual void peekConsume (size_t consume) { _peekPointer += consume; } + virtual const char* peekBuffer () override { return _peekPointer < _size? _buffer + _peekPointer: nullptr; } + virtual void peekConsume (size_t consume) override { _peekPointer += consume; } }; /////////////////////////////////////////////// From 915647a702024c8241c3cb94aeb0c3cb3b703977 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 10 Aug 2020 17:59:36 +0200 Subject: [PATCH 124/207] ::readBytesUntil() swallows but not transfers the delimiter: fix ::toUntil() (that was transfering it) --- cores/esp8266/Stream.h | 2 +- cores/esp8266/StreamDev.cpp | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index d2e962bbc8..8ee466e8a3 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -176,7 +176,7 @@ class Stream: public Print { template size_t toAll (T& to, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toAll((Print*)&to, timeoutMs); } - // transfers data until a char is encountered (the char is also transfered) with timeout + // transfers data until a char is encountered (the char is swallowed but not transfered) with timeout // returns number of transfered bytes size_t toUntil (Print* to, const int readUntilChar, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toFull(to, -1, readUntilChar, timeoutMs); } template diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index 8cf1ae7a4c..6718ec9991 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -84,7 +84,7 @@ size_t Stream::toFull (Print* to, const char* last = (const char*)memchr(directbuf, readUntilChar, w); if (last) { - w = std::min((size_t)(last - directbuf + 1), w); + w = std::min((size_t)(last - directbuf), w); foundChar = true; } } @@ -94,8 +94,11 @@ size_t Stream::toFull (Print* to, written += w; if (maxLen) timedOut.reset(); - if (foundChar) - break; + } + if (foundChar) + { + peekConsume(1); + break; } } @@ -133,6 +136,8 @@ size_t Stream::toFull (Print* to, int c = read(); if (c != -1) { + if (c == readUntilChar) + break; w = to->write(c); if (w != 1) { @@ -142,8 +147,6 @@ size_t Stream::toFull (Print* to, written += 1; if (maxLen) timedOut.reset(); - if (c == readUntilChar) - break; } if (!w && !maxLen && readUntilChar < 0) From a2cb9b16258fa01dc36362c38fea1ee677d06378 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 14 Aug 2020 22:48:50 +0200 Subject: [PATCH 125/207] rework StreamString --- cores/esp8266/StreamDev.cpp | 86 ++++-- cores/esp8266/StreamDev.h | 247 ++++++++++++++---- cores/esp8266/StreamString.cpp | 71 ----- cores/esp8266/StreamString.h | 208 +++++++++++++-- .../src/ESP8266HTTPClient.cpp | 4 +- .../src/ESP8266WebServer-impl.h | 2 +- libraries/ESP8266WebServer/src/Parsing-impl.h | 17 +- tests/restyle.sh | 1 + 8 files changed, 446 insertions(+), 190 deletions(-) delete mode 100644 cores/esp8266/StreamString.cpp diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index 6718ec9991..5f66818af7 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -1,22 +1,22 @@ /* - StreamDev.cpp - 1-copy transfer function - Copyright (c) 2019 David Gauchard. All right reserved. + StreamDev.cpp - 1-copy transfer function + Copyright (c) 2019 David Gauchard. All right reserved. - This library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. - You should have received a copy of the GNU Lesser General Public - License along with this library; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - parsing functions based on TextFinder library by Michael Margolis + parsing functions based on TextFinder library by Michael Margolis */ @@ -26,15 +26,17 @@ using esp8266::polledTimeout::oneShotFastMs; using esp8266::polledTimeout::periodicFastMs; -size_t Stream::toFull (Print* to, - const ssize_t len, - const int readUntilChar, - const oneShotFastMs::timeType timeoutMs) +size_t Stream::toFull(Print* to, + const ssize_t len, + const int readUntilChar, + const oneShotFastMs::timeType timeoutMs) { setWriteError(STREAMTO_SUCCESS); if (len == 0) - return 0; // conveniently avoids timeout for no requested data + { + return 0; // conveniently avoids timeout for no requested data + } // There are two timeouts: // - read (network, serial, ...) @@ -46,7 +48,7 @@ size_t Stream::toFull (Print* to, // (also when inputTimeoutPossible() is false) // "neverExpires (default, impossible)" is translated to default timeout - oneShotFastMs timedOut(timeoutMs >= oneShotFastMs::neverExpires? getTimeout(): timeoutMs); + oneShotFastMs timedOut(timeoutMs >= oneShotFastMs::neverExpires ? getTimeout() : timeoutMs); // yield about every 5ms (XXX SHOULD BE A SYSTEM-WIDE CONSTANT?) periodicFastMs yieldNow(5); @@ -65,16 +67,22 @@ size_t Stream::toFull (Print* to, size_t avpk = availableForPeek(); if (avpk == 0 && !inputTimeoutPossible()) // no more data to read, ever + { break; + } size_t w = to->availableForWrite(); if (w == 0 && !outputTimeoutPossible()) // no more data can be written, ever + { break; + } w = std::min(w, avpk); if (maxLen) + { w = std::min(w, maxLen - written); + } if (w) { const char* directbuf = peekBuffer(); @@ -93,7 +101,9 @@ size_t Stream::toFull (Print* to, peekConsume(w); written += w; if (maxLen) + { timedOut.reset(); + } } if (foundChar) { @@ -104,16 +114,22 @@ size_t Stream::toFull (Print* to, if (!w && !maxLen && readUntilChar < 0) // nothing has been transferred and no specific condition is requested + { break; + } if (timedOut) // either (maxLen>0) nothing has been transferred for too long // or readUntilChar >= 0 but char is not encountered for too long // or (maxLen=0) too much time has been spent here + { break; + } if (yieldNow) + { yield(); + } } else if (readUntilChar >= 0) @@ -126,18 +142,24 @@ size_t Stream::toFull (Print* to, size_t avpk = availableForPeek(); if (avpk == 0 && !inputTimeoutPossible()) // no more data to read, ever + { break; + } size_t w = to->availableForWrite(); if (w == 0 && !outputTimeoutPossible()) // no more data can be written, ever + { break; + } int c = read(); if (c != -1) { if (c == readUntilChar) + { break; + } w = to->write(c); if (w != 1) { @@ -146,21 +168,29 @@ size_t Stream::toFull (Print* to, } written += 1; if (maxLen) + { timedOut.reset(); + } } if (!w && !maxLen && readUntilChar < 0) // nothing has been transferred and no specific condition is requested + { break; + } if (timedOut) // either (maxLen>0) nothing has been transferred for too long // or readUntilChar >= 0 but char is not encountered for too long // or (maxLen=0) too much time has been spent here + { break; + } if (yieldNow) + { yield(); + } } else @@ -173,12 +203,16 @@ size_t Stream::toFull (Print* to, size_t avr = available(); if (avr == 0 && !inputTimeoutPossible()) // no more data to read, ever + { break; + } size_t w = to->availableForWrite(); if (w == 0 && !to->outputTimeoutPossible()) // no more data can be written, ever + { break; + } w = std::min(w, avr); w = std::min(w, (decltype(w))64); //XXX FIXME 64 is a constant @@ -199,27 +233,37 @@ size_t Stream::toFull (Print* to, break; } if (maxLen && w) + { timedOut.reset(); + } } if (!w && !maxLen && readUntilChar < 0) // nothing has been transferred and no specific condition is requested + { break; + } if (timedOut) // either (maxLen>0) nothing has been transferred for too long // or readUntilChar >= 0 but char is not encountered for too long // or (maxLen=0) too much time has been spent here + { break; + } if (yieldNow) + { yield(); + } } if (getWriteError() == STREAMTO_SUCCESS && maxLen > 0) { if (timeoutMs && timedOut) + { setWriteError(STREAMTO_TIMED_OUT); + } else if ((ssize_t)written != len) // This is happening when source cannot timeout (ex: a String) // but has not enough data, or a dest has closed or cannot @@ -227,7 +271,9 @@ size_t Stream::toFull (Print* to, // // Mark it as an error because user usually wants to get what is // asked for. + { setWriteError(STREAMTO_SHORT); + } } return written; } diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index d7da1e5355..3dc2d6a288 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -1,22 +1,22 @@ /* - StreamDev.h - Stream helpers - Copyright (c) 2019 David Gauchard. All right reserved. + StreamDev.h - Stream helpers + Copyright (c) 2019 David Gauchard. All right reserved. - This library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. - You should have received a copy of the GNU Lesser General Public - License along with this library; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - parsing functions based on TextFinder library by Michael Margolis + parsing functions based on TextFinder library by Michael Margolis */ #ifndef __STREAMDEV_H @@ -34,20 +34,59 @@ class StreamNull: public Stream public: // Print - virtual size_t write(uint8_t) override { return 1; } - virtual size_t write(const uint8_t* buffer, size_t size) override { (void)buffer; (void)size; return size; } - virtual int availableForWrite() override { return 32767; } + virtual size_t write(uint8_t) override + { + return 1; + } + virtual size_t write(const uint8_t* buffer, size_t size) override + { + (void)buffer; + (void)size; + return size; + } + virtual int availableForWrite() override + { + return 32767; + } // Stream - virtual int available() override { return 0; } - virtual int read() override { return -1; } - virtual int peek() override { return -1; } - virtual size_t readBytes(char* buffer, size_t len) override { (void)buffer; (void)len; return 0; } - virtual int read(uint8_t* buffer, size_t len) override { (void)buffer; (void)len; return 0; } - virtual bool outputTimeoutPossible () override { return false; } - virtual bool inputTimeoutPossible () override { return false; } - - virtual ssize_t streamSize () override { return 0; } + virtual int available() override + { + return 0; + } + virtual int read() override + { + return -1; + } + virtual int peek() override + { + return -1; + } + virtual size_t readBytes(char* buffer, size_t len) override + { + (void)buffer; + (void)len; + return 0; + } + virtual int read(uint8_t* buffer, size_t len) override + { + (void)buffer; + (void)len; + return 0; + } + virtual bool outputTimeoutPossible() override + { + return false; + } + virtual bool inputTimeoutPossible() override + { + return false; + } + + virtual ssize_t streamSize() override + { + return 0; + } }; /////////////////////////////////////////////// @@ -60,16 +99,36 @@ class StreamZero: public StreamNull protected: char _x; public: - StreamZero (char x = 0): _x(x) { } + StreamZero(char x = 0): _x(x) { } // Stream - virtual int available() override { return 32767; } - virtual int read() override { return _x; } - virtual int peek() override { return _x; } - virtual size_t readBytes(char* buffer, size_t len) override { memset(buffer, _x, len); return len; } - virtual int read(uint8_t* buffer, size_t len) override { memset((char*)buffer, _x, len); return len; } + virtual int available() override + { + return 32767; + } + virtual int read() override + { + return _x; + } + virtual int peek() override + { + return _x; + } + virtual size_t readBytes(char* buffer, size_t len) override + { + memset(buffer, _x, len); + return len; + } + virtual int read(uint8_t* buffer, size_t len) override + { + memset((char*)buffer, _x, len); + return len; + } - virtual ssize_t streamSize () override { return 32767; } + virtual ssize_t streamSize() override + { + return 32767; + } }; /////////////////////////////////////////////// @@ -86,38 +145,75 @@ class StreamPtr: public StreamNull size_t _peekPointer = 0; public: - StreamPtr (const char* buffer, size_t size, bool in_flash = false): _buffer(buffer), _size(size), _in_flash(in_flash) { } - StreamPtr (const uint8_t* buffer, size_t size, bool in_flash = false): _buffer((const char*)buffer), _size(size), _in_flash(in_flash) { } - StreamPtr (const __FlashStringHelper* buffer, size_t size): _buffer(reinterpret_cast(buffer)), _size(size), _in_flash(true) { } - StreamPtr (const __FlashStringHelper* buffer): _buffer(reinterpret_cast(buffer)), _size(strlen_P(_buffer)), _in_flash(true) { } + StreamPtr(const String& string): _buffer(string.c_str()), _size(string.length()), _in_flash(false) { } + StreamPtr(const char* buffer, size_t size, bool in_flash = false): _buffer(buffer), _size(size), _in_flash(in_flash) { } + StreamPtr(const uint8_t* buffer, size_t size, bool in_flash = false): _buffer((const char*)buffer), _size(size), _in_flash(in_flash) { } + StreamPtr(const __FlashStringHelper* buffer, size_t size): _buffer(reinterpret_cast(buffer)), _size(size), _in_flash(true) { } + StreamPtr(const __FlashStringHelper* buffer): _buffer(reinterpret_cast(buffer)), _size(strlen_P(_buffer)), _in_flash(true) { } - void peekPointerReset (int pointer = 0) { _peekPointer = pointer; } + void peekPointerReset(int pointer = 0) + { + _peekPointer = pointer; + } // Stream - virtual int available() override { return availableForPeek(); } - virtual int read() override { return _peekPointer < _size? _buffer[_peekPointer++]: -1; } - virtual int peek() override { return _peekPointer < _size? _buffer[_peekPointer]: -1; } + virtual int available() override + { + return availableForPeek(); + } + virtual int read() override + { + return _peekPointer < _size ? _buffer[_peekPointer++] : -1; + } + virtual int peek() override + { + return _peekPointer < _size ? _buffer[_peekPointer] : -1; + } virtual size_t readBytes(char* buffer, size_t len) override { if (_peekPointer >= _size) + { return 0; + } size_t cpylen = std::min(_size - _peekPointer, len); if (_in_flash) + { memcpy_P(buffer, _buffer + _peekPointer, cpylen); + } else + { memcpy(buffer, _buffer + _peekPointer, cpylen); + } _peekPointer += cpylen; return cpylen; } - virtual int read(uint8_t* buffer, size_t len) override { return readBytes((char*)buffer, len); } + virtual int read(uint8_t* buffer, size_t len) override + { + return readBytes((char*)buffer, len); + } - virtual ssize_t streamSize () override { return _size; } + virtual ssize_t streamSize() override + { + return _size; + } // peekBuffer - virtual bool peekBufferAPI () const override { return !_in_flash; } - virtual size_t availableForPeek () override { return _peekPointer < _size? _size - _peekPointer: 0; } - virtual const char* peekBuffer () override { return _peekPointer < _size? _buffer + _peekPointer: nullptr; } - virtual void peekConsume (size_t consume) override { _peekPointer += consume; } + virtual bool peekBufferAPI() const override + { + return !_in_flash; + } + virtual size_t availableForPeek() override + { + return _peekPointer < _size ? _size - _peekPointer : 0; + } + virtual const char* peekBuffer() override + { + return _peekPointer < _size ? _buffer + _peekPointer : nullptr; + } + virtual void peekConsume(size_t consume) override + { + _peekPointer += consume; + } }; /////////////////////////////////////////////// @@ -137,24 +233,41 @@ class SerialStreamArray: public Stream public: // not writable - virtual size_t write(uint8_t) override { return 0; } - virtual size_t write(const uint8_t* buffer, size_t size) override { (void)buffer; (void)size; return 0; } - virtual int availableForWrite() override { return 0; } - virtual bool outputTimeoutPossible () override { return false; } + virtual size_t write(uint8_t) override + { + return 0; + } + virtual size_t write(const uint8_t* buffer, size_t size) override + { + (void)buffer; + (void)size; + return 0; + } + virtual int availableForWrite() override + { + return 0; + } + virtual bool outputTimeoutPossible() override + { + return false; + } // not offering peekBuffer because one streamed element can be not compatible // (Stream:: is by default not peekBuffer-enabled) // input timeout may be possible: - virtual bool inputTimeoutPossible () override { return true; } + virtual bool inputTimeoutPossible() override + { + return true; + } - SerialStreamArray () {} + SerialStreamArray() {} - bool addref (Stream& s) + bool addref(Stream& s) { return addref(&s); } - bool addref (Stream* s) + bool addref(Stream* s) { if (m_size >= MaxSegments) { @@ -173,11 +286,15 @@ class SerialStreamArray: public Stream { if (m_current >= m_size) // end of all + { return 0; + } int ret = m_segments[m_current]->available(); if (ret > 0) + { return ret; + } m_current++; } @@ -189,11 +306,15 @@ class SerialStreamArray: public Stream { if (m_current >= m_size) // end of all + { return 0; + } int ret = m_segments[m_current]->read(); if (ret > 0) + { return ret; + } m_current++; } @@ -205,11 +326,15 @@ class SerialStreamArray: public Stream { if (m_current >= m_size) // end of all + { return 0; + } int ret = m_segments[m_current]->peek(); if (ret > 0) + { return ret; + } m_current++; } @@ -221,11 +346,15 @@ class SerialStreamArray: public Stream { if (m_current >= m_size) // end of all + { return 0; + } size_t ret = m_segments[m_current]->readBytes(buffer, len); if (ret > 0) + { return ret; + } m_current++; } @@ -237,24 +366,30 @@ class SerialStreamArray: public Stream { if (m_current >= m_size) // end of all + { return 0; + } int ret = m_segments[m_current]->read(buffer, len); if (ret > 0) + { return ret; + } m_current++; } } - virtual ssize_t streamSize () override + virtual ssize_t streamSize() override { ssize_t ret = 0; for (int i = 0; i < m_size; i++) { ssize_t s = m_segments[i]->size(); if (s == -1) + { return -1; + } ret += s; } return ret; diff --git a/cores/esp8266/StreamString.cpp b/cores/esp8266/StreamString.cpp deleted file mode 100644 index 0f79eb427c..0000000000 --- a/cores/esp8266/StreamString.cpp +++ /dev/null @@ -1,71 +0,0 @@ -/** - StreamString.cpp - - Copyright (c) 2015 Markus Sattler. All rights reserved. - This file is part of the esp8266 core for Arduino environment. - - This library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with this library; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - - */ - - -#include "StreamString.h" - -#if !STRING_IS_STREAM - -size_t StreamString::write(const uint8_t *data, size_t size) { - if(size && data) { - const unsigned int newlen = length() + size; - if(reserve(newlen + 1)) { - memcpy((void *) (wbuffer() + len()), (const void *) data, size); - setLen(newlen); - *(wbuffer() + newlen) = 0x00; // add null for string end - return size; - } - DEBUGV(":stream2string: OOM (%d->%d)\n", length(), newlen+1); - } - return 0; -} - -size_t StreamString::write(uint8_t data) { - return concat((char) data); -} - -int StreamString::available() { - return length(); -} - -int StreamString::read() { - if(length()) { - char c = charAt(0); - remove(0, 1); - return c; - - } - return -1; -} - -int StreamString::peek() { - if(length()) { - char c = charAt(0); - return c; - } - return -1; -} - -void StreamString::flush() { -} - -#endif // !STRING_IS_STREAM diff --git a/cores/esp8266/StreamString.h b/cores/esp8266/StreamString.h index e5e2d9bb92..653fc3354c 100644 --- a/cores/esp8266/StreamString.h +++ b/cores/esp8266/StreamString.h @@ -1,30 +1,31 @@ /** - StreamString.h + StreamString.h - Copyright (c) 2015 Markus Sattler. All rights reserved. - This file is part of the esp8266 core for Arduino environment. + Copyright (c) 2020 D. Gauchard. All rights reserved. + This file is part of the esp8266 core for Arduino environment. - This library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. - This library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. - You should have received a copy of the GNU Lesser General Public - License along with this library; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ -#ifndef STREAMSTRING_H_ -#define STREAMSTRING_H_ +#ifndef __STREAMSTRING_H +#define __STREAMSTRING_H #include "WString.h" +/////////////////////////////////////////////////////////////// #if STRING_IS_STREAM // StreamString has been integrated into String @@ -32,17 +33,176 @@ using StreamString = String; #else // !STRING_IS_STREAM -class StreamString: public Stream, public String { +/////////////////////////////////////////////////////////////// +// using sstream helper as Stream and String pointer + +class sstream: public Stream +{ public: - size_t write(const uint8_t *buffer, size_t size) override; - size_t write(uint8_t data) override; - int available() override; - int read() override; - int peek() override; - void flush() override; + sstream(String& string): string(&string) + { + } + + sstream(String* string): string(string) + { + } + + virtual int available() override + { + return string->length(); + } + + virtual int read() override + { + if (peekPointer < 0) + { + // consume chars + if (string->length()) + { + char c = string->charAt(0); + string->remove(0, 1); + return c; + } + } + else if (peekPointer < (int)string->length()) + // return pointed and move pointer + { + return string->charAt(peekPointer++); + } + + // everything is read + return -1; + } + + virtual size_t write(uint8_t data) override + { + return string->concat((char)data); + } + + virtual int read(uint8_t* buffer, size_t len) override + { + if (peekPointer < 0) + { + // string will be consumed + size_t l = std::min(len, (size_t)string->length()); + memcpy(buffer, string->c_str(), l); + string->remove(0, l); + return l; + } + + if (peekPointer >= (int)string->length()) + { + return 0; + } + + // only the pointer is moved + size_t l = std::min(len, (size_t)(string->length() - peekPointer)); + memcpy(buffer, string->c_str() + peekPointer, l); + peekPointer += l; + return l; + } + + virtual size_t write(const uint8_t *buffer, size_t size) override + { + return string->concat((const char*)buffer, size) ? size : 0; + } + + virtual int peek() override + { + if (peekPointer < 0) + { + if (string->length()) + { + return string->charAt(0); + } + } + else if (peekPointer < (int)string->length()) + { + return string->charAt(peekPointer); + } + + return -1; + } + + virtual void flush() override + { + // nothing to do + } + + virtual bool inputTimeoutPossible() override + { + return false; + } + virtual bool outputTimeoutPossible() override + { + return false; + } + + //// Stream's peekBufferAPI + + virtual const char* peekBuffer() override + { + if (peekPointer < 0) + { + return string->c_str(); + } + if (peekPointer < (int)string->length()) + { + return string->c_str() + peekPointer; + } + return nullptr; + } + + virtual void peekConsume(size_t consume) override + { + if (peekPointer < 0) + // string is really consumed + { + string->remove(0, consume); + } + else + // only the pointer is moved + { + peekPointer = std::min((size_t)string->length(), peekPointer + consume); + } + } + + // calling peekPointerSetConsume() will consume bytes as they are stream-read + void peekPointerSetConsume() + { + peekPointer = -1; + } + void peekPointerReset(int pointer = 0) + { + peekPointer = pointer; + } + +protected: + + String* string; + + // peekPointer is used with peekBufferAPI, + // on peekConsume(), chars can either: + // - be really consumed = disappeared + // (case when peekPointer==-1) + // - marked as read + // (peekPointer >=0 is increased) + int peekPointer = 0; +}; + +// StreamString is a String and a sstream pointing to itself-as-String + +class StreamString: public String, public sstream +{ +public: + StreamString(String&& string): String(string), sstream(this) { } + StreamString(const String& string): String(string), sstream(this) { } + StreamString(StreamString&& bro): String(bro), sstream(this) { } + StreamString(const StreamString& bro): String(bro), sstream(this) { } + StreamString(): String(), sstream(this) { } }; #endif // !STRING_IS_STREAM -#endif /* STREAMSTRING_H_ */ +#endif // __STREAMSTRING_H diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 45aef4b8af..bd629632a7 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -1105,9 +1105,9 @@ bool HTTPClient::sendHeader(const char * type) // transfer all of it, with timeout #if STRING_IS_STREAM - return header.toSize(_client, header.length()) == header.length(); + return header.toAll(_client, header.length()) == header.length(); #else - return (_client->write((const uint8_t *) header.c_str(), header.length()) == header.length()); + return StreamPtr(header).toAll(_client) == header.length(); #endif } diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index d312b9e09e..4f7e926250 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -493,7 +493,7 @@ void ESP8266WebServerTemplate::send(int code, const char* content_ty #if STRING_IS_STREAM size_t sent = header.toAll(&_currentClient); #else - size_t sent = StreamPtr(header.c_str(), header.length()).toAll(&_currentClient); + size_t sent = StreamPtr(header).toAll(&_currentClient); #endif if (sent != header.length()) DBGWS("HTTPServer: error: sent %zd on %u bytes\n", sent, header.length()); diff --git a/libraries/ESP8266WebServer/src/Parsing-impl.h b/libraries/ESP8266WebServer/src/Parsing-impl.h index 43f1eeab2f..4efd351476 100644 --- a/libraries/ESP8266WebServer/src/Parsing-impl.h +++ b/libraries/ESP8266WebServer/src/Parsing-impl.h @@ -45,22 +45,7 @@ static bool readBytesWithTimeout(typename ServerType::ClientType& client, size_t template static bool readBytesWithTimeout(typename ServerType::ClientType& client, size_t maxLength, String& data, int timeout_ms) { - if (!data.reserve(maxLength + 1)) - return false; - data[0] = 0; // data.clear()?? - while (data.length() < maxLength) { - int tries = timeout_ms; - size_t avail; - while (!(avail = client.available()) && tries--) - delay(1); - if (!avail) - break; - if (data.length() + avail > maxLength) - avail = maxLength - data.length(); - while (avail--) - data += (char)client.read(); - } - return data.length() == maxLength; + return client.toSize(sstream(data), maxLength, timeout_ms) == maxLength; } #endif diff --git a/tests/restyle.sh b/tests/restyle.sh index 86c1f337d8..e8946e29e0 100755 --- a/tests/restyle.sh +++ b/tests/restyle.sh @@ -14,6 +14,7 @@ all=" libraries/ESP8266mDNS libraries/Wire cores/esp8266/core_esp8266_si2c.cpp +cores/esp8266/StreamString.h cores/esp8266/StreamDev.h cores/esp8266/StreamDev.cpp " # core From 9021ed82b9b9adc8c5edca12cf990e8471bc793d Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 15 Aug 2020 15:16:18 +0200 Subject: [PATCH 126/207] rewrite StreamString (wip) --- cores/esp8266/StreamDev.cpp | 54 +++++++++++++---- cores/esp8266/StreamDev.h | 17 +++++- cores/esp8266/StreamString.h | 30 ++++++---- cores/esp8266/WString.cpp | 112 ----------------------------------- cores/esp8266/WString.h | 54 ----------------- 5 files changed, 75 insertions(+), 192 deletions(-) diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index 5f66818af7..82c286ebbd 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -66,15 +66,15 @@ size_t Stream::toFull(Print* to, { size_t avpk = availableForPeek(); if (avpk == 0 && !inputTimeoutPossible()) - // no more data to read, ever { + // no more data to read, ever break; } size_t w = to->availableForWrite(); if (w == 0 && !outputTimeoutPossible()) - // no more data can be written, ever { + // no more data can be written, ever break; } @@ -113,16 +113,16 @@ size_t Stream::toFull(Print* to, } if (!w && !maxLen && readUntilChar < 0) - // nothing has been transferred and no specific condition is requested { + // nothing has been transferred and no specific condition is requested break; } if (timedOut) + { // either (maxLen>0) nothing has been transferred for too long // or readUntilChar >= 0 but char is not encountered for too long // or (maxLen=0) too much time has been spent here - { break; } @@ -141,15 +141,15 @@ size_t Stream::toFull(Print* to, { size_t avpk = availableForPeek(); if (avpk == 0 && !inputTimeoutPossible()) - // no more data to read, ever { + // no more data to read, ever break; } size_t w = to->availableForWrite(); if (w == 0 && !outputTimeoutPossible()) - // no more data can be written, ever { + // no more data can be written, ever break; } @@ -174,16 +174,16 @@ size_t Stream::toFull(Print* to, } if (!w && !maxLen && readUntilChar < 0) - // nothing has been transferred and no specific condition is requested { + // nothing has been transferred and no specific condition is requested break; } if (timedOut) + { // either (maxLen>0) nothing has been transferred for too long // or readUntilChar >= 0 but char is not encountered for too long // or (maxLen=0) too much time has been spent here - { break; } @@ -202,8 +202,8 @@ size_t Stream::toFull(Print* to, { size_t avr = available(); if (avr == 0 && !inputTimeoutPossible()) - // no more data to read, ever { + // no more data to read, ever break; } @@ -239,16 +239,16 @@ size_t Stream::toFull(Print* to, } if (!w && !maxLen && readUntilChar < 0) - // nothing has been transferred and no specific condition is requested { + // nothing has been transferred and no specific condition is requested break; } if (timedOut) + { // either (maxLen>0) nothing has been transferred for too long // or readUntilChar >= 0 but char is not encountered for too long // or (maxLen=0) too much time has been spent here - { break; } @@ -265,19 +265,49 @@ size_t Stream::toFull(Print* to, setWriteError(STREAMTO_TIMED_OUT); } else if ((ssize_t)written != len) + { // This is happening when source cannot timeout (ex: a String) // but has not enough data, or a dest has closed or cannot // timeout but is too small (String, buffer...) // // Mark it as an error because user usually wants to get what is // asked for. - { setWriteError(STREAMTO_SHORT); } } return written; } +Stream& operator << (Stream& out, String& string) +{ + StreamPtr(string).toNow(out); + return out; +} + +Stream& operator << (Stream& out, Stream& stream) +{ + stream.toNow(out); + return out; +} + +Stream& operator << (Stream& out, StreamString& stream) +{ + stream.toNow(out); + return out; +} + +Stream& operator << (Stream& out, const char* text) +{ + StreamPtr(text).toNow(out); + return out; +} + +Stream& operator << (Stream& out, const __FlashStringHelper* text) +{ + StreamPtr(text).toNow(text); + return out; +} + #if !defined(NO_GLOBAL_INSTANCES) && !defined(NO_GLOBAL_STREAMDEV) StreamNull devnull; #endif diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 3dc2d6a288..a33ee0e74e 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -22,7 +22,7 @@ #ifndef __STREAMDEV_H #define __STREAMDEV_H -#include +#include /////////////////////////////////////////////// // /dev/null @@ -149,7 +149,7 @@ class StreamPtr: public StreamNull StreamPtr(const char* buffer, size_t size, bool in_flash = false): _buffer(buffer), _size(size), _in_flash(in_flash) { } StreamPtr(const uint8_t* buffer, size_t size, bool in_flash = false): _buffer((const char*)buffer), _size(size), _in_flash(in_flash) { } StreamPtr(const __FlashStringHelper* buffer, size_t size): _buffer(reinterpret_cast(buffer)), _size(size), _in_flash(true) { } - StreamPtr(const __FlashStringHelper* buffer): _buffer(reinterpret_cast(buffer)), _size(strlen_P(_buffer)), _in_flash(true) { } +// StreamPtr(const __FlashStringHelper* text): _buffer(reinterpret_cast(text)), _size(strlen_P(text)), _in_flash(true) { } void peekPointerReset(int pointer = 0) { @@ -161,14 +161,17 @@ class StreamPtr: public StreamNull { return availableForPeek(); } + virtual int read() override { return _peekPointer < _size ? _buffer[_peekPointer++] : -1; } + virtual int peek() override { return _peekPointer < _size ? _buffer[_peekPointer] : -1; } + virtual size_t readBytes(char* buffer, size_t len) override { if (_peekPointer >= _size) @@ -187,6 +190,7 @@ class StreamPtr: public StreamNull _peekPointer += cpylen; return cpylen; } + virtual int read(uint8_t* buffer, size_t len) override { return readBytes((char*)buffer, len); @@ -202,20 +206,29 @@ class StreamPtr: public StreamNull { return !_in_flash; } + virtual size_t availableForPeek() override { return _peekPointer < _size ? _size - _peekPointer : 0; } + virtual const char* peekBuffer() override { return _peekPointer < _size ? _buffer + _peekPointer : nullptr; } + virtual void peekConsume(size_t consume) override { _peekPointer += consume; } }; +Stream& operator << (Stream& out, String& string); +Stream& operator << (Stream& out, Stream& stream); +Stream& operator << (Stream& out, StreamString& stream); +Stream& operator << (Stream& out, const char* text); +Stream& operator << (Stream& out, const __FlashStringHelper* text); + /////////////////////////////////////////////// // serialization: // combine multiple input Stream into one diff --git a/cores/esp8266/StreamString.h b/cores/esp8266/StreamString.h index 653fc3354c..4222781827 100644 --- a/cores/esp8266/StreamString.h +++ b/cores/esp8266/StreamString.h @@ -25,14 +25,6 @@ #include "WString.h" -/////////////////////////////////////////////////////////////// -#if STRING_IS_STREAM - -// StreamString has been integrated into String -using StreamString = String; - -#else // !STRING_IS_STREAM - /////////////////////////////////////////////////////////////// // using sstream helper as Stream and String pointer @@ -53,6 +45,11 @@ class sstream: public Stream return string->length(); } + virtual int availableForWrite() override + { + return 256; // XXX + } + virtual int read() override { if (peekPointer < 0) @@ -103,9 +100,9 @@ class sstream: public Stream return l; } - virtual size_t write(const uint8_t *buffer, size_t size) override + virtual size_t write(const uint8_t* buffer, size_t len) override { - return string->concat((const char*)buffer, size) ? size : 0; + return string->concat((const char*)buffer, len) ? len : 0; } virtual int peek() override @@ -200,9 +197,18 @@ class StreamString: public String, public sstream StreamString(const String& string): String(string), sstream(this) { } StreamString(StreamString&& bro): String(bro), sstream(this) { } StreamString(const StreamString& bro): String(bro), sstream(this) { } + StreamString(const char* text): String(text), sstream(this) { } + StreamString(const __FlashStringHelper *str): String(str), sstream(this) { } StreamString(): String(), sstream(this) { } -}; -#endif // !STRING_IS_STREAM + explicit StreamString(char c): String(c), sstream(this) { } + explicit StreamString(unsigned char c, unsigned char base = 10): String(c, base), sstream(this) { } + explicit StreamString(int i, unsigned char base = 10): String(i, base), sstream(this) { } + explicit StreamString(unsigned int i, unsigned char base = 10): String(i, base), sstream(this) { } + explicit StreamString(long l, unsigned char base = 10): String(l, base), sstream(this) { } + explicit StreamString(unsigned long l, unsigned char base = 10): String(l, base), sstream(this) { } + explicit StreamString(float f, unsigned char decimalPlaces = 2): String(f, decimalPlaces), sstream(this) { } + explicit StreamString(double d, unsigned char decimalPlaces = 2): String(d, decimalPlaces), sstream(this) { } +}; #endif // __STREAMSTRING_H diff --git a/cores/esp8266/WString.cpp b/cores/esp8266/WString.cpp index 43ac69df18..bb1097a619 100644 --- a/cores/esp8266/WString.cpp +++ b/cores/esp8266/WString.cpp @@ -851,115 +851,3 @@ double String::toDouble(void) const // global empty string to allow returning const String& with nothing const String emptyString; - -#if STRING_IS_STREAM - -///////////////////////////////////////////// -// Stream API: - -//// imported from former StreamString.cpp - -size_t String::write(const uint8_t *data, size_t size) -{ - if(size && data) - { - const unsigned int newlen = length() + size; - if(reserve(newlen + 1)) - { - memcpy((void *) (wbuffer() + len()), (const void *) data, size); - setLen(newlen); - *(wbuffer() + newlen) = 0x00; // add null for string end - return size; - } - DEBUGV(":stream2string: OOM (%d->%d)\n", length(), newlen+1); - } - return 0; -} - -size_t String::write(uint8_t data) -{ - return concat((char) data); -} - -int String::available() -{ - return peekPointer < 0? length(): length() - peekPointer; -} - -int String::read() -{ - if (peekPointer < 0) - { - if (length()) - { - char c = charAt(0); - remove(0, 1); - return c; - } - } - else if (peekPointer < (int)length()) - return charAt(peekPointer++); - - return -1; -} - -int String::peek() -{ - if (peekPointer < 0) - { - if (length()) - return charAt(0); - } - else if (peekPointer < (int)length()) - return charAt(peekPointer); - - return -1; -} - -void String::flush() -{ -} - -//// Stream's peekBufferAPI - -const char* String::peekBuffer () -{ - if (peekPointer < 0) - return buffer(); - if (peekPointer < (int)length()) - return buffer() + peekPointer; - return nullptr; -} - -void String::peekConsume (size_t consume) -{ - if (peekPointer < 0) - // string is really consumed - remove(0, consume); - else - // only the pointer is moved - peekPointer = std::min((size_t)length(), peekPointer + consume); -} - -int String::read (uint8_t* buffer, size_t len) -{ - if (peekPointer < 0) - { - // string will be consumed - size_t l = std::min(len, (size_t)length()); - memcpy(buffer, String::buffer(), l); - remove(0, l); - return l; - } - - if (peekPointer >= (int)length()) - return 0; - - // only the pointer is moved - size_t l = std::min(len, (size_t)(length() - peekPointer)); - memcpy(buffer, String::buffer() + peekPointer, l); - peekPointer += l; - return l; -} - -#endif // STRING_IS_STREAM diff --git a/cores/esp8266/WString.h b/cores/esp8266/WString.h index ebb2c84c07..b2d7db5918 100644 --- a/cores/esp8266/WString.h +++ b/cores/esp8266/WString.h @@ -40,13 +40,8 @@ class __FlashStringHelper; #define FPSTR(pstr_pointer) (reinterpret_cast(pstr_pointer)) #define F(string_literal) (FPSTR(PSTR(string_literal))) -#define STRING_IS_STREAM 1 // helper to avoid overloading functions using String:: with StreamString:: - // The string class class String -#if STRING_IS_STREAM - : public Stream -#endif { // use a function pointer to allow for "if (s)" without the // complications of an operator bool(). for more information, see: @@ -322,55 +317,6 @@ class String String & copy(const char *cstr, unsigned int length); String & copy(const __FlashStringHelper *pstr, unsigned int length); void move(String &rhs); - -#if STRING_IS_STREAM - ///////////////////////////////////////////// - // Print/Stream/peekBuffer API: - -protected: - - // peekPointer is used with peekBufferAPI, - // on peekConsume(), chars can either: - // - be really consumed = disappeared - // (case when peekPointer==-1) - // - marked as read - // (peekPointer >=0 is increased) - int peekPointer = 0; - -public: - - //// Stream: - - size_t write(const uint8_t *buffer, size_t size) override; - size_t write(uint8_t data) override; - - int available() override; - int read() override; - int peek() override; - void flush() override; - - //// peekBuffer API: - - virtual bool peekBufferAPI () const override { return true; } - virtual size_t availableForPeek () override { return available(); } - virtual const char* peekBuffer () override; - virtual void peekConsume (size_t consume) override; - virtual bool inputTimeoutPossible () override { return false; } - virtual int read (uint8_t* buffer, size_t len) override; - - // calling peekPointerSetConsume() will consume bytes as they are stream-read - void peekPointerSetConsume () { peekPointer = -1; } - void peekPointerReset (int pointer = 0) { peekPointer = pointer; } - - virtual ssize_t streamSize () override { return length(); } - - //// Print: - - virtual bool outputTimeoutPossible () override { return false; } - virtual int availableForWrite() override { return 64; } // or biggestChunk()/4 or max(1,reserved-length)? - -#endif // STRING_IS_STREAM - }; class StringSumHelper: public String { From b52e61bac628138da2d96466e7e3f5838f9a44ee Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 15 Aug 2020 17:00:41 +0200 Subject: [PATCH 127/207] ditto --- cores/esp8266/StreamDev.cpp | 10 ++-- cores/esp8266/StreamDev.h | 4 +- cores/esp8266/StreamString.h | 103 +++++++++++++++++++++++------------ 3 files changed, 75 insertions(+), 42 deletions(-) diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index 82c286ebbd..841181be1b 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -280,31 +280,31 @@ size_t Stream::toFull(Print* to, Stream& operator << (Stream& out, String& string) { - StreamPtr(string).toNow(out); + StreamPtr(string).toAll(out); return out; } Stream& operator << (Stream& out, Stream& stream) { - stream.toNow(out); + stream.toAll(out); return out; } Stream& operator << (Stream& out, StreamString& stream) { - stream.toNow(out); + stream.toAll(out); return out; } Stream& operator << (Stream& out, const char* text) { - StreamPtr(text).toNow(out); + StreamPtr(text).toAll(out); return out; } Stream& operator << (Stream& out, const __FlashStringHelper* text) { - StreamPtr(text).toNow(text); + StreamPtr(text).toAll(text); return out; } diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index a33ee0e74e..efa496a950 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -149,9 +149,9 @@ class StreamPtr: public StreamNull StreamPtr(const char* buffer, size_t size, bool in_flash = false): _buffer(buffer), _size(size), _in_flash(in_flash) { } StreamPtr(const uint8_t* buffer, size_t size, bool in_flash = false): _buffer((const char*)buffer), _size(size), _in_flash(in_flash) { } StreamPtr(const __FlashStringHelper* buffer, size_t size): _buffer(reinterpret_cast(buffer)), _size(size), _in_flash(true) { } -// StreamPtr(const __FlashStringHelper* text): _buffer(reinterpret_cast(text)), _size(strlen_P(text)), _in_flash(true) { } + StreamPtr(const __FlashStringHelper* text): _buffer(reinterpret_cast(text)), _size(strlen_P((PGM_P)text)), _in_flash(true) { } - void peekPointerReset(int pointer = 0) + void reset(int pointer = 0) { _peekPointer = pointer; } diff --git a/cores/esp8266/StreamString.h b/cores/esp8266/StreamString.h index 4222781827..2df15bdbe4 100644 --- a/cores/esp8266/StreamString.h +++ b/cores/esp8266/StreamString.h @@ -26,17 +26,18 @@ #include "WString.h" /////////////////////////////////////////////////////////////// -// using sstream helper as Stream and String pointer +// SStream points to a String and makes it a Stream +// (it is also the helper for StreamString) -class sstream: public Stream +class SStream: public Stream { public: - sstream(String& string): string(&string) + SStream(String& string): string(&string) { } - sstream(String* string): string(string) + SStream(String* string): string(string) { } @@ -47,7 +48,7 @@ class sstream: public Stream virtual int availableForWrite() override { - return 256; // XXX + return 32767; } virtual int read() override @@ -63,8 +64,8 @@ class sstream: public Stream } } else if (peekPointer < (int)string->length()) - // return pointed and move pointer { + // return pointed and move pointer return string->charAt(peekPointer++); } @@ -131,6 +132,7 @@ class sstream: public Stream { return false; } + virtual bool outputTimeoutPossible() override { return false; @@ -154,23 +156,28 @@ class sstream: public Stream virtual void peekConsume(size_t consume) override { if (peekPointer < 0) - // string is really consumed { + // string is really consumed string->remove(0, consume); } else - // only the pointer is moved { + // only the pointer is moved peekPointer = std::min((size_t)string->length(), peekPointer + consume); } } - // calling peekPointerSetConsume() will consume bytes as they are stream-read - void peekPointerSetConsume() + // calling setConsume() will consume bytes as the stream is read + // (not enabled by default) + void setConsume() { peekPointer = -1; } - void peekPointerReset(int pointer = 0) + + // Reading this stream will mark the string as read without consuming + // This is the default. + // Calling reset() resets the read state and allows rereading. + void reset(int pointer = 0) { peekPointer = pointer; } @@ -179,36 +186,62 @@ class sstream: public Stream String* string; - // peekPointer is used with peekBufferAPI, - // on peekConsume(), chars can either: - // - be really consumed = disappeared - // (case when peekPointer==-1) - // - marked as read - // (peekPointer >=0 is increased) + // default (0): read marks as read without consuming(erasing) the string: int peekPointer = 0; }; -// StreamString is a String and a sstream pointing to itself-as-String -class StreamString: public String, public sstream +// StreamString is a SStream holding the String + +class StreamString: public String, public SStream { public: - StreamString(String&& string): String(string), sstream(this) { } - StreamString(const String& string): String(string), sstream(this) { } - StreamString(StreamString&& bro): String(bro), sstream(this) { } - StreamString(const StreamString& bro): String(bro), sstream(this) { } - StreamString(const char* text): String(text), sstream(this) { } - StreamString(const __FlashStringHelper *str): String(str), sstream(this) { } - StreamString(): String(), sstream(this) { } - - explicit StreamString(char c): String(c), sstream(this) { } - explicit StreamString(unsigned char c, unsigned char base = 10): String(c, base), sstream(this) { } - explicit StreamString(int i, unsigned char base = 10): String(i, base), sstream(this) { } - explicit StreamString(unsigned int i, unsigned char base = 10): String(i, base), sstream(this) { } - explicit StreamString(long l, unsigned char base = 10): String(l, base), sstream(this) { } - explicit StreamString(unsigned long l, unsigned char base = 10): String(l, base), sstream(this) { } - explicit StreamString(float f, unsigned char decimalPlaces = 2): String(f, decimalPlaces), sstream(this) { } - explicit StreamString(double d, unsigned char decimalPlaces = 2): String(d, decimalPlaces), sstream(this) { } + + StreamString(StreamString&& bro): String(bro), SStream(this) { } + StreamString(const StreamString& bro): String(bro), SStream(this) { } + + // duplicate String contructors and operator=: + + StreamString(const char* text = nullptr): String(text), SStream(this) { } + StreamString(const String& string): String(string), SStream(this) { } + StreamString(const __FlashStringHelper *str): String(str), SStream(this) { } + StreamString(String&& string): String(string), SStream(this) { } + StreamString(StringSumHelper&& sum): String(sum), SStream(this) { } + + explicit StreamString(char c): String(c), SStream(this) { } + explicit StreamString(unsigned char c, unsigned char base = 10): String(c, base), SStream(this) { } + explicit StreamString(int i, unsigned char base = 10): String(i, base), SStream(this) { } + explicit StreamString(unsigned int i, unsigned char base = 10): String(i, base), SStream(this) { } + explicit StreamString(long l, unsigned char base = 10): String(l, base), SStream(this) { } + explicit StreamString(unsigned long l, unsigned char base = 10): String(l, base), SStream(this) { } + explicit StreamString(float f, unsigned char decimalPlaces = 2): String(f, decimalPlaces), SStream(this) { } + explicit StreamString(double d, unsigned char decimalPlaces = 2): String(d, decimalPlaces), SStream(this) { } + + StreamString& operator= (const String& rhs) + { + String::operator=(rhs); + return *this; + } + StreamString& operator= (const char* cstr) + { + String::operator=(cstr); + return *this; + } + StreamString& operator= (const __FlashStringHelper* str) + { + String::operator=(str); + return *this; + } + StreamString& operator= (String&& rval) + { + String::operator=(rval); + return *this; + } + StreamString& operator= (StringSumHelper&& rval) + { + String::operator=(rval); + return *this; + } }; #endif // __STREAMSTRING_H From 46808ccd68ccf03ba05d49bfba49ddccb359a164 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 15 Aug 2020 17:13:34 +0200 Subject: [PATCH 128/207] ditto --- cores/esp8266/StreamString.h | 44 +++++++++---------- cores/esp8266/WString.cpp | 3 -- cores/esp8266/WString.h | 5 +-- .../src/ESP8266HTTPClient.cpp | 4 -- .../src/ESP8266WebServer-impl.h | 4 -- libraries/ESP8266WebServer/src/Parsing-impl.h | 14 +----- 6 files changed, 24 insertions(+), 50 deletions(-) diff --git a/cores/esp8266/StreamString.h b/cores/esp8266/StreamString.h index 2df15bdbe4..1deb7e38fe 100644 --- a/cores/esp8266/StreamString.h +++ b/cores/esp8266/StreamString.h @@ -26,18 +26,18 @@ #include "WString.h" /////////////////////////////////////////////////////////////// -// SStream points to a String and makes it a Stream +// S2Stream points to a String and makes it a Stream // (it is also the helper for StreamString) -class SStream: public Stream +class S2Stream: public Stream { public: - SStream(String& string): string(&string) + S2Stream(String& string): string(&string) { } - SStream(String* string): string(string) + S2Stream(String* string): string(string) { } @@ -191,31 +191,31 @@ class SStream: public Stream }; -// StreamString is a SStream holding the String +// StreamString is a S2Stream holding the String -class StreamString: public String, public SStream +class StreamString: public String, public S2Stream { public: - StreamString(StreamString&& bro): String(bro), SStream(this) { } - StreamString(const StreamString& bro): String(bro), SStream(this) { } + StreamString(StreamString&& bro): String(bro), S2Stream(this) { } + StreamString(const StreamString& bro): String(bro), S2Stream(this) { } // duplicate String contructors and operator=: - StreamString(const char* text = nullptr): String(text), SStream(this) { } - StreamString(const String& string): String(string), SStream(this) { } - StreamString(const __FlashStringHelper *str): String(str), SStream(this) { } - StreamString(String&& string): String(string), SStream(this) { } - StreamString(StringSumHelper&& sum): String(sum), SStream(this) { } - - explicit StreamString(char c): String(c), SStream(this) { } - explicit StreamString(unsigned char c, unsigned char base = 10): String(c, base), SStream(this) { } - explicit StreamString(int i, unsigned char base = 10): String(i, base), SStream(this) { } - explicit StreamString(unsigned int i, unsigned char base = 10): String(i, base), SStream(this) { } - explicit StreamString(long l, unsigned char base = 10): String(l, base), SStream(this) { } - explicit StreamString(unsigned long l, unsigned char base = 10): String(l, base), SStream(this) { } - explicit StreamString(float f, unsigned char decimalPlaces = 2): String(f, decimalPlaces), SStream(this) { } - explicit StreamString(double d, unsigned char decimalPlaces = 2): String(d, decimalPlaces), SStream(this) { } + StreamString(const char* text = nullptr): String(text), S2Stream(this) { } + StreamString(const String& string): String(string), S2Stream(this) { } + StreamString(const __FlashStringHelper *str): String(str), S2Stream(this) { } + StreamString(String&& string): String(string), S2Stream(this) { } + StreamString(StringSumHelper&& sum): String(sum), S2Stream(this) { } + + explicit StreamString(char c): String(c), S2Stream(this) { } + explicit StreamString(unsigned char c, unsigned char base = 10): String(c, base), S2Stream(this) { } + explicit StreamString(int i, unsigned char base = 10): String(i, base), S2Stream(this) { } + explicit StreamString(unsigned int i, unsigned char base = 10): String(i, base), S2Stream(this) { } + explicit StreamString(long l, unsigned char base = 10): String(l, base), S2Stream(this) { } + explicit StreamString(unsigned long l, unsigned char base = 10): String(l, base), S2Stream(this) { } + explicit StreamString(float f, unsigned char decimalPlaces = 2): String(f, decimalPlaces), S2Stream(this) { } + explicit StreamString(double d, unsigned char decimalPlaces = 2): String(d, decimalPlaces), S2Stream(this) { } StreamString& operator= (const String& rhs) { diff --git a/cores/esp8266/WString.cpp b/cores/esp8266/WString.cpp index bb1097a619..2bac5a9725 100644 --- a/cores/esp8266/WString.cpp +++ b/cores/esp8266/WString.cpp @@ -6,9 +6,6 @@ Modified by Ivan Grokhotkov, 2014 - esp8266 support Modified by Michael C. Miller, 2015 - esp8266 progmem support - imported: - StringStream Copyright (c) 2015 Markus Sattler. All rights reserved. - This library is free software; you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either diff --git a/cores/esp8266/WString.h b/cores/esp8266/WString.h index b2d7db5918..6a463f06eb 100644 --- a/cores/esp8266/WString.h +++ b/cores/esp8266/WString.h @@ -28,8 +28,6 @@ #include #include -#include - // An inherited class for holding the result of a concatenation. These // result objects are assumed to be writable by subsequent concatenations. class StringSumHelper; @@ -41,8 +39,7 @@ class __FlashStringHelper; #define F(string_literal) (FPSTR(PSTR(string_literal))) // The string class -class String -{ +class String { // use a function pointer to allow for "if (s)" without the // complications of an operator bool(). for more information, see: // http://www.artima.com/cppsource/safebool.html diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index bd629632a7..d63d53cda3 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -1104,11 +1104,7 @@ bool HTTPClient::sendHeader(const char * type) DEBUG_HTTPCLIENT("[HTTP-Client] sending request header\n-----\n%s-----\n", header.c_str()); // transfer all of it, with timeout -#if STRING_IS_STREAM - return header.toAll(_client, header.length()) == header.length(); -#else return StreamPtr(header).toAll(_client) == header.length(); -#endif } /** diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index 4f7e926250..60ad6816cf 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -490,11 +490,7 @@ void ESP8266WebServerTemplate::send(int code, const char* content_ty if (content_length == 0) content_length = std::max((ssize_t)0, stream->streamSize()); _prepareHeader(header, code, content_type, content_length); -#if STRING_IS_STREAM - size_t sent = header.toAll(&_currentClient); -#else size_t sent = StreamPtr(header).toAll(&_currentClient); -#endif if (sent != header.length()) DBGWS("HTTPServer: error: sent %zd on %u bytes\n", sent, header.length()); if (content_length) diff --git a/libraries/ESP8266WebServer/src/Parsing-impl.h b/libraries/ESP8266WebServer/src/Parsing-impl.h index 4efd351476..72fa44c45c 100644 --- a/libraries/ESP8266WebServer/src/Parsing-impl.h +++ b/libraries/ESP8266WebServer/src/Parsing-impl.h @@ -32,24 +32,12 @@ static const char Content_Type[] PROGMEM = "Content-Type"; static const char filename[] PROGMEM = "filename"; -#if STRING_IS_STREAM - template static bool readBytesWithTimeout(typename ServerType::ClientType& client, size_t maxLength, String& data, int timeout_ms) { - return client.toSize(data, maxLength, timeout_ms) == maxLength; + return client.toSize(S2Stream(data), maxLength, timeout_ms) == maxLength; } -#else - -template -static bool readBytesWithTimeout(typename ServerType::ClientType& client, size_t maxLength, String& data, int timeout_ms) -{ - return client.toSize(sstream(data), maxLength, timeout_ms) == maxLength; -} - -#endif - template typename ESP8266WebServerTemplate::ClientFuture ESP8266WebServerTemplate::_parseRequest(ClientType& client) { // Read the first line of HTTP request From 5f80dcedf354e89d1577dc7848fd1d23c61fa3c7 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 15 Aug 2020 18:14:49 +0200 Subject: [PATCH 129/207] properly reset peekpointer in StreamString --- cores/esp8266/StreamString.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cores/esp8266/StreamString.h b/cores/esp8266/StreamString.h index 1deb7e38fe..ffd0396d46 100644 --- a/cores/esp8266/StreamString.h +++ b/cores/esp8266/StreamString.h @@ -195,6 +195,14 @@ class S2Stream: public Stream class StreamString: public String, public S2Stream { +protected: + + void resetpp () + { + if (peekPointer > 0) + peekPointer = 0; + } + public: StreamString(StreamString&& bro): String(bro), S2Stream(this) { } @@ -220,26 +228,35 @@ class StreamString: public String, public S2Stream StreamString& operator= (const String& rhs) { String::operator=(rhs); + resetpp(); return *this; } + StreamString& operator= (const char* cstr) { String::operator=(cstr); + resetpp(); return *this; } + StreamString& operator= (const __FlashStringHelper* str) { String::operator=(str); + resetpp(); return *this; } + StreamString& operator= (String&& rval) { String::operator=(rval); + resetpp(); return *this; } + StreamString& operator= (StringSumHelper&& rval) { String::operator=(rval); + resetpp(); return *this; } }; From f692c09de788256ac7fcc449c04cfa3310fdaacc Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 16 Aug 2020 18:48:19 +0200 Subject: [PATCH 130/207] fixes, device test, (=) example --- cores/esp8266/FS.h | 2 +- cores/esp8266/Stream.h | 2 +- cores/esp8266/StreamDev.cpp | 26 ++- cores/esp8266/StreamDev.h | 201 ++---------------- cores/esp8266/StreamString.h | 30 ++- .../examples/StreamString/StreamString.ino | 156 ++++++++++++++ .../test_sw_StreamString.ino | 25 +++ tests/host/Makefile | 6 +- 8 files changed, 255 insertions(+), 193 deletions(-) create mode 100644 libraries/esp8266/examples/StreamString/StreamString.ino create mode 100644 tests/device/test_sw_StreamString/test_sw_StreamString.ino diff --git a/cores/esp8266/FS.h b/cores/esp8266/FS.h index 083729f3d2..cdd60e68b6 100644 --- a/cores/esp8266/FS.h +++ b/cores/esp8266/FS.h @@ -73,7 +73,7 @@ class File : public Stream } size_t position() const; size_t size() const; - virtual ssize_t streamSize() override { return size(); } + virtual ssize_t streamSize() override { return size() - position(); } void close(); operator bool() const; const char* name() const; diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 8ee466e8a3..55d3dfa4d7 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -188,7 +188,7 @@ class Stream: public Print { template size_t toSize (T& to, const ssize_t maxLen, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toSize((Print*)&to, maxLen, timeoutMs); } - // size of input (-1 by default = unknown) + // remaining size (-1 by default = unknown) virtual ssize_t streamSize () { return -1; } typedef enum diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index 841181be1b..0e7c58eeab 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -215,6 +215,10 @@ size_t Stream::toFull(Print* to, } w = std::min(w, avr); + if (maxLen) + { + w = std::min(w, maxLen - written); + } w = std::min(w, (decltype(w))64); //XXX FIXME 64 is a constant if (w) { @@ -284,15 +288,31 @@ Stream& operator << (Stream& out, String& string) return out; } -Stream& operator << (Stream& out, Stream& stream) +Stream& operator << (Stream& out, StreamString& stream) { stream.toAll(out); return out; } -Stream& operator << (Stream& out, StreamString& stream) +Stream& operator << (Stream& out, Stream& stream) { - stream.toAll(out); + if (stream.streamSize() < 0) + { + if (stream.inputTimeoutPossible()) + { + // restrict with only what's buffered on input + stream.toNow(out); + } + else + { + // take all what is in input + stream.toAll(out); + } + } + else + { + stream.toSize(out, stream.streamSize()); + } return out; } diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index efa496a950..da5c961609 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -38,12 +38,14 @@ class StreamNull: public Stream { return 1; } + virtual size_t write(const uint8_t* buffer, size_t size) override { (void)buffer; (void)size; return size; } + virtual int availableForWrite() override { return 32767; @@ -54,30 +56,36 @@ class StreamNull: public Stream { return 0; } + virtual int read() override { return -1; } + virtual int peek() override { return -1; } + virtual size_t readBytes(char* buffer, size_t len) override { (void)buffer; (void)len; return 0; } + virtual int read(uint8_t* buffer, size_t len) override { (void)buffer; (void)len; return 0; } + virtual bool outputTimeoutPossible() override { return false; } + virtual bool inputTimeoutPossible() override { return false; @@ -97,8 +105,11 @@ class StreamNull: public Stream class StreamZero: public StreamNull { protected: + char _x; + public: + StreamZero(char x = 0): _x(x) { } // Stream @@ -106,19 +117,23 @@ class StreamZero: public StreamNull { return 32767; } + virtual int read() override { return _x; } + virtual int peek() override { return _x; } + virtual size_t readBytes(char* buffer, size_t len) override { memset(buffer, _x, len); return len; } + virtual int read(uint8_t* buffer, size_t len) override { memset((char*)buffer, _x, len); @@ -223,196 +238,14 @@ class StreamPtr: public StreamNull } }; +/////////////////////////////////////////////// + Stream& operator << (Stream& out, String& string); Stream& operator << (Stream& out, Stream& stream); Stream& operator << (Stream& out, StreamString& stream); Stream& operator << (Stream& out, const char* text); Stream& operator << (Stream& out, const __FlashStringHelper* text); -/////////////////////////////////////////////// -// serialization: -// combine multiple input Stream into one -// useful when sending HTML content (strings, files. ...) - -template -class SerialStreamArray: public Stream -{ -protected: - - Stream* m_segments [MaxSegments]; - int m_size = 0; - int m_current = 0; - -public: - - // not writable - virtual size_t write(uint8_t) override - { - return 0; - } - virtual size_t write(const uint8_t* buffer, size_t size) override - { - (void)buffer; - (void)size; - return 0; - } - virtual int availableForWrite() override - { - return 0; - } - virtual bool outputTimeoutPossible() override - { - return false; - } - - // not offering peekBuffer because one streamed element can be not compatible - // (Stream:: is by default not peekBuffer-enabled) - // input timeout may be possible: - virtual bool inputTimeoutPossible() override - { - return true; - } - - SerialStreamArray() {} - - bool addref(Stream& s) - { - return addref(&s); - } - - bool addref(Stream* s) - { - if (m_size >= MaxSegments) - { - //XXXDEBUG: print exhausted message - return false; - } - - m_segments[m_size++] = s; - return true; - } - - // Stream - virtual int available() override - { - while (true) - { - if (m_current >= m_size) - // end of all - { - return 0; - } - - int ret = m_segments[m_current]->available(); - if (ret > 0) - { - return ret; - } - - m_current++; - } - } - - virtual int read() override - { - while (true) - { - if (m_current >= m_size) - // end of all - { - return 0; - } - - int ret = m_segments[m_current]->read(); - if (ret > 0) - { - return ret; - } - - m_current++; - } - } - - virtual int peek() override - { - while (true) - { - if (m_current >= m_size) - // end of all - { - return 0; - } - - int ret = m_segments[m_current]->peek(); - if (ret > 0) - { - return ret; - } - - m_current++; - } - } - - virtual size_t readBytes(char* buffer, size_t len) override - { - while (true) - { - if (m_current >= m_size) - // end of all - { - return 0; - } - - size_t ret = m_segments[m_current]->readBytes(buffer, len); - if (ret > 0) - { - return ret; - } - - m_current++; - } - } - - virtual int read(uint8_t* buffer, size_t len) override - { - while (true) - { - if (m_current >= m_size) - // end of all - { - return 0; - } - - int ret = m_segments[m_current]->read(buffer, len); - if (ret > 0) - { - return ret; - } - - m_current++; - } - } - - virtual ssize_t streamSize() override - { - ssize_t ret = 0; - for (int i = 0; i < m_size; i++) - { - ssize_t s = m_segments[i]->size(); - if (s == -1) - { - return -1; - } - ret += s; - } - return ret; - } -}; - -class SerialStream: public SerialStreamArray<> -{ -}; - /////////////////////////////////////////////// #if !defined(NO_GLOBAL_INSTANCES) && !defined(NO_GLOBAL_STREAMDEV) diff --git a/cores/esp8266/StreamString.h b/cores/esp8266/StreamString.h index ffd0396d46..5ae0e675a4 100644 --- a/cores/esp8266/StreamString.h +++ b/cores/esp8266/StreamString.h @@ -140,6 +140,20 @@ class S2Stream: public Stream //// Stream's peekBufferAPI + virtual bool peekBufferAPI() const override + { + return true; + } + + virtual size_t availableForPeek() + { + if (peekPointer < 0) + { + return string->length(); + } + return string->length() - peekPointer; + } + virtual const char* peekBuffer() override { if (peekPointer < 0) @@ -167,6 +181,11 @@ class S2Stream: public Stream } } + virtual ssize_t streamSize() override + { + return peekPointer < 0 ? string->length() : string->length() - peekPointer; + } + // calling setConsume() will consume bytes as the stream is read // (not enabled by default) void setConsume() @@ -197,10 +216,12 @@ class StreamString: public String, public S2Stream { protected: - void resetpp () + void resetpp() { if (peekPointer > 0) + { peekPointer = 0; + } } public: @@ -225,6 +246,13 @@ class StreamString: public String, public S2Stream explicit StreamString(float f, unsigned char decimalPlaces = 2): String(f, decimalPlaces), S2Stream(this) { } explicit StreamString(double d, unsigned char decimalPlaces = 2): String(d, decimalPlaces), S2Stream(this) { } + StreamString& operator= (const StreamString& rhs) + { + String::operator=(rhs); + resetpp(); + return *this; + } + StreamString& operator= (const String& rhs) { String::operator=(rhs); diff --git a/libraries/esp8266/examples/StreamString/StreamString.ino b/libraries/esp8266/examples/StreamString/StreamString.ino new file mode 100644 index 0000000000..fa7cea8b60 --- /dev/null +++ b/libraries/esp8266/examples/StreamString/StreamString.ino @@ -0,0 +1,156 @@ + +#include +#include + +void loop() { + delay(1000); +} + +void checksketch(const char* what, const char* res1, const char* res2) { + if (strcmp(res1, res2) == 0) { + Serial << "passed: Test " << what << " (result: '" << res1 << "')\n"; + } else { + Serial << "Test " << what << " failed: '" << res1 << "' <> '" << res2 << "' !\n"; + } +} + +#ifndef check +#define check(what, res1, res2) checksketch(what, res1, res2) +#endif + +void testStream() { + String inputString = "hello"; + StreamString result; + + { + // use StreamString or S2Stream(String) to make a r/w Stream out of a String, + // prefer the lighter StreamPtr(String) to make a read-only Stream out of a String + + result.clear(); + StreamPtr(inputString).toAll(result); + StreamPtr(inputString).toAll(result); + StreamPtr(inputString).toAll(result); + check("StreamPtr.toAll(StreamString)", result.c_str(), "hellohellohello"); + } + + { + // equivalent of the above + + result.clear(); + result << inputString; + result << inputString << inputString; + check("StreamString< +#include + +#define check(what, res1, res2) CHECK(strcmp(res1, res2) == 0) + +#include "../../../libraries/esp8266/examples/StreamString/StreamString.ino" + +BS_ENV_DECLARE(); + +bool pretest () +{ + return true; +} + +void setup () +{ + Serial.begin(115200); + BS_RUN(Serial); +} + +TEST_CASE("StreamSring tests", "[StreamString]") +{ + testStream(); +} diff --git a/tests/host/Makefile b/tests/host/Makefile index cf0ed4fe9f..1c534356ea 100644 --- a/tests/host/Makefile +++ b/tests/host/Makefile @@ -321,12 +321,12 @@ INC_PATHS += $(USERLIBDIRS) INC_PATHS += -I$(INODIR)/.. CPP_OBJECTS_CORE_EMU = $(CPP_SOURCES_CORE_EMU:.cpp=.cpp$(E32).o) $(USERLIBSRCS:.cpp=.cpp$(E32).o) -bin/fullcore.a: $(C_OBJECTS) $(CPP_OBJECTS_CORE_EMU) +bin/fullcore$(E32).a: $(C_OBJECTS) $(CPP_OBJECTS_CORE_EMU) $(VERBAR) ar -rcu $@ $^ $(VERBAR) ranlib $@ -%: %.ino.cpp$(E32).o bin/fullcore.a - $(VERBLD) $(CXX) $(LDFLAGS) $< bin/fullcore.a $(LIBSSL) -o $@ +%: %.ino.cpp$(E32).o bin/fullcore$(E32).a + $(VERBLD) $(CXX) $(LDFLAGS) $< bin/fullcore$(E32).a $(LIBSSL) -o $@ @echo "----> $@ <----" ################################################# From ff1ea5b6e33c6d6a636a2f61cefd5f4b1a546cf4 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 16 Aug 2020 22:40:51 +0200 Subject: [PATCH 131/207] stream: test: show 0 heap occupation with streaming progmem strings --- .../examples/StreamString/StreamString.ino | 37 ++++++++++++++++++- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/libraries/esp8266/examples/StreamString/StreamString.ino b/libraries/esp8266/examples/StreamString/StreamString.ino index fa7cea8b60..ef44b87b06 100644 --- a/libraries/esp8266/examples/StreamString/StreamString.ino +++ b/libraries/esp8266/examples/StreamString/StreamString.ino @@ -8,9 +8,9 @@ void loop() { void checksketch(const char* what, const char* res1, const char* res2) { if (strcmp(res1, res2) == 0) { - Serial << "passed: Test " << what << " (result: '" << res1 << "')\n"; + Serial << "PASSED: Test " << what << " (result: '" << res1 << "')\n"; } else { - Serial << "Test " << what << " failed: '" << res1 << "' <> '" << res2 << "' !\n"; + Serial << "FAILED: Test " << what << ": '" << res1 << "' <> '" << res2 << "' !\n"; } } @@ -18,6 +18,18 @@ void checksketch(const char* what, const char* res1, const char* res2) { #define check(what, res1, res2) checksketch(what, res1, res2) #endif +void testProgmem() { + static const char inProgmem [] PROGMEM = "I am in progmem"; + auto inProgmem2 = F("I am too in progmem"); + + int heap = (int)ESP.getFreeHeap(); + auto stream1 = StreamPtr(inProgmem, sizeof(inProgmem) - 1, true); + auto stream2 = StreamPtr(inProgmem2); + Serial << stream1 << " - " << stream2 << "\n"; + heap -= (int)ESP.getFreeHeap(); + check("NO heap occupation while streaming progmem strings", String(heap).c_str(), "0"); +} + void testStream() { String inputString = "hello"; StreamString result; @@ -139,6 +151,27 @@ void testStream() { result = 23.2; check("StreamString = float", result.c_str(), "23.20"); } + +#if !CORE_MOCK + + testProgmem(); + + { + int heap = (int)ESP.getFreeHeap(); + auto stream = StreamString(F("I am in progmem")); + Serial << stream << "\n"; + heap -= (int)ESP.getFreeHeap(); + String heapStr(heap); + if (heap != 0) { + check("heap is occupied by String/StreamString(progmem)", heapStr.c_str(), heapStr.c_str()); + } else { + check("ERROR: heap should be occupied by String/StreamString(progmem)", heapStr.c_str(), "-1"); + } + } + + testProgmem(); + +#endif } #ifndef TEST_CASE From ddb72423d29ee9b77527495798c597204786d443 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 17 Aug 2020 13:17:09 +0200 Subject: [PATCH 132/207] doc --- doc/reference.rst | 104 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/doc/reference.rst b/doc/reference.rst index 9551d9d035..8e8254a9dc 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -356,3 +356,107 @@ C++ SomeClass* scs = arduino_newarray(SomeClass, 42); delete [] scs; + +Streams +------- + +### Arduino API + +Stream is one of the core classes in the Arduino API. Wire, serial, network and +filesystems are streams, from which data are read or written. + +Making a transfer with streams is quite common, like for example the +historical WiFiSerial sketch: + +.. code:: cpp + + //check clients for data + for (i = 0; i < MAX_SRV_CLIENTS; i++) { + if (serverClients[i] && serverClients[i].connected()) { + if (serverClients[i].available()) { + //get data from the telnet client and push it to the UART + while (serverClients[i].available()) { + Serial.write(serverClients[i].read()); + } + } + } + } + //check UART for data + if (Serial.available()) { + size_t len = Serial.available(); + uint8_t sbuf[len]; + Serial.readBytes(sbuf, len); + //push UART data to all connected telnet clients + for (i = 0; i < MAX_SRV_CLIENTS; i++) { + if (serverClients[i] && serverClients[i].connected()) { + serverClients[i].write(sbuf, len); + delay(1); + } + } + } + +One will notice that in the network to serial direction, data are transfered +byte by byte while data are available. On the other direction, a temporary +buffer is created on stack, filled with available serial data, then +transfered to network. + +The ``readBytes(buffer, length)`` method includes a timeout to ensure that +all required bytes are received. The ``write(buffer, length)`` (inherited +from ``Print::`) function is also usually blocking until the full buffer is +transmitted. Both functions return the number of transmitted bytes. + +That's the way the Stream class works and is commonly used. + +Classes derived from ``Stream::`` also usually introduce the ``read(buffer, +len)`` method, which is similar to ``readBytes(buffer, len)`` without +timeout: the returned value can be less than the requested size, so special +care must be taken with this function, introduced in the Arduino +``Client::`` class. This function has also been introduced in other classes +not derivating from ``Client::``, e.g. ``HardwareSerial::``. + +### Stream extensions + +Proposed Stream extensions are designed to be compatible with Arduino API, +and offer a additional methods to make transfers more efficients and easier +to use. + +- User facing API: ``Stream::to()`` + + The goal of streams is to transfer data between producers and consumers, + like the telnet/serial example above. Four methods are proposed, all of + them return the number of transmitted bytes: + + - ``Stream::toSize(dest, size [, timeout])`` + + This method waits up to the given or default timeout to transfer + ``size`` bytes to the the ``dest`` Stream. + + - ``Stream::toUntil(dest, delim [, timeout])`` + + This method waits up to the given or default timeout to transfer data + until the character ``delim`` is met. The delimiter is read but not + transfered. + + - ``Stream::toNow(dest)`` + + This method transfer all already available data to the destination. + There is no timeout and the returned value is 0 when there is nothing to + transfer or no room in the destination. + + - ``Stream::toAll(dest [, timeout])`` + + This method waits up to the given or default timeout to transfer all + available data. It is useful when source is able to tell that no more + data will be available for this call. + + For example, a source String will not grow during the transfer, or a + particular network connection supposed to send a fixed amount of data + before closing. ``::toAll()`` will receive all bytes. Timeout is used + when source is unable to tell that no more data will come, or with + destination when it needs processing time (e.g. network or serial input + buffer full). + +- String helpers + +- internal Stream API: peekBuffer + From dcb1085d9cf84df6f0c39f977ac693cc93e2706a Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 17 Aug 2020 13:20:30 +0200 Subject: [PATCH 133/207] doc --- doc/reference.rst | 142 +++++++++++++++++++++++----------------------- 1 file changed, 71 insertions(+), 71 deletions(-) diff --git a/doc/reference.rst b/doc/reference.rst index 8e8254a9dc..dd32000b58 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -360,103 +360,103 @@ C++ Streams ------- -### Arduino API +Arduino API -Stream is one of the core classes in the Arduino API. Wire, serial, network and -filesystems are streams, from which data are read or written. + Stream is one of the core classes in the Arduino API. Wire, serial, network and + filesystems are streams, from which data are read or written. -Making a transfer with streams is quite common, like for example the -historical WiFiSerial sketch: + Making a transfer with streams is quite common, like for example the + historical WiFiSerial sketch: -.. code:: cpp + .. code:: cpp - //check clients for data - for (i = 0; i < MAX_SRV_CLIENTS; i++) { - if (serverClients[i] && serverClients[i].connected()) { - if (serverClients[i].available()) { - //get data from the telnet client and push it to the UART - while (serverClients[i].available()) { - Serial.write(serverClients[i].read()); + //check clients for data + for (i = 0; i < MAX_SRV_CLIENTS; i++) { + if (serverClients[i] && serverClients[i].connected()) { + if (serverClients[i].available()) { + //get data from the telnet client and push it to the UART + while (serverClients[i].available()) { + Serial.write(serverClients[i].read()); + } } } } - } - //check UART for data - if (Serial.available()) { - size_t len = Serial.available(); - uint8_t sbuf[len]; - Serial.readBytes(sbuf, len); - //push UART data to all connected telnet clients - for (i = 0; i < MAX_SRV_CLIENTS; i++) { - if (serverClients[i] && serverClients[i].connected()) { - serverClients[i].write(sbuf, len); - delay(1); + //check UART for data + if (Serial.available()) { + size_t len = Serial.available(); + uint8_t sbuf[len]; + Serial.readBytes(sbuf, len); + //push UART data to all connected telnet clients + for (i = 0; i < MAX_SRV_CLIENTS; i++) { + if (serverClients[i] && serverClients[i].connected()) { + serverClients[i].write(sbuf, len); + delay(1); + } } } - } -One will notice that in the network to serial direction, data are transfered -byte by byte while data are available. On the other direction, a temporary -buffer is created on stack, filled with available serial data, then -transfered to network. + One will notice that in the network to serial direction, data are transfered + byte by byte while data are available. On the other direction, a temporary + buffer is created on stack, filled with available serial data, then + transfered to network. -The ``readBytes(buffer, length)`` method includes a timeout to ensure that -all required bytes are received. The ``write(buffer, length)`` (inherited -from ``Print::`) function is also usually blocking until the full buffer is -transmitted. Both functions return the number of transmitted bytes. + The ``readBytes(buffer, length)`` method includes a timeout to ensure that + all required bytes are received. The ``write(buffer, length)`` (inherited + from ``Print::`) function is also usually blocking until the full buffer is + transmitted. Both functions return the number of transmitted bytes. -That's the way the Stream class works and is commonly used. + That's the way the Stream class works and is commonly used. -Classes derived from ``Stream::`` also usually introduce the ``read(buffer, -len)`` method, which is similar to ``readBytes(buffer, len)`` without -timeout: the returned value can be less than the requested size, so special -care must be taken with this function, introduced in the Arduino -``Client::`` class. This function has also been introduced in other classes -not derivating from ``Client::``, e.g. ``HardwareSerial::``. + Classes derived from ``Stream::`` also usually introduce the ``read(buffer, + len)`` method, which is similar to ``readBytes(buffer, len)`` without + timeout: the returned value can be less than the requested size, so special + care must be taken with this function, introduced in the Arduino + ``Client::`` class. This function has also been introduced in other classes + not derivating from ``Client::``, e.g. ``HardwareSerial::``. -### Stream extensions +Stream extensions -Proposed Stream extensions are designed to be compatible with Arduino API, -and offer a additional methods to make transfers more efficients and easier -to use. + Proposed Stream extensions are designed to be compatible with Arduino API, + and offer a additional methods to make transfers more efficients and easier + to use. -- User facing API: ``Stream::to()`` + - User facing API: ``Stream::to()`` - The goal of streams is to transfer data between producers and consumers, - like the telnet/serial example above. Four methods are proposed, all of - them return the number of transmitted bytes: + The goal of streams is to transfer data between producers and consumers, + like the telnet/serial example above. Four methods are proposed, all of + them return the number of transmitted bytes: - - ``Stream::toSize(dest, size [, timeout])`` + - ``Stream::toSize(dest, size [, timeout])`` - This method waits up to the given or default timeout to transfer - ``size`` bytes to the the ``dest`` Stream. + This method waits up to the given or default timeout to transfer + ``size`` bytes to the the ``dest`` Stream. - - ``Stream::toUntil(dest, delim [, timeout])`` + - ``Stream::toUntil(dest, delim [, timeout])`` - This method waits up to the given or default timeout to transfer data - until the character ``delim`` is met. The delimiter is read but not - transfered. + This method waits up to the given or default timeout to transfer data + until the character ``delim`` is met. The delimiter is read but not + transfered. - - ``Stream::toNow(dest)`` + - ``Stream::toNow(dest)`` - This method transfer all already available data to the destination. - There is no timeout and the returned value is 0 when there is nothing to - transfer or no room in the destination. + This method transfer all already available data to the destination. + There is no timeout and the returned value is 0 when there is nothing to + transfer or no room in the destination. - - ``Stream::toAll(dest [, timeout])`` + - ``Stream::toAll(dest [, timeout])`` - This method waits up to the given or default timeout to transfer all - available data. It is useful when source is able to tell that no more - data will be available for this call. + This method waits up to the given or default timeout to transfer all + available data. It is useful when source is able to tell that no more + data will be available for this call. - For example, a source String will not grow during the transfer, or a - particular network connection supposed to send a fixed amount of data - before closing. ``::toAll()`` will receive all bytes. Timeout is used - when source is unable to tell that no more data will come, or with - destination when it needs processing time (e.g. network or serial input - buffer full). + For example, a source String will not grow during the transfer, or a + particular network connection supposed to send a fixed amount of data + before closing. ``::toAll()`` will receive all bytes. Timeout is used + when source is unable to tell that no more data will come, or with + destination when it needs processing time (e.g. network or serial input + buffer full). -- String helpers + - String helpers -- internal Stream API: peekBuffer + - internal Stream API: peekBuffer From 45f7f6c397b07f218004852c43325c292a018838 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 17 Aug 2020 13:46:21 +0200 Subject: [PATCH 134/207] doc --- doc/reference.rst | 64 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 9 deletions(-) diff --git a/doc/reference.rst b/doc/reference.rst index dd32000b58..35317f7284 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -439,24 +439,70 @@ Stream extensions - ``Stream::toNow(dest)`` - This method transfer all already available data to the destination. - There is no timeout and the returned value is 0 when there is nothing to - transfer or no room in the destination. + This method transfers all already available data to the destination. + There is no timeout and the returned value is 0 when there is nothing + to transfer or no room in the destination. - ``Stream::toAll(dest [, timeout])`` This method waits up to the given or default timeout to transfer all available data. It is useful when source is able to tell that no more - data will be available for this call. + data will be available for this call, or when destination is able to + tell that it will no more be able to receive. For example, a source String will not grow during the transfer, or a particular network connection supposed to send a fixed amount of data - before closing. ``::toAll()`` will receive all bytes. Timeout is used - when source is unable to tell that no more data will come, or with - destination when it needs processing time (e.g. network or serial input - buffer full). + before closing. ``::toAll()`` will receive all bytes. Timeout is + useful when destination needs processing time (e.g. network or serial + input buffer full = please wait a bit). - - String helpers + - String, flash strings helpers + + Two additional classes are proposed. + + - ``StreamPtr::`` is designed to hold a constant buffer (in ram or flash). + + A `Stream::` is made from `const char*`, `F("some words in flash")` + or `PROGMEM` strings. This class makes no copy, even with data in + flash. For them, byte-by-byte transfers is a consequence. Others + can be transfered at once when possible. + + .. code:: cpp + + StreamPtr css(F("my long css data")); // no heap/stack allocation + server.toAll(css); + + - ``S2Stream::`` is designed to make a ``Stream::` out of a ``String::`` without copy. + + With examples: + .. code:: cpp + + String helloString("hello"); + S2Stream hello(helloString); + + hello.toAll(Serial); // shows "hello"; + hello.toAll(Serial); // shows nothing, content has already been read + hello.reset(); // reset content pointer + hello.toAll(Serial); // shows "hello"; + hello.reset(3); // reset content pointer to a specific position + hello.toAll(Serial); // shows "lo"; + + hello.setConsume(); + Serial.println(helloString.length()); // shows 5 + hello.toAll(Serial); // shows "hello"; + Serial.println(helloString.length()); // shows 0, string is consumed + + ``StreamString::``, which derives from ``S2Stream`` is now a r/w class: + + .. code:: cpp + + StreamString contentStream; + client.toSize(contentStream, SOME_SIZE); // receives at most SOME_SIZE bytes + + // equivalent to: + String content; + S2Stream contentStream(content); + client.toSize(contentStream, SOME_SIZE); // receives at most SOME_SIZE bytes - internal Stream API: peekBuffer From e47c162d383cc6d78a5800456327f20e850b62d9 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 17 Aug 2020 13:48:21 +0200 Subject: [PATCH 135/207] doc --- doc/reference.rst | 63 +++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/doc/reference.rst b/doc/reference.rst index 35317f7284..bda561b359 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -460,49 +460,48 @@ Stream extensions Two additional classes are proposed. - - ``StreamPtr::`` is designed to hold a constant buffer (in ram or flash). + - ``StreamPtr::`` is designed to hold a constant buffer (in ram or flash). - A `Stream::` is made from `const char*`, `F("some words in flash")` - or `PROGMEM` strings. This class makes no copy, even with data in - flash. For them, byte-by-byte transfers is a consequence. Others - can be transfered at once when possible. + A `Stream::` is made from `const char*`, `F("some words in flash")` or + `PROGMEM` strings. This class makes no copy, even with data in flash. + For them, byte-by-byte transfers is a consequence. Others can be + transfered at once when possible. - .. code:: cpp + .. code:: cpp - StreamPtr css(F("my long css data")); // no heap/stack allocation - server.toAll(css); + StreamPtr css(F("my long css data")); // no heap/stack allocation + server.toAll(css); - - ``S2Stream::`` is designed to make a ``Stream::` out of a ``String::`` without copy. + - ``S2Stream::`` is designed to make a ``Stream::` out of a ``String::`` without copy. - With examples: - .. code:: cpp + With examples: + .. code:: cpp - String helloString("hello"); - S2Stream hello(helloString); + String helloString("hello"); + S2Stream hello(helloString); - hello.toAll(Serial); // shows "hello"; - hello.toAll(Serial); // shows nothing, content has already been read - hello.reset(); // reset content pointer - hello.toAll(Serial); // shows "hello"; - hello.reset(3); // reset content pointer to a specific position - hello.toAll(Serial); // shows "lo"; + hello.toAll(Serial); // shows "hello"; + hello.toAll(Serial); // shows nothing, content has already been read + hello.reset(); // reset content pointer + hello.toAll(Serial); // shows "hello"; + hello.reset(3); // reset content pointer to a specific position + hello.toAll(Serial); // shows "lo"; - hello.setConsume(); - Serial.println(helloString.length()); // shows 5 - hello.toAll(Serial); // shows "hello"; - Serial.println(helloString.length()); // shows 0, string is consumed + hello.setConsume(); + Serial.println(helloString.length()); // shows 5 + hello.toAll(Serial); // shows "hello"; + Serial.println(helloString.length()); // shows 0, string is consumed - ``StreamString::``, which derives from ``S2Stream`` is now a r/w class: + ``StreamString::``, which derives from ``S2Stream`` is now a r/w class: - .. code:: cpp + .. code:: cpp - StreamString contentStream; - client.toSize(contentStream, SOME_SIZE); // receives at most SOME_SIZE bytes + StreamString contentStream; + client.toSize(contentStream, SOME_SIZE); // receives at most SOME_SIZE bytes - // equivalent to: - String content; - S2Stream contentStream(content); - client.toSize(contentStream, SOME_SIZE); // receives at most SOME_SIZE bytes + // equivalent to: + String content; + S2Stream contentStream(content); + client.toSize(contentStream, SOME_SIZE); // receives at most SOME_SIZE bytes - internal Stream API: peekBuffer - From d4f32afe1ca6b351e4d0ce894304cc6213d37ae8 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 17 Aug 2020 13:59:45 +0200 Subject: [PATCH 136/207] doc --- doc/reference.rst | 40 +++++++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/doc/reference.rst b/doc/reference.rst index bda561b359..61ece52e54 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -462,19 +462,18 @@ Stream extensions - ``StreamPtr::`` is designed to hold a constant buffer (in ram or flash). - A `Stream::` is made from `const char*`, `F("some words in flash")` or - `PROGMEM` strings. This class makes no copy, even with data in flash. - For them, byte-by-byte transfers is a consequence. Others can be - transfered at once when possible. + A ``Stream::`` is made from ``const char*``, ``F("some words in + flash")`` or ``PROGMEM`` strings. This class makes no copy, even with + data in flash. For them, byte-by-byte transfers is a consequence. + Others can be transfered at once when possible. .. code:: cpp StreamPtr css(F("my long css data")); // no heap/stack allocation server.toAll(css); - - ``S2Stream::`` is designed to make a ``Stream::` out of a ``String::`` without copy. + - ``S2Stream::`` is designed to make a ``Stream::`` out of a ``String::`` without copy. - With examples: .. code:: cpp String helloString("hello"); @@ -500,8 +499,35 @@ Stream extensions client.toSize(contentStream, SOME_SIZE); // receives at most SOME_SIZE bytes // equivalent to: + String content; S2Stream contentStream(content); client.toSize(contentStream, SOME_SIZE); // receives at most SOME_SIZE bytes + // content has the data + + - Internal Stream API: ``peekBuffer`` + + Here is the method list and their significations. It is currently + implemented in ``HardwareSerial``, ``WiFiClient`` and + ``WiFiClientSecure``. + + - ``virtual bool peekBufferAPI ()`` returns ``true`` when the API is present in the class + + - ``virtual size_t availableForPeek ()`` returns the number of reachable bytes + + - ``virtual const char* peekBuffer ()`` returns the pointer to these bytes + + This API requires that any kind of ``"read"`` function must not be called after ``peekBuffer()`` + and until ``peekConsume()`` is called. + + - ``virtual void peekConsume (size_t consume)`` tells to discard that number of bytes + + - ``virtual bool inputTimeoutPossible ()`` + + A StringStream will return false. A closed network connection returns false. + This function allows ``Stream::toAll()`` to return early. + + - ``virtual bool outputTimeoutPossible ()`` - - internal Stream API: peekBuffer + A closed network connection returns false. + This function allows ``Stream::toAll()`` to return early. From ef998c0fecec5466d26fb23b2cd6deb47fa6c0a6 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 17 Aug 2020 14:08:59 +0200 Subject: [PATCH 137/207] doc --- doc/reference.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/reference.rst b/doc/reference.rst index 61ece52e54..c0b1954874 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -464,8 +464,8 @@ Stream extensions A ``Stream::`` is made from ``const char*``, ``F("some words in flash")`` or ``PROGMEM`` strings. This class makes no copy, even with - data in flash. For them, byte-by-byte transfers is a consequence. - Others can be transfered at once when possible. + data in flash. For flash content, byte-by-byte transfers is a + consequence. Other contents can be transfered at once when possible. .. code:: cpp @@ -479,16 +479,16 @@ Stream extensions String helloString("hello"); S2Stream hello(helloString); - hello.toAll(Serial); // shows "hello"; + hello.toAll(Serial); // shows "hello" hello.toAll(Serial); // shows nothing, content has already been read hello.reset(); // reset content pointer - hello.toAll(Serial); // shows "hello"; + hello.toAll(Serial); // shows "hello" hello.reset(3); // reset content pointer to a specific position - hello.toAll(Serial); // shows "lo"; + hello.toAll(Serial); // shows "lo" hello.setConsume(); Serial.println(helloString.length()); // shows 5 - hello.toAll(Serial); // shows "hello"; + hello.toAll(Serial); // shows "hello" Serial.println(helloString.length()); // shows 0, string is consumed ``StreamString::``, which derives from ``S2Stream`` is now a r/w class: @@ -507,7 +507,7 @@ Stream extensions - Internal Stream API: ``peekBuffer`` - Here is the method list and their significations. It is currently + Here is the method list and their significations. They are currently implemented in ``HardwareSerial``, ``WiFiClient`` and ``WiFiClientSecure``. @@ -524,7 +524,7 @@ Stream extensions - ``virtual bool inputTimeoutPossible ()`` - A StringStream will return false. A closed network connection returns false. + A ``StringStream`` will return false. A closed network connection returns false. This function allows ``Stream::toAll()`` to return early. - ``virtual bool outputTimeoutPossible ()`` From 39f6f563f17b05af56b35f61ccaa8f230aae1d6f Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 17 Aug 2020 14:17:44 +0200 Subject: [PATCH 138/207] doc --- doc/reference.rst | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/doc/reference.rst b/doc/reference.rst index c0b1954874..05a3cb23ef 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -371,27 +371,19 @@ Arduino API .. code:: cpp //check clients for data - for (i = 0; i < MAX_SRV_CLIENTS; i++) { - if (serverClients[i] && serverClients[i].connected()) { - if (serverClients[i].available()) { - //get data from the telnet client and push it to the UART - while (serverClients[i].available()) { - Serial.write(serverClients[i].read()); - } - } - } + //get data from the telnet client and push it to the UART + while (serverClient.available()) { + Serial.write(serverClient.read()); } + //check UART for data if (Serial.available()) { size_t len = Serial.available(); uint8_t sbuf[len]; Serial.readBytes(sbuf, len); //push UART data to all connected telnet clients - for (i = 0; i < MAX_SRV_CLIENTS; i++) { - if (serverClients[i] && serverClients[i].connected()) { - serverClients[i].write(sbuf, len); - delay(1); - } + if (serverClient && serverClient.connected()) { + serverClient.write(sbuf, len); } } @@ -420,6 +412,24 @@ Stream extensions and offer a additional methods to make transfers more efficients and easier to use. + The serial to network transfer above can be written like this: + + .. code:: cpp + + serverClient.toNow(Serial); // chunk by chunk + Serial.toNow(serverClient); // chunk by chunk + + An echo service can be written like this: + + .. code:: cpp + + serverClient.toNow(serverClient); // tcp echo service + + Serial.toNow(Serial); // serial software loopback + + Beside reducing coding time, these methods optimize transfers by avoiding + buffer copies and when possible. + - User facing API: ``Stream::to()`` The goal of streams is to transfer data between producers and consumers, From 112313d2916bdb4169b7b27b8de028dd6ddd0126 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 17 Aug 2020 16:04:46 +0200 Subject: [PATCH 139/207] fix transition from String as Stream to S2Stream --- libraries/ESP8266WebServer/src/Parsing-impl.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/ESP8266WebServer/src/Parsing-impl.h b/libraries/ESP8266WebServer/src/Parsing-impl.h index 72fa44c45c..ab60b1aac5 100644 --- a/libraries/ESP8266WebServer/src/Parsing-impl.h +++ b/libraries/ESP8266WebServer/src/Parsing-impl.h @@ -35,7 +35,8 @@ static const char filename[] PROGMEM = "filename"; template static bool readBytesWithTimeout(typename ServerType::ClientType& client, size_t maxLength, String& data, int timeout_ms) { - return client.toSize(S2Stream(data), maxLength, timeout_ms) == maxLength; + S2Stream dataStream(data); + return client.toSize(dataStream, maxLength, timeout_ms) == maxLength; } template From a378aff1562fb0ce14763e81ea3cac22b9fe7a60 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 17 Aug 2020 16:08:30 +0200 Subject: [PATCH 140/207] doc --- doc/reference.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/reference.rst b/doc/reference.rst index 05a3cb23ef..b41cc43132 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -394,7 +394,7 @@ Arduino API The ``readBytes(buffer, length)`` method includes a timeout to ensure that all required bytes are received. The ``write(buffer, length)`` (inherited - from ``Print::`) function is also usually blocking until the full buffer is + from ``Print::``) function is also usually blocking until the full buffer is transmitted. Both functions return the number of transmitted bytes. That's the way the Stream class works and is commonly used. @@ -428,7 +428,7 @@ Stream extensions Serial.toNow(Serial); // serial software loopback Beside reducing coding time, these methods optimize transfers by avoiding - buffer copies and when possible. + buffer copies when possible. - User facing API: ``Stream::to()`` From f9a5f18cd1ad92999a78e9244e4b1d806dc26836 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 17 Aug 2020 16:27:39 +0200 Subject: [PATCH 141/207] fix webserver::sendContent API --- libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h | 4 ++-- libraries/ESP8266WebServer/src/ESP8266WebServer.h | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index 60ad6816cf..d6f0fb6329 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -513,8 +513,8 @@ template void ESP8266WebServerTemplate::sendContent(Stream* content, ssize_t content_length /* = 0*/) { if (_currentMethod == HTTP_HEAD) return; - if (content_length == 0) - content_length = std::max((ssize_t)0, content->streamSize()); + if (content_length <= 0) + content_length = std::max((ssize_t)0, content->streamSize()); if(_chunked) { _currentClient.printf("%zx\r\n", content_length); } diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer.h b/libraries/ESP8266WebServer/src/ESP8266WebServer.h index 557ee57cdf..909a36d336 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer.h @@ -166,6 +166,9 @@ class ESP8266WebServerTemplate void setContentLength(const size_t contentLength); void sendHeader(const String& name, const String& value, bool first = false); void sendContent(const String& content); + void sendContent(String& content) { + sendContent((const String&)content); + } void sendContent_P(PGM_P content); void sendContent_P(PGM_P content, size_t size); void sendContent(const char *content) { sendContent_P(content); } @@ -173,7 +176,9 @@ class ESP8266WebServerTemplate void sendContent(Stream* content, ssize_t content_length = 0); template - void sendContent(T& content, ssize_t content_length = 0) { sendContent(&content, content_length); } + void sendContent(T& content, ssize_t content_length = 0) { + sendContent(&content, content_length); + } bool chunkedResponseModeStart_P (int code, PGM_P content_type) { if (_currentVersion == 0) From 6d84820d96b0cf5b036b5f64174c3ba24d7c0fb3 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 19 Aug 2020 06:30:49 +0200 Subject: [PATCH 142/207] update doc --- doc/reference.rst | 6 +++++- tests/host/Makefile | 5 +++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/doc/reference.rst b/doc/reference.rst index b41cc43132..735244a2fd 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -501,7 +501,7 @@ Stream extensions hello.toAll(Serial); // shows "hello" Serial.println(helloString.length()); // shows 0, string is consumed - ``StreamString::``, which derives from ``S2Stream`` is now a r/w class: + ``StreamString::``, derives from ``S2Stream`` .. code:: cpp @@ -541,3 +541,7 @@ Stream extensions A closed network connection returns false. This function allows ``Stream::toAll()`` to return early. + + - ``virtual ssize_t streamSize()`` + + -1 when stream size is unknown, depending on implementation (string size, file size..) diff --git a/tests/host/Makefile b/tests/host/Makefile index 1c534356ea..ef64dc3030 100644 --- a/tests/host/Makefile +++ b/tests/host/Makefile @@ -11,6 +11,11 @@ DEFSYM_FS ?= -Wl,--defsym,_FS_start=0x40300000 -Wl,--defsym,_FS_end=0x411FA000 - MAKEFILE = $(word 1, $(MAKEFILE_LIST)) +ifeq ($(shell g++-8 -v; echo $$?),0) +CC = gcc-8 +CXX = g++-8 +endif + # I wasn't able to build with clang when -coverage flag is enabled, forcing GCC on OS X ifeq ($(shell uname -s),Darwin) CC ?= gcc From 9bc8146675f17e3ff71cc1c7bc7d4dd45dbf42e3 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 19 Aug 2020 06:49:06 +0200 Subject: [PATCH 143/207] StreamString's read by default swallows the string (like legacy) --- cores/esp8266/StreamString.h | 14 +++++++------- .../esp8266/examples/StreamString/StreamString.ino | 8 ++++++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/cores/esp8266/StreamString.h b/cores/esp8266/StreamString.h index 5ae0e675a4..78fad52874 100644 --- a/cores/esp8266/StreamString.h +++ b/cores/esp8266/StreamString.h @@ -33,11 +33,13 @@ class S2Stream: public Stream { public: - S2Stream(String& string): string(&string) + S2Stream(String& string, int peekPointer = -1): + string(&string), peekPointer(peekPointer) { } - S2Stream(String* string): string(string) + S2Stream(String* string, int peekPointer = -1): + string(string), peekPointer(peekPointer) { } @@ -187,14 +189,14 @@ class S2Stream: public Stream } // calling setConsume() will consume bytes as the stream is read - // (not enabled by default) + // (enabled by default) void setConsume() { peekPointer = -1; } // Reading this stream will mark the string as read without consuming - // This is the default. + // (not enabled by default) // Calling reset() resets the read state and allows rereading. void reset(int pointer = 0) { @@ -204,9 +206,7 @@ class S2Stream: public Stream protected: String* string; - - // default (0): read marks as read without consuming(erasing) the string: - int peekPointer = 0; + int peekPointer; // -1:String is consumed / >=0:resettable pointer }; diff --git a/libraries/esp8266/examples/StreamString/StreamString.ino b/libraries/esp8266/examples/StreamString/StreamString.ino index ef44b87b06..cbd9b0b9a8 100644 --- a/libraries/esp8266/examples/StreamString/StreamString.ino +++ b/libraries/esp8266/examples/StreamString/StreamString.ino @@ -34,6 +34,9 @@ void testStream() { String inputString = "hello"; StreamString result; + // string will not be modified by read: + result.reset(); + { // use StreamString or S2Stream(String) to make a r/w Stream out of a String, // prefer the lighter StreamPtr(String) to make a read-only Stream out of a String @@ -55,12 +58,13 @@ void testStream() { } { - // inputString made into a Stream + // inputString made into a Stream, in reset mode // after input is loaded once, there's nothing to get from the stream // but the String is left untouched result.clear(); S2Stream input(inputString); + input.reset(); input.toAll(result); input.toAll(result); @@ -90,7 +94,7 @@ void testStream() { result.clear(); S2Stream input(inputString); // reading stream will consume the string - input.setConsume(); + input.setConsume(); // can be ommitted, this is the default input.toSize(result, 1); input.toSize(result, 2); From a99c41849ca451b8b1aaa682b2cf6d2f766e5620 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 19 Aug 2020 06:53:04 +0200 Subject: [PATCH 144/207] doc update --- doc/reference.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/reference.rst b/doc/reference.rst index 735244a2fd..608fe7f9e3 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -488,6 +488,7 @@ Stream extensions String helloString("hello"); S2Stream hello(helloString); + hello.reset(0); // prevents ::read() to consume the string hello.toAll(Serial); // shows "hello" hello.toAll(Serial); // shows nothing, content has already been read @@ -496,7 +497,7 @@ Stream extensions hello.reset(3); // reset content pointer to a specific position hello.toAll(Serial); // shows "lo" - hello.setConsume(); + hello.setConsume(); // ::read() will consume, this is the default Serial.println(helloString.length()); // shows 5 hello.toAll(Serial); // shows "hello" Serial.println(helloString.length()); // shows 0, string is consumed From b24b7eff95a68df4a48e41048f71c5a05b647b4f Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 25 Aug 2020 17:50:36 +0200 Subject: [PATCH 145/207] fix api --- cores/esp8266/Stream.h | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 55d3dfa4d7..c084e28a89 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -167,26 +167,22 @@ class Stream: public Print { // transfers already buffered / immediately available data (no timeout) // returns number of transfered bytes size_t toNow (Print* to) { return toFull(to, -1, -1, oneShotMs::alwaysExpired); } - template - size_t toNow (T& to) { return toNow((Print*)&to); } + size_t toNow (Print& to) { return toNow(&to); } // transfers data until timeout // returns number of transfered bytes size_t toAll (Print* to, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toFull(to, -1, -1, timeoutMs); } - template - size_t toAll (T& to, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toAll((Print*)&to, timeoutMs); } + size_t toAll (Print& to, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toAll(&to, timeoutMs); } // transfers data until a char is encountered (the char is swallowed but not transfered) with timeout // returns number of transfered bytes size_t toUntil (Print* to, const int readUntilChar, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toFull(to, -1, readUntilChar, timeoutMs); } - template - size_t toUntil (T& to, const int readUntilChar, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toUntil((Print*)&to, readUntilChar, timeoutMs); } + size_t toUntil (Print& to, const int readUntilChar, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toUntil(&to, readUntilChar, timeoutMs); } // transfers data until requested size or timeout // returns number of transfered bytes size_t toSize (Print* to, const ssize_t maxLen, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toFull(to, maxLen, -1, timeoutMs); } - template - size_t toSize (T& to, const ssize_t maxLen, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toSize((Print*)&to, maxLen, timeoutMs); } + size_t toSize (Print& to, const ssize_t maxLen, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toSize(&to, maxLen, timeoutMs); } // remaining size (-1 by default = unknown) virtual ssize_t streamSize () { return -1; } From 0eb0b5541363a89cae6b3bab3f50866169f168f3 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 25 Aug 2020 17:54:31 +0200 Subject: [PATCH 146/207] fix << --- cores/esp8266/StreamDev.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index 0e7c58eeab..cad0742e14 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -324,7 +324,7 @@ Stream& operator << (Stream& out, const char* text) Stream& operator << (Stream& out, const __FlashStringHelper* text) { - StreamPtr(text).toAll(text); + StreamPtr(text).toAll(out); return out; } From 211c914f63a3d4a9e87299d7b141bf6a6e293d8a Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 21 Sep 2020 23:08:02 +0200 Subject: [PATCH 147/207] fix int/size_t changes in Netdump --- libraries/Netdump/src/Netdump.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/Netdump/src/Netdump.h b/libraries/Netdump/src/Netdump.h index 1011a8e955..1dce190167 100644 --- a/libraries/Netdump/src/Netdump.h +++ b/libraries/Netdump/src/Netdump.h @@ -75,7 +75,7 @@ class Netdump WiFiClient tcpDumpClient; char* packetBuffer = nullptr; - size_t bufferIndex = 0; + int bufferIndex = 0; static constexpr int tcpBufferSize = 2048; static constexpr int maxPcapLength = 1024; From 826880732de208a328cca4e3c4959c98690e9059 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 24 Oct 2020 22:04:26 +0200 Subject: [PATCH 148/207] fix emulation on host --- tests/host/common/mock.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/host/common/mock.h b/tests/host/common/mock.h index f96194a345..bd123427d6 100644 --- a/tests/host/common/mock.h +++ b/tests/host/common/mock.h @@ -114,6 +114,8 @@ extern uint32_t global_source_address; // 0 = INADDR_ANY by default #define NO_GLOBAL_BINDING 0xffffffff extern uint32_t global_ipv4_netfmt; // selected interface addresse to bind to +void init_milliscros (); + #ifdef __cplusplus } #endif From 9eca03c8d177d9d64609445acf89a5b0c9c3392b Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 9 Nov 2020 14:08:57 +0100 Subject: [PATCH 149/207] sync with master --- cores/esp8266/Print.h | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/cores/esp8266/Print.h b/cores/esp8266/Print.h index 150ae04f47..311b4a9b39 100644 --- a/cores/esp8266/Print.h +++ b/cores/esp8266/Print.h @@ -22,13 +22,10 @@ #include #include -#include -#include +#include "WString.h" #include "Printable.h" -class String; -class __FlashStringHelper; #include "stdlib_noniso.h" #define DEC 10 @@ -115,10 +112,6 @@ class Print { virtual void flush() { /* Empty implementation for backward compatibility */ } - // default to zero, meaning "a single write may block" - // should be overriden by subclasses with buffering - virtual int availableForWrite() { return 0; } - // by default write timeout is possible (outgoing data from network,serial..) // (children can override to false (like String)) virtual bool outputTimeoutPossible () { return true; } From e24e26cda7511a8f1760d201f7d103550013d6ec Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 22 Dec 2020 01:56:34 +0100 Subject: [PATCH 150/207] update softserial --- libraries/SoftwareSerial | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/SoftwareSerial b/libraries/SoftwareSerial index 5ba1e68d3a..7389070e22 160000 --- a/libraries/SoftwareSerial +++ b/libraries/SoftwareSerial @@ -1 +1 @@ -Subproject commit 5ba1e68d3a0390304202de94461af071e2526bbd +Subproject commit 7389070e2223297c46ba7ae915e4a8309f66cb7e From cde30e4f49ad51c28e2c581a31ea352eb6ab34f2 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 22 Dec 2020 01:57:54 +0100 Subject: [PATCH 151/207] uodate SoftwareSerial --- libraries/SoftwareSerial | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/SoftwareSerial b/libraries/SoftwareSerial index 7389070e22..34dac630eb 160000 --- a/libraries/SoftwareSerial +++ b/libraries/SoftwareSerial @@ -1 +1 @@ -Subproject commit 7389070e2223297c46ba7ae915e4a8309f66cb7e +Subproject commit 34dac630eb3f4c20105e621254aa0728733e55b0 From 0f5d28382752db9d20219115c84e3c695f4ef36b Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 20 Jan 2021 00:58:40 +0100 Subject: [PATCH 152/207] sserial update --- libraries/SoftwareSerial | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/SoftwareSerial b/libraries/SoftwareSerial index 320f033d6b..a174c00e59 160000 --- a/libraries/SoftwareSerial +++ b/libraries/SoftwareSerial @@ -1 +1 @@ -Subproject commit 320f033d6b77977c79ef4c02f5968d61dbab3e68 +Subproject commit a174c00e59662a598d2c273f80a8fe522fe8d9f5 From ca821a989e0d5f160492c3861608f67a68032515 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 24 Jan 2021 23:07:34 +0100 Subject: [PATCH 153/207] replace "periodicFastMs yieldNow(5)" by optimistic_yield(1000) --- cores/esp8266/StreamDev.cpp | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index cad0742e14..1fa699b5eb 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -50,9 +50,6 @@ size_t Stream::toFull(Print* to, // "neverExpires (default, impossible)" is translated to default timeout oneShotFastMs timedOut(timeoutMs >= oneShotFastMs::neverExpires ? getTimeout() : timeoutMs); - // yield about every 5ms (XXX SHOULD BE A SYSTEM-WIDE CONSTANT?) - periodicFastMs yieldNow(5); - size_t written = 0; // len==-1 => maxLen=0 <=> until starvation @@ -126,10 +123,7 @@ size_t Stream::toFull(Print* to, break; } - if (yieldNow) - { - yield(); - } + optimistic_yield(1000); } else if (readUntilChar >= 0) @@ -187,10 +181,7 @@ size_t Stream::toFull(Print* to, break; } - if (yieldNow) - { - yield(); - } + optimistic_yield(1000); } else @@ -256,10 +247,7 @@ size_t Stream::toFull(Print* to, break; } - if (yieldNow) - { - yield(); - } + optimistic_yield(1000); } if (getWriteError() == STREAMTO_SUCCESS && maxLen > 0) From 3a308c2d04358e26df6df2168a1ba329d9a77929 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 24 Jan 2021 23:08:09 +0100 Subject: [PATCH 154/207] style --- cores/esp8266/debug.cpp | 2 +- cores/esp8266/debug.h | 2 +- tests/restyle.sh | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cores/esp8266/debug.cpp b/cores/esp8266/debug.cpp index 7e861f0579..7cffc08cfc 100644 --- a/cores/esp8266/debug.cpp +++ b/cores/esp8266/debug.cpp @@ -23,7 +23,7 @@ #include "osapi.h" #ifdef DEBUG_ESP_CORE -void iamslow (const char* what) +void iamslow(const char* what) { DEBUGV("%s should be overridden for better efficiency\r\n", what); } diff --git a/cores/esp8266/debug.h b/cores/esp8266/debug.h index d6cba4fbab..e34762a700 100644 --- a/cores/esp8266/debug.h +++ b/cores/esp8266/debug.h @@ -27,7 +27,7 @@ void __panic_func(const char* file, int line, const char* func) __attribute__((n #define panic() __panic_func(PSTR(__FILE__), __LINE__, __func__) #ifdef DEBUG_ESP_CORE -extern void iamslow (const char* what); +extern void iamslow(const char* what); #define IAMSLOW() \ do { \ static bool once = false; \ diff --git a/tests/restyle.sh b/tests/restyle.sh index 242b3897f3..2b550cb49f 100755 --- a/tests/restyle.sh +++ b/tests/restyle.sh @@ -15,11 +15,11 @@ libraries/ESP8266mDNS libraries/Wire libraries/lwIP* cores/esp8266/Lwip* -cores/esp8266/core_esp8266_si2c.cpp cores/esp8266/debug* -libraries/Netdump cores/esp8266/core_esp8266_si2c.cpp -cores/esp8266/StreamString.h cores/esp8266/StreamDev.h cores/esp8266/StreamDev.cpp +cores/esp8266/StreamString.* +cores/esp8266/StreamDev.* +libraries/Netdump " # core From afd595432a67c2c52ed4721ccefccd9732948b34 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 25 Jan 2021 09:28:33 +0100 Subject: [PATCH 155/207] availableForPeek => peekAvailable --- cores/esp8266/HardwareSerial.h | 2 +- cores/esp8266/Stream.h | 4 ++-- cores/esp8266/StreamDev.cpp | 4 ++-- cores/esp8266/StreamDev.h | 4 ++-- cores/esp8266/StreamString.h | 2 +- doc/reference.rst | 2 +- libraries/ESP8266WiFi/src/WiFiClient.cpp | 6 +++--- libraries/ESP8266WiFi/src/WiFiClient.h | 4 ++-- .../ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp | 2 +- libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h | 8 ++++---- libraries/ESP8266WiFi/src/include/ClientContext.h | 12 ++++++------ tests/host/common/ClientContextSocket.cpp | 4 ++-- tests/host/common/include/ClientContext.h | 4 ++-- 13 files changed, 29 insertions(+), 29 deletions(-) diff --git a/cores/esp8266/HardwareSerial.h b/cores/esp8266/HardwareSerial.h index 5be6de803a..6ff4562df1 100644 --- a/cores/esp8266/HardwareSerial.h +++ b/cores/esp8266/HardwareSerial.h @@ -149,7 +149,7 @@ class HardwareSerial: public Stream } // return number of byte accessible by peekBuffer() - size_t availableForPeek () override + size_t peekAvailable () override { return uart_peek_available(_uart); } diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 1c005b0377..b7783fb228 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -124,9 +124,9 @@ class Stream: public Print { virtual bool peekBufferAPI () const { return false; } // returns number of byte accessible by peekBuffer() - virtual size_t availableForPeek () { return 0; } + virtual size_t peekAvailable () { return 0; } - // returns a pointer to available data buffer (size = availableForPeek()) + // returns a pointer to available data buffer (size = peekAvailable()) // semantic forbids any kind of ::read() // - after calling peekBuffer() // - and before calling peekConsume() diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index 1fa699b5eb..5ce7f715b7 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -61,7 +61,7 @@ size_t Stream::toFull(Print* to, while (!maxLen || written < maxLen) { - size_t avpk = availableForPeek(); + size_t avpk = peekAvailable(); if (avpk == 0 && !inputTimeoutPossible()) { // no more data to read, ever @@ -133,7 +133,7 @@ size_t Stream::toFull(Print* to, while (!maxLen || written < maxLen) { - size_t avpk = availableForPeek(); + size_t avpk = peekAvailable(); if (avpk == 0 && !inputTimeoutPossible()) { // no more data to read, ever diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index da5c961609..29697332e5 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -174,7 +174,7 @@ class StreamPtr: public StreamNull // Stream virtual int available() override { - return availableForPeek(); + return peekAvailable(); } virtual int read() override @@ -222,7 +222,7 @@ class StreamPtr: public StreamNull return !_in_flash; } - virtual size_t availableForPeek() override + virtual size_t peekAvailable() override { return _peekPointer < _size ? _size - _peekPointer : 0; } diff --git a/cores/esp8266/StreamString.h b/cores/esp8266/StreamString.h index 78fad52874..5c809760e8 100644 --- a/cores/esp8266/StreamString.h +++ b/cores/esp8266/StreamString.h @@ -147,7 +147,7 @@ class S2Stream: public Stream return true; } - virtual size_t availableForPeek() + virtual size_t peekAvailable() { if (peekPointer < 0) { diff --git a/doc/reference.rst b/doc/reference.rst index 6aed3a4fef..35dbe08907 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -491,7 +491,7 @@ Stream extensions - ``virtual bool peekBufferAPI ()`` returns ``true`` when the API is present in the class - - ``virtual size_t availableForPeek ()`` returns the number of reachable bytes + - ``virtual size_t peekAvailable ()`` returns the number of reachable bytes - ``virtual const char* peekBuffer ()`` returns the pointer to these bytes diff --git a/libraries/ESP8266WiFi/src/WiFiClient.cpp b/libraries/ESP8266WiFi/src/WiFiClient.cpp index aa25834a12..1a26ef7386 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.cpp +++ b/libraries/ESP8266WiFi/src/WiFiClient.cpp @@ -422,7 +422,7 @@ bool WiFiClient::peekBufferAPI () const return true; } -// return a pointer to available data buffer (size = availableForPeek()) +// return a pointer to available data buffer (size = peekAvailable()) // semantic forbids any kind of read() before calling peekConsume() const char* WiFiClient::peekBuffer () { @@ -430,9 +430,9 @@ const char* WiFiClient::peekBuffer () } // return number of byte accessible by peekBuffer() -size_t WiFiClient::availableForPeek () +size_t WiFiClient::peekAvailable () { - return _client? _client->availableForPeek(): 0; + return _client? _client->peekAvailable(): 0; } // consume bytes after use (see peekBuffer) diff --git a/libraries/ESP8266WiFi/src/WiFiClient.h b/libraries/ESP8266WiFi/src/WiFiClient.h index 71ef9610ec..4d44da57a7 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.h +++ b/libraries/ESP8266WiFi/src/WiFiClient.h @@ -126,9 +126,9 @@ class WiFiClient : public Client, public SList { virtual bool peekBufferAPI () const override; // return number of byte accessible by peekBuffer() - virtual size_t availableForPeek () override; + virtual size_t peekAvailable () override; - // return a pointer to available data buffer (size = availableForPeek()) + // return a pointer to available data buffer (size = peekAvailable()) // semantic forbids any kind of read() before calling peekConsume() virtual const char* peekBuffer () override; diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp index 2a455deaa1..08dc3784a7 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp @@ -362,7 +362,7 @@ int WiFiClientSecureCtx::read(uint8_t *buf, size_t size) { return 0; // If we're connected, no error but no read. } -// return a pointer to available data buffer (size = availableForPeek()) +// return a pointer to available data buffer (size = peekAvailable()) // semantic forbids any kind of read() before calling peekConsume() const char* WiFiClientSecureCtx::peekBuffer () { diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h index 7ac9c3f400..607c8b2988 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h @@ -125,9 +125,9 @@ class WiFiClientSecureCtx : public WiFiClient { virtual bool peekBufferAPI () const override { return true; } // return number of byte accessible by peekBuffer() - virtual size_t availableForPeek () override { return WiFiClientSecureCtx::available(); } + virtual size_t peekAvailable () override { return WiFiClientSecureCtx::available(); } - // return a pointer to available data buffer (size = availableForPeek()) + // return a pointer to available data buffer (size = peekAvailable()) // semantic forbids any kind of read() before calling peekConsume() virtual const char* peekBuffer () override; @@ -305,9 +305,9 @@ class WiFiClientSecure : public WiFiClient { virtual bool peekBufferAPI () const override { return true; } // return number of byte accessible by peekBuffer() - virtual size_t availableForPeek () override { return _ctx->available(); } + virtual size_t peekAvailable () override { return _ctx->available(); } - // return a pointer to available data buffer (size = availableForPeek()) + // return a pointer to available data buffer (size = peekAvailable()) // semantic forbids any kind of read() before calling peekConsume() virtual const char* peekBuffer () override { return _ctx->peekBuffer(); } diff --git a/libraries/ESP8266WiFi/src/include/ClientContext.h b/libraries/ESP8266WiFi/src/include/ClientContext.h index 092f3d923a..040ebb0c5f 100644 --- a/libraries/ESP8266WiFi/src/include/ClientContext.h +++ b/libraries/ESP8266WiFi/src/include/ClientContext.h @@ -438,7 +438,7 @@ class ClientContext _sync = sync; } - // return a pointer to available data buffer (size = availableForPeek()) + // return a pointer to available data buffer (size = peekAvailable()) // semantic forbids any kind of read() before calling peekConsume() const char* peekBuffer () { @@ -448,7 +448,7 @@ class ClientContext } // return number of byte accessible by peekBuffer() - size_t availableForPeek () + size_t peekAvailable () { if (!_rx_buf) return 0; @@ -519,20 +519,20 @@ class ClientContext return false; } - DEBUGV(":wr %d %d\r\n", _datasource->availableForPeek(), _written); + DEBUGV(":wr %d %d\r\n", _datasource->peekAvailable(), _written); bool has_written = false; while (_datasource) { if (state() == CLOSED) return false; - size_t next_chunk_size = std::min((size_t)tcp_sndbuf(_pcb), _datasource->availableForPeek()); + size_t next_chunk_size = std::min((size_t)tcp_sndbuf(_pcb), _datasource->peekAvailable()); if (!next_chunk_size) break; const char* buf = _datasource->peekBuffer(); uint8_t flags = 0; - if (next_chunk_size < _datasource->availableForPeek()) + if (next_chunk_size < _datasource->peekAvailable()) // PUSH is meant for peer, telling to give data to user app as soon as received // PUSH "may be set" when sender has finished sending a "meaningful" data block // PUSH does not break Nagle @@ -546,7 +546,7 @@ class ClientContext err_t err = tcp_write(_pcb, buf, next_chunk_size, flags); - DEBUGV(":wrc %d %d %d\r\n", next_chunk_size, _datasource->availableForPeek(), (int)err); + DEBUGV(":wrc %d %d %d\r\n", next_chunk_size, _datasource->peekAvailable(), (int)err); if (err == ERR_OK) { _datasource->peekConsume(next_chunk_size); diff --git a/tests/host/common/ClientContextSocket.cpp b/tests/host/common/ClientContextSocket.cpp index 3ff7293540..2d1d7c6e83 100644 --- a/tests/host/common/ClientContextSocket.cpp +++ b/tests/host/common/ClientContextSocket.cpp @@ -110,7 +110,7 @@ ssize_t mockFillInBuf (int sock, char* ccinbuf, size_t& ccinbufsize) ssize_t mockPeekBytes (int sock, char* dst, size_t usersize, int timeout_ms, char* ccinbuf, size_t& ccinbufsize) { - // usersize==0: availableForPeek() + // usersize==0: peekAvailable() if (usersize > CCBUFSIZE) mockverbose("CCBUFSIZE(%d) should be increased by %zd bytes (-> %zd)\n", CCBUFSIZE, usersize - CCBUFSIZE, usersize); @@ -133,7 +133,7 @@ ssize_t mockPeekBytes (int sock, char* dst, size_t usersize, int timeout_ms, cha } if (usersize == 0 && ccinbufsize) - // availableForPeek + // peekAvailable return ccinbufsize; if (usersize <= ccinbufsize) diff --git a/tests/host/common/include/ClientContext.h b/tests/host/common/include/ClientContext.h index 8d295a9a45..4eaee8acca 100644 --- a/tests/host/common/include/ClientContext.h +++ b/tests/host/common/include/ClientContext.h @@ -300,7 +300,7 @@ class ClientContext _sync = sync; } - // return a pointer to available data buffer (size = availableForPeek()) + // return a pointer to available data buffer (size = peekAvailable()) // semantic forbids any kind of read() before calling peekConsume() const char* peekBuffer () { @@ -308,7 +308,7 @@ class ClientContext } // return number of byte accessible by peekBuffer() - size_t availableForPeek () + size_t peekAvailable () { ssize_t ret = mockPeekBytes(_sock, nullptr, 0, 0, _inbuf, _inbufsize); if (ret < 0) From e786a13b665d401591498b317f0c06299c92029d Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 25 Jan 2021 09:48:33 +0100 Subject: [PATCH 156/207] remove user-obfuscating "in_flash" parameter --- cores/esp8266/StreamDev.h | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 29697332e5..44e982bfa8 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -156,15 +156,15 @@ class StreamPtr: public StreamNull protected: const char* _buffer; size_t _size; - bool _in_flash; + bool _byteAddressible; size_t _peekPointer = 0; public: - StreamPtr(const String& string): _buffer(string.c_str()), _size(string.length()), _in_flash(false) { } - StreamPtr(const char* buffer, size_t size, bool in_flash = false): _buffer(buffer), _size(size), _in_flash(in_flash) { } - StreamPtr(const uint8_t* buffer, size_t size, bool in_flash = false): _buffer((const char*)buffer), _size(size), _in_flash(in_flash) { } - StreamPtr(const __FlashStringHelper* buffer, size_t size): _buffer(reinterpret_cast(buffer)), _size(size), _in_flash(true) { } - StreamPtr(const __FlashStringHelper* text): _buffer(reinterpret_cast(text)), _size(strlen_P((PGM_P)text)), _in_flash(true) { } + StreamPtr(const String& string): _buffer(string.c_str()), _size(string.length()), _byteAddressible(true) { } + StreamPtr(const char* buffer, size_t size): _buffer(buffer), _size(size), _byteAddressible(buffer < 0x4000000) { } + StreamPtr(const uint8_t* buffer, size_t size): _buffer((const char*)buffer), _size(size), _byteAddressible(buffer < 0x4000000) { } + StreamPtr(const __FlashStringHelper* buffer, size_t size): _buffer(reinterpret_cast(buffer)), _size(size), _byteAddressible(false) { } + StreamPtr(const __FlashStringHelper* text): _buffer(reinterpret_cast(text)), _size(strlen_P((PGM_P)text)), _byteAddressible(false) { } void reset(int pointer = 0) { @@ -194,14 +194,7 @@ class StreamPtr: public StreamNull return 0; } size_t cpylen = std::min(_size - _peekPointer, len); - if (_in_flash) - { - memcpy_P(buffer, _buffer + _peekPointer, cpylen); - } - else - { - memcpy(buffer, _buffer + _peekPointer, cpylen); - } + memcpy_P(buffer, _buffer + _peekPointer, cpylen); // whether byte adressible is true _peekPointer += cpylen; return cpylen; } @@ -219,7 +212,7 @@ class StreamPtr: public StreamNull // peekBuffer virtual bool peekBufferAPI() const override { - return !_in_flash; + return _byteAddressible; } virtual size_t peekAvailable() override From 5e4342d9770a9324bc323416a0f1f9a65b81dcc9 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 25 Jan 2021 17:03:01 +0100 Subject: [PATCH 157/207] fix byte-addressible address --- cores/esp8266/StreamDev.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 44e982bfa8..f1e227844a 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -161,8 +161,8 @@ class StreamPtr: public StreamNull public: StreamPtr(const String& string): _buffer(string.c_str()), _size(string.length()), _byteAddressible(true) { } - StreamPtr(const char* buffer, size_t size): _buffer(buffer), _size(size), _byteAddressible(buffer < 0x4000000) { } - StreamPtr(const uint8_t* buffer, size_t size): _buffer((const char*)buffer), _size(size), _byteAddressible(buffer < 0x4000000) { } + StreamPtr(const char* buffer, size_t size): _buffer(buffer), _size(size), _byteAddressible(buffer < 0x40000000) { } + StreamPtr(const uint8_t* buffer, size_t size): _buffer((const char*)buffer), _size(size), _byteAddressible(buffer < 0x40000000) { } StreamPtr(const __FlashStringHelper* buffer, size_t size): _buffer(reinterpret_cast(buffer)), _size(size), _byteAddressible(false) { } StreamPtr(const __FlashStringHelper* text): _buffer(reinterpret_cast(text)), _size(strlen_P((PGM_P)text)), _byteAddressible(false) { } From ad580e072a164aa933f239deda9774d449154a55 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 25 Jan 2021 22:33:01 +0100 Subject: [PATCH 158/207] uses system constants instead of numerical constants --- cores/esp8266/StreamDev.h | 5 +++-- cores/esp8266/esp_priv.h | 25 +++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 cores/esp8266/esp_priv.h diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index f1e227844a..9f8473aa74 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -22,6 +22,7 @@ #ifndef __STREAMDEV_H #define __STREAMDEV_H +#include #include /////////////////////////////////////////////// @@ -161,8 +162,8 @@ class StreamPtr: public StreamNull public: StreamPtr(const String& string): _buffer(string.c_str()), _size(string.length()), _byteAddressible(true) { } - StreamPtr(const char* buffer, size_t size): _buffer(buffer), _size(size), _byteAddressible(buffer < 0x40000000) { } - StreamPtr(const uint8_t* buffer, size_t size): _buffer((const char*)buffer), _size(size), _byteAddressible(buffer < 0x40000000) { } + StreamPtr(const char* buffer, size_t size): _buffer(buffer), _size(size), _byteAddressible(__byteAddressible(buffer)) { } + StreamPtr(const uint8_t* buffer, size_t size): _buffer((const char*)buffer), _size(size), _byteAddressible(__byteAddressible(buffer)) { } StreamPtr(const __FlashStringHelper* buffer, size_t size): _buffer(reinterpret_cast(buffer)), _size(size), _byteAddressible(false) { } StreamPtr(const __FlashStringHelper* text): _buffer(reinterpret_cast(text)), _size(strlen_P((PGM_P)text)), _byteAddressible(false) { } diff --git a/cores/esp8266/esp_priv.h b/cores/esp8266/esp_priv.h new file mode 100644 index 0000000000..d460071c9e --- /dev/null +++ b/cores/esp8266/esp_priv.h @@ -0,0 +1,25 @@ + +#ifndef __ESP_PRIV +#define __ESP_PRIV + +#if defined(CORE_MOCK) + +constexpr bool __byteAddressible(const void* addr) +{ + (void)addr; + return true; +} + +#else // on hardware + +#include + +// returns true when addr can be used without "pgm_" functions or non32xfer service +constexpr bool __byteAddressible(const void* addr) +{ + return addr < (const void*)(XCHAL_DATARAM0_VADDR + XCHAL_DATARAM0_SIZE); +} + +#endif // on hardware + +#endif // __ESP_PRIV From 8e064be387e1501fa520c6f78e8c02d249999de9 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 25 Jan 2021 22:50:52 +0100 Subject: [PATCH 159/207] good practice: add '__' in front of internal function name --- cores/esp8266/debug.cpp | 2 +- cores/esp8266/debug.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cores/esp8266/debug.cpp b/cores/esp8266/debug.cpp index 7cffc08cfc..4acee8443f 100644 --- a/cores/esp8266/debug.cpp +++ b/cores/esp8266/debug.cpp @@ -23,7 +23,7 @@ #include "osapi.h" #ifdef DEBUG_ESP_CORE -void iamslow(const char* what) +void __iamslow(const char* what) { DEBUGV("%s should be overridden for better efficiency\r\n", what); } diff --git a/cores/esp8266/debug.h b/cores/esp8266/debug.h index e34762a700..c61aacf7ff 100644 --- a/cores/esp8266/debug.h +++ b/cores/esp8266/debug.h @@ -27,14 +27,14 @@ void __panic_func(const char* file, int line, const char* func) __attribute__((n #define panic() __panic_func(PSTR(__FILE__), __LINE__, __func__) #ifdef DEBUG_ESP_CORE -extern void iamslow(const char* what); +extern void __iamslow(const char* what); #define IAMSLOW() \ do { \ static bool once = false; \ if (!once) { \ once = true; \ /*if (__GXX_RTTI) DEBUGV(typeid(*this).name());*/ \ - iamslow((PGM_P)FPSTR(__FUNCTION__)); \ + __iamslow((PGM_P)FPSTR(__FUNCTION__)); \ } \ } while (0) #else From 93c8cc7fba3c561b0da8be65a0b49e6a0d154771 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 25 Jan 2021 23:02:57 +0100 Subject: [PATCH 160/207] doc update, removed "propose" --- doc/reference.rst | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/doc/reference.rst b/doc/reference.rst index 35dbe08907..86800cd421 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -375,9 +375,9 @@ Arduino API Stream extensions - Proposed Stream extensions are designed to be compatible with Arduino API, - and offer a additional methods to make transfers more efficients and easier - to use. + Stream extensions are designed to be compatible with Arduino API, and + offer additional methods to make transfers more efficient and easier to + use. The serial to network transfer above can be written like this: @@ -400,7 +400,7 @@ Stream extensions - User facing API: ``Stream::to()`` The goal of streams is to transfer data between producers and consumers, - like the telnet/serial example above. Four methods are proposed, all of + like the telnet/serial example above. Four methods are provided, all of them return the number of transmitted bytes: - ``Stream::toSize(dest, size [, timeout])`` @@ -435,14 +435,15 @@ Stream extensions - String, flash strings helpers - Two additional classes are proposed. + Two additional classes are provided. - ``StreamPtr::`` is designed to hold a constant buffer (in ram or flash). - A ``Stream::`` is made from ``const char*``, ``F("some words in - flash")`` or ``PROGMEM`` strings. This class makes no copy, even with - data in flash. For flash content, byte-by-byte transfers is a - consequence. Other contents can be transfered at once when possible. + With this class, a ``Stream::`` can be made from ``const char*``, + ``F("some words in flash")`` or ``PROGMEM`` strings. This class makes + no copy, even with data in flash. For flash content, byte-by-byte + transfers is a consequence when "memcpy_P" cannot be used. Other + contents can be transfered at once when possible. .. code:: cpp @@ -512,4 +513,5 @@ Stream extensions - ``virtual ssize_t streamSize()`` - -1 when stream size is unknown, depending on implementation (string size, file size..) + It returns -1 when stream size is unknown, depending on implementation + (string size, file size..). From 30be04c98f42a4d5a15c933779bbc7dd465d941e Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 25 Jan 2021 23:20:34 +0100 Subject: [PATCH 161/207] rename: ::to => ::send --- cores/esp8266/Stream.h | 28 +++++++++---------- cores/esp8266/StreamDev.cpp | 23 ++++++++------- doc/reference.rst | 2 +- .../src/ESP8266HTTPClient.cpp | 6 ++-- .../src/ESP8266WebServer-impl.h | 2 +- .../ESP8266WebServer/src/ESP8266WebServer.h | 2 +- libraries/ESP8266WebServer/src/Parsing-impl.h | 2 +- .../examples/WiFiEcho/WiFiEcho.ino | 6 ++-- 8 files changed, 35 insertions(+), 36 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index b7783fb228..d9599bc1be 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -164,23 +164,23 @@ class Stream: public Print { // transfers already buffered / immediately available data (no timeout) // returns number of transfered bytes - size_t toNow (Print* to) { return toFull(to, -1, -1, oneShotMs::alwaysExpired); } - size_t toNow (Print& to) { return toNow(&to); } + size_t sendNow (Print* to) { return sendGeneric(to, -1, -1, oneShotMs::alwaysExpired); } + size_t sendNow (Print& to) { return sendNow(&to); } // transfers data until timeout // returns number of transfered bytes - size_t toAll (Print* to, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toFull(to, -1, -1, timeoutMs); } - size_t toAll (Print& to, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toAll(&to, timeoutMs); } + size_t sendAll (Print* to, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return sendGeneric(to, -1, -1, timeoutMs); } + size_t sendAll (Print& to, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return sendAll(&to, timeoutMs); } // transfers data until a char is encountered (the char is swallowed but not transfered) with timeout // returns number of transfered bytes - size_t toUntil (Print* to, const int readUntilChar, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toFull(to, -1, readUntilChar, timeoutMs); } - size_t toUntil (Print& to, const int readUntilChar, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toUntil(&to, readUntilChar, timeoutMs); } + size_t sendUntil (Print* to, const int readUntilChar, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return sendGeneric(to, -1, readUntilChar, timeoutMs); } + size_t sendUntil (Print& to, const int readUntilChar, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return sendUntil(&to, readUntilChar, timeoutMs); } // transfers data until requested size or timeout // returns number of transfered bytes - size_t toSize (Print* to, const ssize_t maxLen, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toFull(to, maxLen, -1, timeoutMs); } - size_t toSize (Print& to, const ssize_t maxLen, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return toSize(&to, maxLen, timeoutMs); } + size_t sendSize (Print* to, const ssize_t maxLen, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return sendGeneric(to, maxLen, -1, timeoutMs); } + size_t sendSize (Print& to, const ssize_t maxLen, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return sendSize(&to, maxLen, timeoutMs); } // remaining size (-1 by default = unknown) virtual ssize_t streamSize () { return -1; } @@ -192,17 +192,17 @@ class Stream: public Print { STREAMTO_READ_ERROR, STREAMTO_WRITE_ERROR, STREAMTO_SHORT, - } toReport_e; + } sendReport_e; - toReport_e getLastTo () /*const*/ { return (toReport_e)getWriteError(); } + sendReport_e getLastSendReport () /*const*/ { return (sendReport_e)getWriteError(); } //////////////////// protected: - size_t toFull (Print* to, - const ssize_t maxLen = -1, - const int readUntilChar = -1, - oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* neverExpires=>getTimeout() */); + size_t sendGeneric (Print* to, + const ssize_t maxLen = -1, + const int readUntilChar = -1, + oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* neverExpires=>getTimeout() */); //////////////////// end of extensions diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index 5ce7f715b7..e089b66eca 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -24,12 +24,11 @@ #include using esp8266::polledTimeout::oneShotFastMs; -using esp8266::polledTimeout::periodicFastMs; -size_t Stream::toFull(Print* to, - const ssize_t len, - const int readUntilChar, - const oneShotFastMs::timeType timeoutMs) +size_t Stream::sendGeneric(Print* to, + const ssize_t len, + const int readUntilChar, + const oneShotFastMs::timeType timeoutMs) { setWriteError(STREAMTO_SUCCESS); @@ -272,13 +271,13 @@ size_t Stream::toFull(Print* to, Stream& operator << (Stream& out, String& string) { - StreamPtr(string).toAll(out); + StreamPtr(string).sendAll(out); return out; } Stream& operator << (Stream& out, StreamString& stream) { - stream.toAll(out); + stream.sendAll(out); return out; } @@ -289,30 +288,30 @@ Stream& operator << (Stream& out, Stream& stream) if (stream.inputTimeoutPossible()) { // restrict with only what's buffered on input - stream.toNow(out); + stream.sendNow(out); } else { // take all what is in input - stream.toAll(out); + stream.sendAll(out); } } else { - stream.toSize(out, stream.streamSize()); + stream.sendSize(out, stream.streamSize()); } return out; } Stream& operator << (Stream& out, const char* text) { - StreamPtr(text).toAll(out); + StreamPtr(text).sendAll(out); return out; } Stream& operator << (Stream& out, const __FlashStringHelper* text) { - StreamPtr(text).toAll(out); + StreamPtr(text).sendAll(out); return out; } diff --git a/doc/reference.rst b/doc/reference.rst index 86800cd421..8e46f03d66 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -447,7 +447,7 @@ Stream extensions .. code:: cpp - StreamPtr css(F("my long css data")); // no heap/stack allocation + StreamPtr css(F("my long css data")); // CSS data not copied to RAM server.toAll(css); - ``S2Stream::`` is designed to make a ``Stream::`` out of a ``String::`` without copy. diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index bf1a07fd45..030b37e388 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -28,7 +28,7 @@ #include #include -static int TO2HTTPC (Stream::toReport_e streamToError) +static int TO2HTTPC (Stream::sendReport_e streamToError) { switch (streamToError) { @@ -443,7 +443,7 @@ int HTTPClient::sendRequest(const char * type, const uint8_t * payload, size_t s } // transfer all of it, with send-timeout - if (size && StreamPtr(payload, size).toAll(_client) != size) + if (size && StreamPtr(payload, size).sendAll(_client) != size) return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); // handle Server Response (Header) @@ -930,7 +930,7 @@ bool HTTPClient::sendHeader(const char * type) DEBUG_HTTPCLIENT("[HTTP-Client] sending request header\n-----\n%s-----\n", header.c_str()); // transfer all of it, with timeout - return StreamPtr(header).toAll(_client) == header.length(); + return StreamPtr(header).sendAll(_client) == header.length(); } /** diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index d6e5b4442d..c4cea83518 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -468,7 +468,7 @@ void ESP8266WebServerTemplate::send(int code, const char* content_ty if (content_length == 0) content_length = std::max((ssize_t)0, stream->streamSize()); _prepareHeader(header, code, content_type, content_length); - size_t sent = StreamPtr(header).toAll(&_currentClient); + size_t sent = StreamPtr(header).sendAll(&_currentClient); if (sent != header.length()) DBGWS("HTTPServer: error: sent %zd on %u bytes\n", sent, header.length()); if (content_length) diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer.h b/libraries/ESP8266WebServer/src/ESP8266WebServer.h index e9756590fd..91755e50a4 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer.h @@ -239,7 +239,7 @@ class ESP8266WebServerTemplate setContentLength(size); send(200, contentType, emptyString); if (requestMethod == HTTP_GET) - size = aStream.toSize(_currentClient, size); + size = aStream.sendSize(_currentClient, size); return size; } diff --git a/libraries/ESP8266WebServer/src/Parsing-impl.h b/libraries/ESP8266WebServer/src/Parsing-impl.h index 44dcf90a64..8e4a6d1ae9 100644 --- a/libraries/ESP8266WebServer/src/Parsing-impl.h +++ b/libraries/ESP8266WebServer/src/Parsing-impl.h @@ -38,7 +38,7 @@ template static bool readBytesWithTimeout(typename ServerType::ClientType& client, size_t maxLength, String& data, int timeout_ms) { S2Stream dataStream(data); - return client.toSize(dataStream, maxLength, timeout_ms) == maxLength; + return client.sendSize(dataStream, maxLength, timeout_ms) == maxLength; } template diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino index 4953ea0cad..3e36561d6c 100644 --- a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino @@ -122,13 +122,13 @@ void loop() { else if (t == 3) { // stream to print, possibly with only one copy if (sizes[s]) { - tot += client.toSize(&client, sizes[s]); + tot += client.sendSize(&client, sizes[s]); } else { - tot += client.toAll(&client); + tot += client.sendAll(&client); } cnt++; - switch (client.getLastTo()) { + switch (client.getLastSendReport()) { case Stream::STREAMTO_SUCCESS: break; case Stream::STREAMTO_TIMED_OUT: Serial.println("Stream::to: timeout"); break; case Stream::STREAMTO_READ_ERROR: Serial.println("Stream::to: read error"); break; From c94ba0b33e414cfdc994e325fc32df77f15670ae Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 26 Jan 2021 00:08:56 +0100 Subject: [PATCH 162/207] properly invalidate ssl buffer after consuming some data --- libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp index 08dc3784a7..9583d14b12 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.cpp @@ -372,7 +372,10 @@ const char* WiFiClientSecureCtx::peekBuffer () // consume bytes after use (see peekBuffer) void WiFiClientSecureCtx::peekConsume (size_t consume) { + // according to WiFiClientSecureCtx::read: br_ssl_engine_recvapp_ack(_eng, consume); + _recvapp_buf = nullptr; + _recvapp_len = 0; } int WiFiClientSecureCtx::read() { From 31e3c39ef6161a239832f72a4365e3d20fca6442 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 26 Jan 2021 00:33:02 +0100 Subject: [PATCH 163/207] more rename: ::to => ::send --- .../src/ESP8266HTTPClient.cpp | 18 +++++++++--------- .../src/ESP8266WebServer-impl.h | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 030b37e388..7197fa106c 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -28,7 +28,7 @@ #include #include -static int TO2HTTPC (Stream::sendReport_e streamToError) +static int SSEND2HTTPC (Stream::sendReport_e streamToError) { switch (streamToError) { @@ -544,7 +544,7 @@ int HTTPClient::sendRequest(const char * type, Stream * stream, size_t size) } // transfer all of it, with timeout - size_t transferred = stream->toSize(_client, size); + size_t transferred = stream->sendSize(_client, size); if (transferred != size) { DEBUG_HTTPCLIENT("[HTTP-Client][sendRequest] short write, asked for %d but got %d failed.\n", size, transferred); @@ -626,11 +626,11 @@ int HTTPClient::writeToStream(Stream * stream) if(_transferEncoding == HTTPC_TE_IDENTITY) { // len < 0: transfer all of it, with timeout // len >= 0: max:len, with timeout - ret = _client->toSize(stream, len); + ret = _client->sendSize(stream, len); // do we have an error? - if(_client->getLastTo() != Stream::STREAMTO_SUCCESS) { - return returnError(TO2HTTPC(_client->getLastTo())); + if(_client->getLastSendReport() != Stream::STREAMTO_SUCCESS) { + return returnError(SSEND2HTTPC(_client->getLastSendReport())); } } else if(_transferEncoding == HTTPC_TE_CHUNKED) { int size = 0; @@ -654,10 +654,10 @@ int HTTPClient::writeToStream(Stream * stream) // data left? if(len > 0) { // read len bytes with timeout - int r = _client->toSize(stream, len); - if (_client->getLastTo() != Stream::STREAMTO_SUCCESS) + int r = _client->sendSize(stream, len); + if (_client->getLastSendReport() != Stream::STREAMTO_SUCCESS) // not all data transferred - return returnError(TO2HTTPC(_client->getLastTo())); + return returnError(SSEND2HTTPC(_client->getLastSendReport())); ret += r; } else { @@ -847,7 +847,7 @@ bool HTTPClient::connect(void) { if(_reuse && _canReuse && connected()) { DEBUG_HTTPCLIENT("[HTTP-Client] connect: already connected, reusing connection\n"); - _client->toNow(devnull); // clear _client's output (all of it, no timeout) + _client->sendNow(devnull); // clear _client's output (all of it, no timeout) return true; } diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index c4cea83518..45e3b454df 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -496,7 +496,7 @@ void ESP8266WebServerTemplate::sendContent(Stream* content, ssize_t if(_chunked) { _currentClient.printf("%zx\r\n", content_length); } - size_t sent = content->toSize(&_currentClient, content_length); + size_t sent = content->sendSize(&_currentClient, content_length); (void)sent; //XXXFIXME if (sent != content_length) print-error-on-console-and-return-false if(_chunked) { _currentClient.printf_P(PSTR("\r\n")); From da1f96d28dc03e39119da9fb9a8400e514a49c14 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 26 Jan 2021 00:50:23 +0100 Subject: [PATCH 164/207] fix api in example&test --- .../examples/StreamString/StreamString.ino | 28 ++++++++++--------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/libraries/esp8266/examples/StreamString/StreamString.ino b/libraries/esp8266/examples/StreamString/StreamString.ino index cbd9b0b9a8..4a418973b3 100644 --- a/libraries/esp8266/examples/StreamString/StreamString.ino +++ b/libraries/esp8266/examples/StreamString/StreamString.ino @@ -1,4 +1,6 @@ +// this example sketch in the public domain is also a host and device test + #include #include @@ -23,7 +25,7 @@ void testProgmem() { auto inProgmem2 = F("I am too in progmem"); int heap = (int)ESP.getFreeHeap(); - auto stream1 = StreamPtr(inProgmem, sizeof(inProgmem) - 1, true); + auto stream1 = StreamPtr(inProgmem, sizeof(inProgmem) - 1); auto stream2 = StreamPtr(inProgmem2); Serial << stream1 << " - " << stream2 << "\n"; heap -= (int)ESP.getFreeHeap(); @@ -42,10 +44,10 @@ void testStream() { // prefer the lighter StreamPtr(String) to make a read-only Stream out of a String result.clear(); - StreamPtr(inputString).toAll(result); - StreamPtr(inputString).toAll(result); - StreamPtr(inputString).toAll(result); - check("StreamPtr.toAll(StreamString)", result.c_str(), "hellohellohello"); + StreamPtr(inputString).sendAll(result); + StreamPtr(inputString).sendAll(result); + StreamPtr(inputString).sendAll(result); + check("StreamPtr.sendAll(StreamString)", result.c_str(), "hellohellohello"); } { @@ -66,9 +68,9 @@ void testStream() { S2Stream input(inputString); input.reset(); - input.toAll(result); - input.toAll(result); - check("S2Stream.toAll(StreamString)", result.c_str(), "hello"); + input.sendAll(result); + input.sendAll(result); + check("S2Stream.sendAll(StreamString)", result.c_str(), "hello"); check("unmodified String given to S2Stream", inputString.c_str(), "hello"); } @@ -82,8 +84,8 @@ void testStream() { // stream position set to offset 2 (0 by default) input.reset(2); - input.toAll(result); - input.toAll(result); + input.sendAll(result); + input.sendAll(result); check("S2Stream.reset(2):", result.c_str(), "llo"); } @@ -96,9 +98,9 @@ void testStream() { // reading stream will consume the string input.setConsume(); // can be ommitted, this is the default - input.toSize(result, 1); - input.toSize(result, 2); - check("setConsume(): S2Stream().toSize(StreamString,3)", result.c_str(), "hel"); + input.sendSize(result, 1); + input.sendSize(result, 2); + check("setConsume(): S2Stream().sendSize(StreamString,3)", result.c_str(), "hel"); check("setConsume(): String given from S2Stream is swallowed", inputString.c_str(), "lo"); } From 531d7949842d7b3364e55b9ed1416457611bd28b Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 26 Jan 2021 00:52:28 +0100 Subject: [PATCH 165/207] remove useless constant --- tests/host/common/Arduino.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/host/common/Arduino.cpp b/tests/host/common/Arduino.cpp index a6227a035f..780c4adc14 100644 --- a/tests/host/common/Arduino.cpp +++ b/tests/host/common/Arduino.cpp @@ -80,5 +80,3 @@ cont_t* g_pcont = NULL; extern "C" void cont_yield(cont_t*) { } - -const char* overrideme PROGMEM = " should be overridden for better efficiency\r\n"; From cd3caad7783fb1e93392ab6fb029c831165e0824 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 26 Jan 2021 01:05:21 +0100 Subject: [PATCH 166/207] sync w/ upstream SoftwareSerial --- libraries/SoftwareSerial | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/SoftwareSerial b/libraries/SoftwareSerial index a174c00e59..27477f7e29 160000 --- a/libraries/SoftwareSerial +++ b/libraries/SoftwareSerial @@ -1 +1 @@ -Subproject commit a174c00e59662a598d2c273f80a8fe522fe8d9f5 +Subproject commit 27477f7e29b01c99aa5665b3d5aef692fec06a9c From 7b35d70ddd97df36bd3a291cdb08ac76c258e824 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 29 Jan 2021 22:14:57 +0100 Subject: [PATCH 167/207] update comment --- cores/esp8266/FS.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cores/esp8266/FS.h b/cores/esp8266/FS.h index a393a88ba1..d372ff273a 100644 --- a/cores/esp8266/FS.h +++ b/cores/esp8266/FS.h @@ -85,7 +85,7 @@ class File : public Stream bool isDirectory() const; // Arduino "class SD" methods for compatibility - //XXX use stream::to / check read(buf,size) result + //TODO use stream::send / check read(buf,size) result template size_t write(T &src){ uint8_t obuf[256]; size_t doneLen = 0; From 21a50ed847e1edf0ee09753c89b4a451180180bc Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 29 Jan 2021 22:17:04 +0100 Subject: [PATCH 168/207] remove useless comment --- cores/esp8266/Stream.h | 1 - 1 file changed, 1 deletion(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index d9599bc1be..ae01208a01 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -100,7 +100,6 @@ class Stream: public Print { } // terminates if length characters have been read or timeout (see setTimeout) // returns the number of characters placed in the buffer (0 means no valid data found) - // return data type: size_t size_t readBytesUntil(char terminator, char *buffer, size_t length); // as readBytes with terminator character size_t readBytesUntil(char terminator, uint8_t *buffer, size_t length) { From 92a6302ef4aeec5c90d84f5a671382f522ae8200 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 29 Jan 2021 22:22:55 +0100 Subject: [PATCH 169/207] addressible => addressable --- cores/esp8266/StreamDev.h | 14 +++++++------- cores/esp8266/esp_priv.h | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 9f8473aa74..1a31747eb5 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -157,15 +157,15 @@ class StreamPtr: public StreamNull protected: const char* _buffer; size_t _size; - bool _byteAddressible; + bool _byteAddressable; size_t _peekPointer = 0; public: - StreamPtr(const String& string): _buffer(string.c_str()), _size(string.length()), _byteAddressible(true) { } - StreamPtr(const char* buffer, size_t size): _buffer(buffer), _size(size), _byteAddressible(__byteAddressible(buffer)) { } - StreamPtr(const uint8_t* buffer, size_t size): _buffer((const char*)buffer), _size(size), _byteAddressible(__byteAddressible(buffer)) { } - StreamPtr(const __FlashStringHelper* buffer, size_t size): _buffer(reinterpret_cast(buffer)), _size(size), _byteAddressible(false) { } - StreamPtr(const __FlashStringHelper* text): _buffer(reinterpret_cast(text)), _size(strlen_P((PGM_P)text)), _byteAddressible(false) { } + StreamPtr(const String& string): _buffer(string.c_str()), _size(string.length()), _byteAddressable(true) { } + StreamPtr(const char* buffer, size_t size): _buffer(buffer), _size(size), _byteAddressable(__byteAddressable(buffer)) { } + StreamPtr(const uint8_t* buffer, size_t size): _buffer((const char*)buffer), _size(size), _byteAddressable(__byteAddressable(buffer)) { } + StreamPtr(const __FlashStringHelper* buffer, size_t size): _buffer(reinterpret_cast(buffer)), _size(size), _byteAddressable(false) { } + StreamPtr(const __FlashStringHelper* text): _buffer(reinterpret_cast(text)), _size(strlen_P((PGM_P)text)), _byteAddressable(false) { } void reset(int pointer = 0) { @@ -213,7 +213,7 @@ class StreamPtr: public StreamNull // peekBuffer virtual bool peekBufferAPI() const override { - return _byteAddressible; + return _byteAddressable; } virtual size_t peekAvailable() override diff --git a/cores/esp8266/esp_priv.h b/cores/esp8266/esp_priv.h index d460071c9e..5ef402a774 100644 --- a/cores/esp8266/esp_priv.h +++ b/cores/esp8266/esp_priv.h @@ -4,7 +4,7 @@ #if defined(CORE_MOCK) -constexpr bool __byteAddressible(const void* addr) +constexpr bool __byteAddressable(const void* addr) { (void)addr; return true; @@ -15,7 +15,7 @@ constexpr bool __byteAddressible(const void* addr) #include // returns true when addr can be used without "pgm_" functions or non32xfer service -constexpr bool __byteAddressible(const void* addr) +constexpr bool __byteAddressable(const void* addr) { return addr < (const void*)(XCHAL_DATARAM0_VADDR + XCHAL_DATARAM0_SIZE); } From 41d1868fb85354c5780b3fba73106e266bb3d784 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 29 Jan 2021 22:23:11 +0100 Subject: [PATCH 170/207] rename private variable (_x => _zero) --- cores/esp8266/StreamDev.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 1a31747eb5..fce8996b72 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -107,11 +107,11 @@ class StreamZero: public StreamNull { protected: - char _x; + char _zero; public: - StreamZero(char x = 0): _x(x) { } + StreamZero(char zero = 0): _zero(zero) { } // Stream virtual int available() override @@ -121,23 +121,23 @@ class StreamZero: public StreamNull virtual int read() override { - return _x; + return _zero; } virtual int peek() override { - return _x; + return _zero; } virtual size_t readBytes(char* buffer, size_t len) override { - memset(buffer, _x, len); + memset(buffer, _zero, len); return len; } virtual int read(uint8_t* buffer, size_t len) override { - memset((char*)buffer, _x, len); + memset((char*)buffer, _zero, len); return len; } From abca00b2bdfeebc0df98e094c40430d18f01c128 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 29 Jan 2021 22:26:29 +0100 Subject: [PATCH 171/207] fix licences --- cores/esp8266/StreamDev.h | 2 -- cores/esp8266/esp_priv.h | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index fce8996b72..549e695a93 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -15,8 +15,6 @@ You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - - parsing functions based on TextFinder library by Michael Margolis */ #ifndef __STREAMDEV_H diff --git a/cores/esp8266/esp_priv.h b/cores/esp8266/esp_priv.h index 5ef402a774..305197e1c9 100644 --- a/cores/esp8266/esp_priv.h +++ b/cores/esp8266/esp_priv.h @@ -1,3 +1,22 @@ +/* + esp_priv.h - private esp8266 helpers + Copyright (c) 2020 esp8266/Arduino community. All right reserved. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +*/ + #ifndef __ESP_PRIV #define __ESP_PRIV From 3c1e104d814c4e8f07fb96839d2d02f74358532b Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 29 Jan 2021 22:39:38 +0100 Subject: [PATCH 172/207] typo in tests --- tests/device/test_sw_StreamString/test_sw_StreamString.ino | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/device/test_sw_StreamString/test_sw_StreamString.ino b/tests/device/test_sw_StreamString/test_sw_StreamString.ino index 5b9b511b7f..d706ee6aba 100644 --- a/tests/device/test_sw_StreamString/test_sw_StreamString.ino +++ b/tests/device/test_sw_StreamString/test_sw_StreamString.ino @@ -19,7 +19,7 @@ void setup () BS_RUN(Serial); } -TEST_CASE("StreamSring tests", "[StreamString]") +TEST_CASE("StreamString tests", "[StreamString]") { testStream(); } From 84d8bc693d9aca86dc0e91f4e85eae876fb9b606 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Fri, 29 Jan 2021 23:33:20 +0100 Subject: [PATCH 173/207] define STREAMSEND_API globally (for checking on its presence) --- cores/esp8266/Stream.h | 21 ++++++++++++------- cores/esp8266/StreamDev.cpp | 14 ++++++------- .../src/ESP8266HTTPClient.cpp | 14 ++++++------- .../examples/HelloServer/HelloServer.ino | 4 ++-- .../examples/WiFiEcho/WiFiEcho.ino | 10 ++++----- 5 files changed, 34 insertions(+), 29 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index ae01208a01..7b1a1b82e9 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -39,11 +39,14 @@ */ // Arduino `Client: public Stream` class defines `virtual int read(uint8_t *buf, size_t size) = 0;` -// This function is now imported into `Stream::` for `Stream::to*()`. +// This function is now imported into `Stream::` for `Stream::send*()`. // Other classes inheriting from `Stream::` and implementing `read(uint8_t *buf, size_t size)` // must consequently use `int` as return type, namely Hardware/SoftwareSerial, FileSystems... #define STREAM_READ_RETURNS_INT 1 +// Stream::send API is present +#define STREAMSEND_API 1 + class Stream: public Print { protected: unsigned long _timeout = 1000; // number of milliseconds to wait for the next char before aborting timed read @@ -158,7 +161,7 @@ class Stream: public Print { // - always stop before timeout when "no-more-input-possible-data" // or "no-more-output-possible-data" condition is met // - always return number of transfered bytes - // When result is 0 or less than requested maxLen, this->getLastTo() + // When result is 0 or less than requested maxLen, Print::getLastSend() // contains an error reason. // transfers already buffered / immediately available data (no timeout) @@ -186,14 +189,14 @@ class Stream: public Print { typedef enum { - STREAMTO_SUCCESS = 0, - STREAMTO_TIMED_OUT, - STREAMTO_READ_ERROR, - STREAMTO_WRITE_ERROR, - STREAMTO_SHORT, + STREAMSEND_SUCCESS = 0, + STREAMSEND_TIMED_OUT, + STREAMSEND_READ_ERROR, + STREAMSEND_WRITE_ERROR, + STREAMSEND_SHORT, } sendReport_e; - sendReport_e getLastSendReport () /*const*/ { return (sendReport_e)getWriteError(); } + sendReport_e getLastSendReport () const { return _sendReport; } //////////////////// @@ -210,6 +213,8 @@ class Stream: public Print { // this allows format characters (typically commas) in values to be ignored float parseFloat(char skipChar); // as parseFloat() but the given skipChar is ignored + + sendReport_e _sendReport = STREAMSEND_SUCCESS; }; #endif diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index e089b66eca..e44d5c7aae 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -30,7 +30,7 @@ size_t Stream::sendGeneric(Print* to, const int readUntilChar, const oneShotFastMs::timeType timeoutMs) { - setWriteError(STREAMTO_SUCCESS); + _sendReport = STREAMSEND_SUCCESS; if (len == 0) { @@ -156,7 +156,7 @@ size_t Stream::sendGeneric(Print* to, w = to->write(c); if (w != 1) { - setWriteError(STREAMTO_WRITE_ERROR); + _sendReport = STREAMSEND_WRITE_ERROR; break; } written += 1; @@ -216,14 +216,14 @@ size_t Stream::sendGeneric(Print* to, ssize_t r = read(temp, w); if (r < 0) { - setWriteError(STREAMTO_READ_ERROR); + _sendReport = STREAMSEND_READ_ERROR; break; } w = to->write(temp, r); written += w; if ((size_t)r != w) { - setWriteError(STREAMTO_WRITE_ERROR); + _sendReport = STREAMSEND_WRITE_ERROR; break; } if (maxLen && w) @@ -249,11 +249,11 @@ size_t Stream::sendGeneric(Print* to, optimistic_yield(1000); } - if (getWriteError() == STREAMTO_SUCCESS && maxLen > 0) + if (getWriteError() == STREAMSEND_SUCCESS && maxLen > 0) { if (timeoutMs && timedOut) { - setWriteError(STREAMTO_TIMED_OUT); + _sendReport = STREAMSEND_TIMED_OUT; } else if ((ssize_t)written != len) { @@ -263,7 +263,7 @@ size_t Stream::sendGeneric(Print* to, // // Mark it as an error because user usually wants to get what is // asked for. - setWriteError(STREAMTO_SHORT); + _sendReport = STREAMSEND_SHORT; } } return written; diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 7197fa106c..9ecd93f909 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -32,11 +32,11 @@ static int SSEND2HTTPC (Stream::sendReport_e streamToError) { switch (streamToError) { - case Stream::STREAMTO_TIMED_OUT: return HTTPC_ERROR_READ_TIMEOUT; - case Stream::STREAMTO_READ_ERROR: return HTTPC_ERROR_NO_STREAM; - case Stream::STREAMTO_WRITE_ERROR: return HTTPC_ERROR_STREAM_WRITE; - case Stream::STREAMTO_SHORT: return HTTPC_ERROR_STREAM_WRITE; - case Stream::STREAMTO_SUCCESS: return 0; + case Stream::STREAMSEND_TIMED_OUT: return HTTPC_ERROR_READ_TIMEOUT; + case Stream::STREAMSEND_READ_ERROR: return HTTPC_ERROR_NO_STREAM; + case Stream::STREAMSEND_WRITE_ERROR: return HTTPC_ERROR_STREAM_WRITE; + case Stream::STREAMSEND_SHORT: return HTTPC_ERROR_STREAM_WRITE; + case Stream::STREAMSEND_SUCCESS: return 0; } return 0; // never reached, keep gcc quiet } @@ -629,7 +629,7 @@ int HTTPClient::writeToStream(Stream * stream) ret = _client->sendSize(stream, len); // do we have an error? - if(_client->getLastSendReport() != Stream::STREAMTO_SUCCESS) { + if(_client->getLastSendReport() != Stream::STREAMSEND_SUCCESS) { return returnError(SSEND2HTTPC(_client->getLastSendReport())); } } else if(_transferEncoding == HTTPC_TE_CHUNKED) { @@ -655,7 +655,7 @@ int HTTPClient::writeToStream(Stream * stream) if(len > 0) { // read len bytes with timeout int r = _client->sendSize(stream, len); - if (_client->getLastSendReport() != Stream::STREAMTO_SUCCESS) + if (_client->getLastSendReport() != Stream::STREAMSEND_SUCCESS) // not all data transferred return returnError(SSEND2HTTPC(_client->getLastSendReport())); ret += r; diff --git a/libraries/ESP8266WebServer/examples/HelloServer/HelloServer.ino b/libraries/ESP8266WebServer/examples/HelloServer/HelloServer.ino index 6715f9ee41..7fdee1667a 100644 --- a/libraries/ESP8266WebServer/examples/HelloServer/HelloServer.ino +++ b/libraries/ESP8266WebServer/examples/HelloServer/HelloServer.ino @@ -115,9 +115,9 @@ void setup(void) { // swallow the exact amount matching the full request+content, // hence the tcp connection cannot be handled anymore by the // webserver. -#ifdef STREAMTO_API +#ifdef STREAMSEND_API // we are lucky - client->toWithTimeout(Serial, 500); + client->sendAll(Serial, 500); #else auto last = millis(); while ((millis() - last) < 500) { diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino index 3e36561d6c..6cd5aad9be 100644 --- a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino @@ -129,11 +129,11 @@ void loop() { cnt++; switch (client.getLastSendReport()) { - case Stream::STREAMTO_SUCCESS: break; - case Stream::STREAMTO_TIMED_OUT: Serial.println("Stream::to: timeout"); break; - case Stream::STREAMTO_READ_ERROR: Serial.println("Stream::to: read error"); break; - case Stream::STREAMTO_WRITE_ERROR: Serial.println("Stream::to: write error"); break; - case Stream::STREAMTO_SHORT: Serial.println("Stream::to: short transfer"); break; + case Stream::STREAMSEND_SUCCESS: break; + case Stream::STREAMSEND_TIMED_OUT: Serial.println("Stream::to: timeout"); break; + case Stream::STREAMSEND_READ_ERROR: Serial.println("Stream::to: read error"); break; + case Stream::STREAMSEND_WRITE_ERROR: Serial.println("Stream::to: write error"); break; + case Stream::STREAMSEND_SHORT: Serial.println("Stream::to: short transfer"); break; } } From 592b44014b84890e4bcad037dc52d0c9d1a1984b Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 31 Jan 2021 21:57:21 +0100 Subject: [PATCH 174/207] ::streamSize => ::streamRemaining --- cores/esp8266/FS.h | 2 +- cores/esp8266/Stream.h | 2 +- cores/esp8266/StreamDev.cpp | 4 ++-- cores/esp8266/StreamDev.h | 6 +++--- cores/esp8266/StreamString.h | 2 +- doc/reference.rst | 4 ++-- libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h | 4 ++-- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cores/esp8266/FS.h b/cores/esp8266/FS.h index d372ff273a..80c0f2e61c 100644 --- a/cores/esp8266/FS.h +++ b/cores/esp8266/FS.h @@ -74,7 +74,7 @@ class File : public Stream } size_t position() const; size_t size() const; - virtual ssize_t streamSize() override { return size() - position(); } + virtual ssize_t streamRemaining() override { return (ssize_t)size() - (ssize_t)position(); } void close(); operator bool() const; const char* name() const; diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 7b1a1b82e9..ca20301b9b 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -185,7 +185,7 @@ class Stream: public Print { size_t sendSize (Print& to, const ssize_t maxLen, const oneShotMs::timeType timeoutMs = oneShotMs::neverExpires) { return sendSize(&to, maxLen, timeoutMs); } // remaining size (-1 by default = unknown) - virtual ssize_t streamSize () { return -1; } + virtual ssize_t streamRemaining () { return -1; } typedef enum { diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index e44d5c7aae..15351dba40 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -283,7 +283,7 @@ Stream& operator << (Stream& out, StreamString& stream) Stream& operator << (Stream& out, Stream& stream) { - if (stream.streamSize() < 0) + if (stream.streamRemaining() < 0) { if (stream.inputTimeoutPossible()) { @@ -298,7 +298,7 @@ Stream& operator << (Stream& out, Stream& stream) } else { - stream.sendSize(out, stream.streamSize()); + stream.sendSize(out, stream.streamRemaining()); } return out; } diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 549e695a93..6e068ebea5 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -90,7 +90,7 @@ class StreamNull: public Stream return false; } - virtual ssize_t streamSize() override + virtual ssize_t streamRemaining() override { return 0; } @@ -139,7 +139,7 @@ class StreamZero: public StreamNull return len; } - virtual ssize_t streamSize() override + virtual ssize_t streamRemaining() override { return 32767; } @@ -203,7 +203,7 @@ class StreamPtr: public StreamNull return readBytes((char*)buffer, len); } - virtual ssize_t streamSize() override + virtual ssize_t streamRemaining() override { return _size; } diff --git a/cores/esp8266/StreamString.h b/cores/esp8266/StreamString.h index 5c809760e8..4e49367837 100644 --- a/cores/esp8266/StreamString.h +++ b/cores/esp8266/StreamString.h @@ -183,7 +183,7 @@ class S2Stream: public Stream } } - virtual ssize_t streamSize() override + virtual ssize_t streamRemaining() override { return peekPointer < 0 ? string->length() : string->length() - peekPointer; } diff --git a/doc/reference.rst b/doc/reference.rst index d3d4e12b6d..3fe212dd10 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -514,7 +514,7 @@ Stream extensions A closed network connection returns false. This function allows ``Stream::toAll()`` to return early. - - ``virtual ssize_t streamSize()`` + - ``virtual ssize_t streamRemaining()`` - It returns -1 when stream size is unknown, depending on implementation + It returns -1 when stream remaining size is unknown, depending on implementation (string size, file size..). diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index 45e3b454df..9e399feeab 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -466,7 +466,7 @@ template void ESP8266WebServerTemplate::send(int code, const char* content_type, Stream* stream, size_t content_length /*= 0*/) { String header; if (content_length == 0) - content_length = std::max((ssize_t)0, stream->streamSize()); + content_length = std::max((ssize_t)0, stream->streamRemaining()); _prepareHeader(header, code, content_type, content_length); size_t sent = StreamPtr(header).sendAll(&_currentClient); if (sent != header.length()) @@ -492,7 +492,7 @@ void ESP8266WebServerTemplate::sendContent(Stream* content, ssize_t if (_currentMethod == HTTP_HEAD) return; if (content_length <= 0) - content_length = std::max((ssize_t)0, content->streamSize()); + content_length = std::max((ssize_t)0, content->streamRemaining()); if(_chunked) { _currentClient.printf("%zx\r\n", content_length); } From 2783c19d6ab01731da41585d61ef15689b171ba1 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 31 Jan 2021 22:06:51 +0100 Subject: [PATCH 175/207] ::peekBufferAPI() => ::hasPeekBufferAPI() --- cores/esp8266/HardwareSerial.h | 2 +- cores/esp8266/Stream.h | 2 +- cores/esp8266/StreamDev.cpp | 2 +- cores/esp8266/StreamDev.h | 2 +- cores/esp8266/StreamString.h | 2 +- doc/reference.rst | 2 +- libraries/ESP8266WiFi/src/WiFiClient.cpp | 2 +- libraries/ESP8266WiFi/src/WiFiClient.h | 2 +- libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h | 4 ++-- 9 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cores/esp8266/HardwareSerial.h b/cores/esp8266/HardwareSerial.h index 6ff4562df1..b8747f09d0 100644 --- a/cores/esp8266/HardwareSerial.h +++ b/cores/esp8266/HardwareSerial.h @@ -136,7 +136,7 @@ class HardwareSerial: public Stream return uart_peek_char(_uart); } - virtual bool peekBufferAPI () const override + virtual bool hasPeekBufferAPI () const override { return true; } diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index ca20301b9b..aa27a21d50 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -123,7 +123,7 @@ class Stream: public Print { // informs user and ::to*() on effective buffered peek API implementation // by default: not available - virtual bool peekBufferAPI () const { return false; } + virtual bool hasPeekBufferAPI () const { return false; } // returns number of byte accessible by peekBuffer() virtual size_t peekAvailable () { return 0; } diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index 15351dba40..4f9970dcbc 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -54,7 +54,7 @@ size_t Stream::sendGeneric(Print* to, // len==-1 => maxLen=0 <=> until starvation size_t maxLen = std::max((ssize_t)0, len); - if (peekBufferAPI()) + if (hasPeekBufferAPI()) // peek-buffer API diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 6e068ebea5..f745184629 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -209,7 +209,7 @@ class StreamPtr: public StreamNull } // peekBuffer - virtual bool peekBufferAPI() const override + virtual bool hasPeekBufferAPI() const override { return _byteAddressable; } diff --git a/cores/esp8266/StreamString.h b/cores/esp8266/StreamString.h index 4e49367837..e31870a0a0 100644 --- a/cores/esp8266/StreamString.h +++ b/cores/esp8266/StreamString.h @@ -142,7 +142,7 @@ class S2Stream: public Stream //// Stream's peekBufferAPI - virtual bool peekBufferAPI() const override + virtual bool hasPeekBufferAPI() const override { return true; } diff --git a/doc/reference.rst b/doc/reference.rst index 3fe212dd10..fa3101a5e3 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -493,7 +493,7 @@ Stream extensions implemented in ``HardwareSerial``, ``WiFiClient`` and ``WiFiClientSecure``. - - ``virtual bool peekBufferAPI ()`` returns ``true`` when the API is present in the class + - ``virtual bool hasPeekBufferAPI ()`` returns ``true`` when the API is present in the class - ``virtual size_t peekAvailable ()`` returns the number of reachable bytes diff --git a/libraries/ESP8266WiFi/src/WiFiClient.cpp b/libraries/ESP8266WiFi/src/WiFiClient.cpp index 1a26ef7386..5420f3a47b 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.cpp +++ b/libraries/ESP8266WiFi/src/WiFiClient.cpp @@ -417,7 +417,7 @@ uint8_t WiFiClient::getKeepAliveCount () const return _client->getKeepAliveCount(); } -bool WiFiClient::peekBufferAPI () const +bool WiFiClient::hasPeekBufferAPI () const { return true; } diff --git a/libraries/ESP8266WiFi/src/WiFiClient.h b/libraries/ESP8266WiFi/src/WiFiClient.h index 4d44da57a7..702b48185c 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.h +++ b/libraries/ESP8266WiFi/src/WiFiClient.h @@ -123,7 +123,7 @@ class WiFiClient : public Client, public SList { void setSync(bool sync); // peek buffer API is present - virtual bool peekBufferAPI () const override; + virtual bool hasPeekBufferAPI () const override; // return number of byte accessible by peekBuffer() virtual size_t peekAvailable () override; diff --git a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h index 607c8b2988..52dd878745 100644 --- a/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h +++ b/libraries/ESP8266WiFi/src/WiFiClientSecureBearSSL.h @@ -122,7 +122,7 @@ class WiFiClientSecureCtx : public WiFiClient { bool setCiphersLessSecure(); // Only use the limited set of RSA ciphers without EC // peek buffer API is present - virtual bool peekBufferAPI () const override { return true; } + virtual bool hasPeekBufferAPI () const override { return true; } // return number of byte accessible by peekBuffer() virtual size_t peekAvailable () override { return WiFiClientSecureCtx::available(); } @@ -302,7 +302,7 @@ class WiFiClientSecure : public WiFiClient { static bool probeMaxFragmentLength(const String& host, uint16_t port, uint16_t len); // peek buffer API is present - virtual bool peekBufferAPI () const override { return true; } + virtual bool hasPeekBufferAPI () const override { return true; } // return number of byte accessible by peekBuffer() virtual size_t peekAvailable () override { return _ctx->available(); } From da1240a9bef9dbd94c83abd945be77bd216b7667 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 31 Jan 2021 22:26:04 +0100 Subject: [PATCH 176/207] {input,output}TimeoutPossible => {input,output}CanTimeout --- cores/esp8266/Print.h | 2 +- cores/esp8266/Stream.h | 6 +++--- cores/esp8266/StreamDev.cpp | 16 ++++++++-------- cores/esp8266/StreamDev.h | 4 ++-- cores/esp8266/StreamString.h | 4 ++-- doc/reference.rst | 4 ++-- libraries/ESP8266WiFi/src/WiFiClient.h | 4 ++-- 7 files changed, 20 insertions(+), 20 deletions(-) diff --git a/cores/esp8266/Print.h b/cores/esp8266/Print.h index 311b4a9b39..746ebc1e20 100644 --- a/cores/esp8266/Print.h +++ b/cores/esp8266/Print.h @@ -114,7 +114,7 @@ class Print { // by default write timeout is possible (outgoing data from network,serial..) // (children can override to false (like String)) - virtual bool outputTimeoutPossible () { return true; } + virtual bool outputCanTimeout () { return true; } }; template<> size_t Print::printNumber(double number, uint8_t digits); diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index aa27a21d50..832675ae25 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -140,9 +140,9 @@ class Stream: public Print { // by default read timeout is possible (incoming data from network,serial..) // children can override to false (like String::) - virtual bool inputTimeoutPossible () { return true; } + virtual bool inputCanTimeout () { return true; } - // (outputTimeoutPossible() is defined in Print::) + // (outputCanTimeout() is defined in Print::) //////////////////// extensions: Streaming streams to streams // Stream::to*() @@ -153,7 +153,7 @@ class Stream: public Print { // - for efficiency, Stream classes should implement peekAPI when // possible // - for an efficient timeout management, Print/Stream classes - // should implement {output,input}TimeoutPossible() + // should implement {output,input}CanTimeout() using oneShotMs = esp8266::polledTimeout::oneShotFastMs; diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index 4f9970dcbc..e58718413d 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -44,7 +44,7 @@ size_t Stream::sendGeneric(Print* to, // - getTimeout() is for reading only // - there is no getOutputTimeout() api // So we use getTimeout() for both, - // (also when inputTimeoutPossible() is false) + // (also when inputCanTimeout() is false) // "neverExpires (default, impossible)" is translated to default timeout oneShotFastMs timedOut(timeoutMs >= oneShotFastMs::neverExpires ? getTimeout() : timeoutMs); @@ -61,14 +61,14 @@ size_t Stream::sendGeneric(Print* to, while (!maxLen || written < maxLen) { size_t avpk = peekAvailable(); - if (avpk == 0 && !inputTimeoutPossible()) + if (avpk == 0 && !inputCanTimeout()) { // no more data to read, ever break; } size_t w = to->availableForWrite(); - if (w == 0 && !outputTimeoutPossible()) + if (w == 0 && !outputCanTimeout()) { // no more data can be written, ever break; @@ -133,14 +133,14 @@ size_t Stream::sendGeneric(Print* to, while (!maxLen || written < maxLen) { size_t avpk = peekAvailable(); - if (avpk == 0 && !inputTimeoutPossible()) + if (avpk == 0 && !inputCanTimeout()) { // no more data to read, ever break; } size_t w = to->availableForWrite(); - if (w == 0 && !outputTimeoutPossible()) + if (w == 0 && !outputCanTimeout()) { // no more data can be written, ever break; @@ -191,14 +191,14 @@ size_t Stream::sendGeneric(Print* to, while (!maxLen || written < maxLen) { size_t avr = available(); - if (avr == 0 && !inputTimeoutPossible()) + if (avr == 0 && !inputCanTimeout()) { // no more data to read, ever break; } size_t w = to->availableForWrite(); - if (w == 0 && !to->outputTimeoutPossible()) + if (w == 0 && !to->outputCanTimeout()) // no more data can be written, ever { break; @@ -285,7 +285,7 @@ Stream& operator << (Stream& out, Stream& stream) { if (stream.streamRemaining() < 0) { - if (stream.inputTimeoutPossible()) + if (stream.inputCanTimeout()) { // restrict with only what's buffered on input stream.sendNow(out); diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index f745184629..7afdbb8d34 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -80,12 +80,12 @@ class StreamNull: public Stream return 0; } - virtual bool outputTimeoutPossible() override + virtual bool outputCanTimeout() override { return false; } - virtual bool inputTimeoutPossible() override + virtual bool inputCanTimeout() override { return false; } diff --git a/cores/esp8266/StreamString.h b/cores/esp8266/StreamString.h index e31870a0a0..b2ebb71dba 100644 --- a/cores/esp8266/StreamString.h +++ b/cores/esp8266/StreamString.h @@ -130,12 +130,12 @@ class S2Stream: public Stream // nothing to do } - virtual bool inputTimeoutPossible() override + virtual bool inputCanTimeout() override { return false; } - virtual bool outputTimeoutPossible() override + virtual bool outputCanTimeout() override { return false; } diff --git a/doc/reference.rst b/doc/reference.rst index fa3101a5e3..cace8dab13 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -504,12 +504,12 @@ Stream extensions - ``virtual void peekConsume (size_t consume)`` tells to discard that number of bytes - - ``virtual bool inputTimeoutPossible ()`` + - ``virtual bool inputCanTimeout ()`` A ``StringStream`` will return false. A closed network connection returns false. This function allows ``Stream::toAll()`` to return early. - - ``virtual bool outputTimeoutPossible ()`` + - ``virtual bool outputCanTimeout ()`` A closed network connection returns false. This function allows ``Stream::toAll()`` to return early. diff --git a/libraries/ESP8266WiFi/src/WiFiClient.h b/libraries/ESP8266WiFi/src/WiFiClient.h index 702b48185c..d59ae6ca5c 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.h +++ b/libraries/ESP8266WiFi/src/WiFiClient.h @@ -135,8 +135,8 @@ class WiFiClient : public Client, public SList { // consume bytes after use (see peekBuffer) virtual void peekConsume (size_t consume) override; - virtual bool outputTimeoutPossible () override { return connected(); } - virtual bool inputTimeoutPossible () override { return connected(); } + virtual bool outputCanTimeout () override { return connected(); } + virtual bool inputCanTimeout () override { return connected(); } protected: From 5ab7415183adbcd27e24465ccf5b4d7a2573d02d Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 31 Jan 2021 23:07:09 +0100 Subject: [PATCH 177/207] ::sendNow() => ::sendAvailable() + fix doc --- cores/esp8266/Stream.h | 4 +- cores/esp8266/StreamDev.cpp | 2 +- doc/reference.rst | 54 +++++++++---------- .../src/ESP8266HTTPClient.cpp | 2 +- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 832675ae25..0a79cf26ce 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -166,8 +166,8 @@ class Stream: public Print { // transfers already buffered / immediately available data (no timeout) // returns number of transfered bytes - size_t sendNow (Print* to) { return sendGeneric(to, -1, -1, oneShotMs::alwaysExpired); } - size_t sendNow (Print& to) { return sendNow(&to); } + size_t sendAvailable (Print* to) { return sendGeneric(to, -1, -1, oneShotMs::alwaysExpired); } + size_t sendAvailable (Print& to) { return sendAvailable(&to); } // transfers data until timeout // returns number of transfered bytes diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index e58718413d..edaabc0985 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -288,7 +288,7 @@ Stream& operator << (Stream& out, Stream& stream) if (stream.inputCanTimeout()) { // restrict with only what's buffered on input - stream.sendNow(out); + stream.sendAvailable(out); } else { diff --git a/doc/reference.rst b/doc/reference.rst index cace8dab13..9a8c2c7a94 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -386,44 +386,44 @@ Stream extensions .. code:: cpp - serverClient.toNow(Serial); // chunk by chunk - Serial.toNow(serverClient); // chunk by chunk + serverClient.sendAvailable(Serial); // chunk by chunk + Serial.sendAvailable(serverClient); // chunk by chunk An echo service can be written like this: .. code:: cpp - serverClient.toNow(serverClient); // tcp echo service + serverClient.sendAvailable(serverClient); // tcp echo service - Serial.toNow(Serial); // serial software loopback + Serial.sendAvailable(Serial); // serial software loopback Beside reducing coding time, these methods optimize transfers by avoiding buffer copies when possible. - - User facing API: ``Stream::to()`` + - User facing API: ``Stream::send()`` The goal of streams is to transfer data between producers and consumers, like the telnet/serial example above. Four methods are provided, all of them return the number of transmitted bytes: - - ``Stream::toSize(dest, size [, timeout])`` + - ``Stream::sendSize(dest, size [, timeout])`` This method waits up to the given or default timeout to transfer ``size`` bytes to the the ``dest`` Stream. - - ``Stream::toUntil(dest, delim [, timeout])`` + - ``Stream::sendUntil(dest, delim [, timeout])`` This method waits up to the given or default timeout to transfer data - until the character ``delim`` is met. The delimiter is read but not - transfered. + until the character ``delim`` is met. + Note: The delimiter is read but not transfered (like ``readBytesUntil``) - - ``Stream::toNow(dest)`` + - ``Stream::sendAvailable(dest)`` This method transfers all already available data to the destination. There is no timeout and the returned value is 0 when there is nothing to transfer or no room in the destination. - - ``Stream::toAll(dest [, timeout])`` + - ``Stream::sendAll(dest [, timeout])`` This method waits up to the given or default timeout to transfer all available data. It is useful when source is able to tell that no more @@ -432,7 +432,7 @@ Stream extensions For example, a source String will not grow during the transfer, or a particular network connection supposed to send a fixed amount of data - before closing. ``::toAll()`` will receive all bytes. Timeout is + before closing. ``::sendAll()`` will receive all bytes. Timeout is useful when destination needs processing time (e.g. network or serial input buffer full = please wait a bit). @@ -451,7 +451,7 @@ Stream extensions .. code:: cpp StreamPtr css(F("my long css data")); // CSS data not copied to RAM - server.toAll(css); + server.sendAll(css); - ``S2Stream::`` is designed to make a ``Stream::`` out of a ``String::`` without copy. @@ -459,32 +459,32 @@ Stream extensions String helloString("hello"); S2Stream hello(helloString); - hello.reset(0); // prevents ::read() to consume the string + hello.reset(0); // prevents ::read() to consume the string - hello.toAll(Serial); // shows "hello" - hello.toAll(Serial); // shows nothing, content has already been read - hello.reset(); // reset content pointer - hello.toAll(Serial); // shows "hello" - hello.reset(3); // reset content pointer to a specific position - hello.toAll(Serial); // shows "lo" + hello.sendAll(Serial); // shows "hello" + hello.sendAll(Serial); // shows nothing, content has already been read + hello.reset(); // reset content pointer + hello.sendAll(Serial); // shows "hello" + hello.reset(3); // reset content pointer to a specific position + hello.sendAll(Serial); // shows "lo" - hello.setConsume(); // ::read() will consume, this is the default + hello.setConsume(); // ::read() will consume, this is the default Serial.println(helloString.length()); // shows 5 - hello.toAll(Serial); // shows "hello" + hello.sendAll(Serial); // shows "hello" Serial.println(helloString.length()); // shows 0, string is consumed - ``StreamString::``, derives from ``S2Stream`` + ``StreamString::`` derives from ``S2Stream`` .. code:: cpp StreamString contentStream; - client.toSize(contentStream, SOME_SIZE); // receives at most SOME_SIZE bytes + client.sendSize(contentStream, SOME_SIZE); // receives at most SOME_SIZE bytes // equivalent to: String content; S2Stream contentStream(content); - client.toSize(contentStream, SOME_SIZE); // receives at most SOME_SIZE bytes + client.sendSize(contentStream, SOME_SIZE); // receives at most SOME_SIZE bytes // content has the data - Internal Stream API: ``peekBuffer`` @@ -507,12 +507,12 @@ Stream extensions - ``virtual bool inputCanTimeout ()`` A ``StringStream`` will return false. A closed network connection returns false. - This function allows ``Stream::toAll()`` to return early. + This function allows ``Stream::sendAll()`` to return earlier. - ``virtual bool outputCanTimeout ()`` A closed network connection returns false. - This function allows ``Stream::toAll()`` to return early. + This function allows ``Stream::sendAll()`` to return earlier. - ``virtual ssize_t streamRemaining()`` diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 9ecd93f909..0d14eb625c 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -847,7 +847,7 @@ bool HTTPClient::connect(void) { if(_reuse && _canReuse && connected()) { DEBUG_HTTPCLIENT("[HTTP-Client] connect: already connected, reusing connection\n"); - _client->sendNow(devnull); // clear _client's output (all of it, no timeout) + _client->sendAvailable(devnull); // clear _client's output (all of it, no timeout) return true; } From b70a00234a3451bac009c4f7b6cf9f3b033af300 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 1 Feb 2021 00:41:42 +0100 Subject: [PATCH 178/207] removed StreamPr::StreamPtr(String) --- cores/esp8266/StreamDev.h | 1 - 1 file changed, 1 deletion(-) diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 7afdbb8d34..41932d8b0c 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -159,7 +159,6 @@ class StreamPtr: public StreamNull size_t _peekPointer = 0; public: - StreamPtr(const String& string): _buffer(string.c_str()), _size(string.length()), _byteAddressable(true) { } StreamPtr(const char* buffer, size_t size): _buffer(buffer), _size(size), _byteAddressable(__byteAddressable(buffer)) { } StreamPtr(const uint8_t* buffer, size_t size): _buffer((const char*)buffer), _size(size), _byteAddressable(__byteAddressable(buffer)) { } StreamPtr(const __FlashStringHelper* buffer, size_t size): _buffer(reinterpret_cast(buffer)), _size(size), _byteAddressable(false) { } From 395dae357457ed47223f2d98fe7fe637af50e6e0 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 1 Feb 2021 01:18:54 +0100 Subject: [PATCH 179/207] reenable StreamPtr(const String &) --- cores/esp8266/StreamDev.h | 1 + 1 file changed, 1 insertion(+) diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 41932d8b0c..7afdbb8d34 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -159,6 +159,7 @@ class StreamPtr: public StreamNull size_t _peekPointer = 0; public: + StreamPtr(const String& string): _buffer(string.c_str()), _size(string.length()), _byteAddressable(true) { } StreamPtr(const char* buffer, size_t size): _buffer(buffer), _size(size), _byteAddressable(__byteAddressable(buffer)) { } StreamPtr(const uint8_t* buffer, size_t size): _buffer((const char*)buffer), _size(size), _byteAddressable(__byteAddressable(buffer)) { } StreamPtr(const __FlashStringHelper* buffer, size_t size): _buffer(reinterpret_cast(buffer)), _size(size), _byteAddressable(false) { } From 9035f51394ce2ac46b33b5b1cfd938f6d0042cbf Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 1 Feb 2021 01:37:52 +0100 Subject: [PATCH 180/207] add comments to StreamString example --- .../examples/StreamString/StreamString.ino | 73 +++++++++---------- 1 file changed, 33 insertions(+), 40 deletions(-) diff --git a/libraries/esp8266/examples/StreamString/StreamString.ino b/libraries/esp8266/examples/StreamString/StreamString.ino index 4a418973b3..f25cae7d89 100644 --- a/libraries/esp8266/examples/StreamString/StreamString.ino +++ b/libraries/esp8266/examples/StreamString/StreamString.ino @@ -20,7 +20,7 @@ void checksketch(const char* what, const char* res1, const char* res2) { #define check(what, res1, res2) checksketch(what, res1, res2) #endif -void testProgmem() { +void testStringPtrProgmem() { static const char inProgmem [] PROGMEM = "I am in progmem"; auto inProgmem2 = F("I am too in progmem"); @@ -32,16 +32,30 @@ void testProgmem() { check("NO heap occupation while streaming progmem strings", String(heap).c_str(), "0"); } -void testStream() { +void testStreamString() { String inputString = "hello"; StreamString result; - // string will not be modified by read: + // By default, reading a S2Stream(String) or a StreamString will consume the String. + // It can be disabled by calling ::reset(), (not default) + // and reenabled by calling ::setConsume(). (default) + // + // In default consume mode, reading a byte or a block will remove it from + // the String. Operations are O(n²). + // + // In non-default non-consume mode, it will just move a pointer. That one + // can be ::reset(pos) anytime. See the example below. + + + // The String included in 'result' will not be modified by read: + // (this is not the default) result.reset(); { - // use StreamString or S2Stream(String) to make a r/w Stream out of a String, - // prefer the lighter StreamPtr(String) to make a read-only Stream out of a String + // We use a a lighter StreamPtr(input) to make a read-only Stream out of + // a String that obviously should not be modified during the time the + // StreamPtr instance is used. It is used as a source to be sent to + // 'result'. result.clear(); StreamPtr(inputString).sendAll(result); @@ -60,14 +74,16 @@ void testStream() { } { - // inputString made into a Stream, in reset mode - // after input is loaded once, there's nothing to get from the stream - // but the String is left untouched + // Now inputString is made into a Stream using S2Stream, + // and set in non-consume mode (using ::reset()). + + // Then, after that input is read once, it won't be anymore readable + // until the pointer is reset. - result.clear(); S2Stream input(inputString); input.reset(); + result.clear(); input.sendAll(result); input.sendAll(result); check("S2Stream.sendAll(StreamString)", result.c_str(), "hello"); @@ -75,9 +91,7 @@ void testStream() { } { - // inputString made into a Stream, with an offset - // after input is loaded once, there's nothing to get from the stream - // but the String is left untouched + // Same as above, with an offset result.clear(); S2Stream input(inputString); @@ -104,6 +118,7 @@ void testStream() { check("setConsume(): String given from S2Stream is swallowed", inputString.c_str(), "lo"); } + // Streaming with common String constructors { StreamString cons(inputString); check("StreamString(String)", cons.c_str(), inputString.c_str()); @@ -133,35 +148,12 @@ void testStream() { check("StreamString(float)", cons.c_str(), "23.20"); } - { - StreamString result("abc"); - result = inputString; - check("StreamString = String", inputString.c_str(), result.c_str()); - - StreamString ss2 = "abc"; - result = ss2; - check("StreamString = StreamString", result.c_str(), ss2.c_str()); - - result = "abc"; - check("StreamString = char*", result.c_str(), "abc"); - - result = F("abc"); - check("StreamString = F()", result.c_str(), "abc"); - - result = 23; - check("StreamString = int", result.c_str(), "23"); - - result = 'a'; - check("StreamString = char", result.c_str(), "a"); - - result = 23.2; - check("StreamString = float", result.c_str(), "23.20"); - } - #if !CORE_MOCK - testProgmem(); + // A progmem won't use Heap when StringPtr is used + testStringPtrProgmem(); + // .. but it does when S2Stream or StreamString is used { int heap = (int)ESP.getFreeHeap(); auto stream = StreamString(F("I am in progmem")); @@ -175,7 +167,8 @@ void testStream() { } } - testProgmem(); + // (check again to be sure) + testStringPtrProgmem(); #endif } @@ -186,7 +179,7 @@ void setup() { Serial.begin(115200); delay(1000); - testStream(); + testStreamString(); Serial.printf("sizeof: String:%d Stream:%d StreamString:%d SStream:%d\n", (int)sizeof(String), (int)sizeof(Stream), (int)sizeof(StreamString), (int)sizeof(S2Stream)); From 94e8d60ece205dce0d4b9f99796c543b0c47a2a4 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 1 Feb 2021 01:42:22 +0100 Subject: [PATCH 181/207] rename ::reset() to ::resetPointer() in StringPtr and S2Stream --- cores/esp8266/StreamDev.h | 2 +- cores/esp8266/StreamString.h | 4 ++-- .../esp8266/examples/StreamString/StreamString.ino | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 7afdbb8d34..f279ba584c 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -165,7 +165,7 @@ class StreamPtr: public StreamNull StreamPtr(const __FlashStringHelper* buffer, size_t size): _buffer(reinterpret_cast(buffer)), _size(size), _byteAddressable(false) { } StreamPtr(const __FlashStringHelper* text): _buffer(reinterpret_cast(text)), _size(strlen_P((PGM_P)text)), _byteAddressable(false) { } - void reset(int pointer = 0) + void resetPointer(int pointer = 0) { _peekPointer = pointer; } diff --git a/cores/esp8266/StreamString.h b/cores/esp8266/StreamString.h index b2ebb71dba..302ae25f95 100644 --- a/cores/esp8266/StreamString.h +++ b/cores/esp8266/StreamString.h @@ -197,8 +197,8 @@ class S2Stream: public Stream // Reading this stream will mark the string as read without consuming // (not enabled by default) - // Calling reset() resets the read state and allows rereading. - void reset(int pointer = 0) + // Calling resetPointer() resets the read state and allows rereading. + void resetPointer(int pointer = 0) { peekPointer = pointer; } diff --git a/libraries/esp8266/examples/StreamString/StreamString.ino b/libraries/esp8266/examples/StreamString/StreamString.ino index f25cae7d89..27d2aea3c5 100644 --- a/libraries/esp8266/examples/StreamString/StreamString.ino +++ b/libraries/esp8266/examples/StreamString/StreamString.ino @@ -37,19 +37,19 @@ void testStreamString() { StreamString result; // By default, reading a S2Stream(String) or a StreamString will consume the String. - // It can be disabled by calling ::reset(), (not default) + // It can be disabled by calling ::resetPointer(), (not default) // and reenabled by calling ::setConsume(). (default) // // In default consume mode, reading a byte or a block will remove it from // the String. Operations are O(n²). // // In non-default non-consume mode, it will just move a pointer. That one - // can be ::reset(pos) anytime. See the example below. + // can be ::resetPointer(pos) anytime. See the example below. // The String included in 'result' will not be modified by read: // (this is not the default) - result.reset(); + result.resetPointer(); { // We use a a lighter StreamPtr(input) to make a read-only Stream out of @@ -75,13 +75,13 @@ void testStreamString() { { // Now inputString is made into a Stream using S2Stream, - // and set in non-consume mode (using ::reset()). + // and set in non-consume mode (using ::resetPointer()). // Then, after that input is read once, it won't be anymore readable // until the pointer is reset. S2Stream input(inputString); - input.reset(); + input.resetPointer(); result.clear(); input.sendAll(result); @@ -96,11 +96,11 @@ void testStreamString() { result.clear(); S2Stream input(inputString); // stream position set to offset 2 (0 by default) - input.reset(2); + input.resetPointer(2); input.sendAll(result); input.sendAll(result); - check("S2Stream.reset(2):", result.c_str(), "llo"); + check("S2Stream.resetPointer(2):", result.c_str(), "llo"); } { From 36a6bce02fe67b47058c80bccd42551ad0f94150 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 1 Feb 2021 01:53:07 +0100 Subject: [PATCH 182/207] rename StreamPtr:: to StreamConstPtr:: --- cores/esp8266/StreamDev.cpp | 6 +++--- cores/esp8266/StreamDev.h | 12 ++++++------ .../ESP8266HTTPClient/src/ESP8266HTTPClient.cpp | 4 ++-- .../ESP8266WebServer/src/ESP8266WebServer-impl.h | 10 +++++----- .../ESP8266WiFi/src/include/ClientContext.h | 4 ++-- .../examples/StreamString/StreamString.ino | 16 ++++++++-------- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index edaabc0985..ed2a63c7f1 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -271,7 +271,7 @@ size_t Stream::sendGeneric(Print* to, Stream& operator << (Stream& out, String& string) { - StreamPtr(string).sendAll(out); + StreamConstPtr(string).sendAll(out); return out; } @@ -305,13 +305,13 @@ Stream& operator << (Stream& out, Stream& stream) Stream& operator << (Stream& out, const char* text) { - StreamPtr(text).sendAll(out); + StreamConstPtr(text).sendAll(out); return out; } Stream& operator << (Stream& out, const __FlashStringHelper* text) { - StreamPtr(text).sendAll(out); + StreamConstPtr(text).sendAll(out); return out; } diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index f279ba584c..3e224420f9 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -150,7 +150,7 @@ class StreamZero: public StreamNull // - black hole as output, swallow everything, availableForWrite = infinite // - Stream buffer out as input, resettable -class StreamPtr: public StreamNull +class StreamConstPtr: public StreamNull { protected: const char* _buffer; @@ -159,11 +159,11 @@ class StreamPtr: public StreamNull size_t _peekPointer = 0; public: - StreamPtr(const String& string): _buffer(string.c_str()), _size(string.length()), _byteAddressable(true) { } - StreamPtr(const char* buffer, size_t size): _buffer(buffer), _size(size), _byteAddressable(__byteAddressable(buffer)) { } - StreamPtr(const uint8_t* buffer, size_t size): _buffer((const char*)buffer), _size(size), _byteAddressable(__byteAddressable(buffer)) { } - StreamPtr(const __FlashStringHelper* buffer, size_t size): _buffer(reinterpret_cast(buffer)), _size(size), _byteAddressable(false) { } - StreamPtr(const __FlashStringHelper* text): _buffer(reinterpret_cast(text)), _size(strlen_P((PGM_P)text)), _byteAddressable(false) { } + StreamConstPtr(const String& string): _buffer(string.c_str()), _size(string.length()), _byteAddressable(true) { } + StreamConstPtr(const char* buffer, size_t size): _buffer(buffer), _size(size), _byteAddressable(__byteAddressable(buffer)) { } + StreamConstPtr(const uint8_t* buffer, size_t size): _buffer((const char*)buffer), _size(size), _byteAddressable(__byteAddressable(buffer)) { } + StreamConstPtr(const __FlashStringHelper* buffer, size_t size): _buffer(reinterpret_cast(buffer)), _size(size), _byteAddressable(false) { } + StreamConstPtr(const __FlashStringHelper* text): _buffer(reinterpret_cast(text)), _size(strlen_P((PGM_P)text)), _byteAddressable(false) { } void resetPointer(int pointer = 0) { diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 0d14eb625c..e4319791a9 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -443,7 +443,7 @@ int HTTPClient::sendRequest(const char * type, const uint8_t * payload, size_t s } // transfer all of it, with send-timeout - if (size && StreamPtr(payload, size).sendAll(_client) != size) + if (size && StreamConstPtr(payload, size).sendAll(_client) != size) return returnError(HTTPC_ERROR_SEND_PAYLOAD_FAILED); // handle Server Response (Header) @@ -930,7 +930,7 @@ bool HTTPClient::sendHeader(const char * type) DEBUG_HTTPCLIENT("[HTTP-Client] sending request header\n-----\n%s-----\n", header.c_str()); // transfer all of it, with timeout - return StreamPtr(header).sendAll(_client) == header.length(); + return StreamConstPtr(header).sendAll(_client) == header.length(); } /** diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index 9e399feeab..441c7ed808 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -458,7 +458,7 @@ void ESP8266WebServerTemplate::send(int code, const String& content_ template void ESP8266WebServerTemplate::sendContent(const String& content) { - StreamPtr ref(content.c_str(), content.length()); + StreamConstPtr ref(content.c_str(), content.length()); sendContent(&ref); } @@ -468,7 +468,7 @@ void ESP8266WebServerTemplate::send(int code, const char* content_ty if (content_length == 0) content_length = std::max((ssize_t)0, stream->streamRemaining()); _prepareHeader(header, code, content_type, content_length); - size_t sent = StreamPtr(header).sendAll(&_currentClient); + size_t sent = StreamConstPtr(header).sendAll(&_currentClient); if (sent != header.length()) DBGWS("HTTPServer: error: sent %zd on %u bytes\n", sent, header.length()); if (content_length) @@ -477,13 +477,13 @@ void ESP8266WebServerTemplate::send(int code, const char* content_ty template void ESP8266WebServerTemplate::send_P(int code, PGM_P content_type, PGM_P content) { - StreamPtr ref(content, strlen_P(content)); + StreamConstPtr ref(content, strlen_P(content)); return send(code, String(content_type).c_str(), &ref); } template void ESP8266WebServerTemplate::send_P(int code, PGM_P content_type, PGM_P content, size_t contentLength) { - StreamPtr ref(content, contentLength); + StreamConstPtr ref(content, contentLength); return send(code, String(content_type).c_str(), &ref); } @@ -513,7 +513,7 @@ void ESP8266WebServerTemplate::sendContent_P(PGM_P content) { template void ESP8266WebServerTemplate::sendContent_P(PGM_P content, size_t size) { - StreamPtr ptr(content, size); + StreamConstPtr ptr(content, size); return sendContent(&ptr, size); } diff --git a/libraries/ESP8266WiFi/src/include/ClientContext.h b/libraries/ESP8266WiFi/src/include/ClientContext.h index 040ebb0c5f..e3c9e6c4b7 100644 --- a/libraries/ESP8266WiFi/src/include/ClientContext.h +++ b/libraries/ESP8266WiFi/src/include/ClientContext.h @@ -375,7 +375,7 @@ class ClientContext if (!_pcb) { return 0; } - StreamPtr ptr(data, size); + StreamConstPtr ptr(data, size); return _write_from_source(&ptr); } @@ -392,7 +392,7 @@ class ClientContext if (!_pcb) { return 0; } - StreamPtr ptr(buf, size); + StreamConstPtr ptr(buf, size); return _write_from_source(&ptr); } diff --git a/libraries/esp8266/examples/StreamString/StreamString.ino b/libraries/esp8266/examples/StreamString/StreamString.ino index 27d2aea3c5..c04f55f35a 100644 --- a/libraries/esp8266/examples/StreamString/StreamString.ino +++ b/libraries/esp8266/examples/StreamString/StreamString.ino @@ -25,8 +25,8 @@ void testStringPtrProgmem() { auto inProgmem2 = F("I am too in progmem"); int heap = (int)ESP.getFreeHeap(); - auto stream1 = StreamPtr(inProgmem, sizeof(inProgmem) - 1); - auto stream2 = StreamPtr(inProgmem2); + auto stream1 = StreamConstPtr(inProgmem, sizeof(inProgmem) - 1); + auto stream2 = StreamConstPtr(inProgmem2); Serial << stream1 << " - " << stream2 << "\n"; heap -= (int)ESP.getFreeHeap(); check("NO heap occupation while streaming progmem strings", String(heap).c_str(), "0"); @@ -52,16 +52,16 @@ void testStreamString() { result.resetPointer(); { - // We use a a lighter StreamPtr(input) to make a read-only Stream out of + // We use a a lighter StreamConstPtr(input) to make a read-only Stream out of // a String that obviously should not be modified during the time the - // StreamPtr instance is used. It is used as a source to be sent to + // StreamConstPtr instance is used. It is used as a source to be sent to // 'result'. result.clear(); - StreamPtr(inputString).sendAll(result); - StreamPtr(inputString).sendAll(result); - StreamPtr(inputString).sendAll(result); - check("StreamPtr.sendAll(StreamString)", result.c_str(), "hellohellohello"); + StreamConstPtr(inputString).sendAll(result); + StreamConstPtr(inputString).sendAll(result); + StreamConstPtr(inputString).sendAll(result); + check("StreamConstPtr.sendAll(StreamString)", result.c_str(), "hellohellohello"); } { From 711bfe8f883fbade5365d5503d3e53d8f9f3affc Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 1 Feb 2021 02:03:42 +0100 Subject: [PATCH 183/207] rename getLastSendReport => getLastSendResult --- cores/esp8266/Stream.h | 4 ++-- cores/esp8266/StreamDev.cpp | 12 ++++++------ .../ESP8266HTTPClient/src/ESP8266HTTPClient.cpp | 8 ++++---- libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 0a79cf26ce..4f1bc64007 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -196,7 +196,7 @@ class Stream: public Print { STREAMSEND_SHORT, } sendReport_e; - sendReport_e getLastSendReport () const { return _sendReport; } + sendReport_e getLastSendResult () const { return _sendResult; } //////////////////// @@ -214,7 +214,7 @@ class Stream: public Print { float parseFloat(char skipChar); // as parseFloat() but the given skipChar is ignored - sendReport_e _sendReport = STREAMSEND_SUCCESS; + sendReport_e _sendResult = STREAMSEND_SUCCESS; }; #endif diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index ed2a63c7f1..2dd359fe10 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -30,7 +30,7 @@ size_t Stream::sendGeneric(Print* to, const int readUntilChar, const oneShotFastMs::timeType timeoutMs) { - _sendReport = STREAMSEND_SUCCESS; + _sendResult = STREAMSEND_SUCCESS; if (len == 0) { @@ -156,7 +156,7 @@ size_t Stream::sendGeneric(Print* to, w = to->write(c); if (w != 1) { - _sendReport = STREAMSEND_WRITE_ERROR; + _sendResult = STREAMSEND_WRITE_ERROR; break; } written += 1; @@ -216,14 +216,14 @@ size_t Stream::sendGeneric(Print* to, ssize_t r = read(temp, w); if (r < 0) { - _sendReport = STREAMSEND_READ_ERROR; + _sendResult = STREAMSEND_READ_ERROR; break; } w = to->write(temp, r); written += w; if ((size_t)r != w) { - _sendReport = STREAMSEND_WRITE_ERROR; + _sendResult = STREAMSEND_WRITE_ERROR; break; } if (maxLen && w) @@ -253,7 +253,7 @@ size_t Stream::sendGeneric(Print* to, { if (timeoutMs && timedOut) { - _sendReport = STREAMSEND_TIMED_OUT; + _sendResult = STREAMSEND_TIMED_OUT; } else if ((ssize_t)written != len) { @@ -263,7 +263,7 @@ size_t Stream::sendGeneric(Print* to, // // Mark it as an error because user usually wants to get what is // asked for. - _sendReport = STREAMSEND_SHORT; + _sendResult = STREAMSEND_SHORT; } } return written; diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index e4319791a9..6a86e37f47 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -629,8 +629,8 @@ int HTTPClient::writeToStream(Stream * stream) ret = _client->sendSize(stream, len); // do we have an error? - if(_client->getLastSendReport() != Stream::STREAMSEND_SUCCESS) { - return returnError(SSEND2HTTPC(_client->getLastSendReport())); + if(_client->getLastSendResult() != Stream::STREAMSEND_SUCCESS) { + return returnError(SSEND2HTTPC(_client->getLastSendResult())); } } else if(_transferEncoding == HTTPC_TE_CHUNKED) { int size = 0; @@ -655,9 +655,9 @@ int HTTPClient::writeToStream(Stream * stream) if(len > 0) { // read len bytes with timeout int r = _client->sendSize(stream, len); - if (_client->getLastSendReport() != Stream::STREAMSEND_SUCCESS) + if (_client->getLastSendResult() != Stream::STREAMSEND_SUCCESS) // not all data transferred - return returnError(SSEND2HTTPC(_client->getLastSendReport())); + return returnError(SSEND2HTTPC(_client->getLastSendResult())); ret += r; } else { diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino index 6cd5aad9be..2bd5a2b910 100644 --- a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino @@ -128,7 +128,7 @@ void loop() { } cnt++; - switch (client.getLastSendReport()) { + switch (client.getLastSendResult()) { case Stream::STREAMSEND_SUCCESS: break; case Stream::STREAMSEND_TIMED_OUT: Serial.println("Stream::to: timeout"); break; case Stream::STREAMSEND_READ_ERROR: Serial.println("Stream::to: read error"); break; From dae26cb85917638a32e8972cb304730c8e42f704 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 1 Feb 2021 02:05:25 +0100 Subject: [PATCH 184/207] remove useless cast --- libraries/ESP8266WiFi/src/CertStoreBearSSL.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/ESP8266WiFi/src/CertStoreBearSSL.cpp b/libraries/ESP8266WiFi/src/CertStoreBearSSL.cpp index 46331bd191..b16af42aaa 100644 --- a/libraries/ESP8266WiFi/src/CertStoreBearSSL.cpp +++ b/libraries/ESP8266WiFi/src/CertStoreBearSSL.cpp @@ -201,7 +201,7 @@ const br_x509_trust_anchor *CertStore::findHashedTA(void *ctx, void *hashed_dn, free(der); return nullptr; } - if (data.read((uint8_t *)der, ci.length) != (int)ci.length) { + if (data.read(der, ci.length) != (int)ci.length) { free(der); return nullptr; } From e1e6fdb5d885a2ae1f9835400d11a10e107b38c9 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 1 Feb 2021 02:08:27 +0100 Subject: [PATCH 185/207] remove another useless cast --- libraries/ESP8266WiFi/src/WiFiClient.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/ESP8266WiFi/src/WiFiClient.cpp b/libraries/ESP8266WiFi/src/WiFiClient.cpp index 5420f3a47b..07e4d99545 100644 --- a/libraries/ESP8266WiFi/src/WiFiClient.cpp +++ b/libraries/ESP8266WiFi/src/WiFiClient.cpp @@ -197,7 +197,7 @@ bool WiFiClient::getSync() const int WiFiClient::availableForWrite () { - return _client? (int)_client->availableForWrite(): 0; + return _client? _client->availableForWrite(): 0; } size_t WiFiClient::write(uint8_t b) From 784771cc6fedf280870c1a9a0bff9b445759f2a2 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 2 Feb 2021 00:55:09 +0100 Subject: [PATCH 186/207] 32767->maxInt16 --- cores/esp8266/StreamDev.h | 7 ++++--- cores/esp8266/StreamString.h | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cores/esp8266/StreamDev.h b/cores/esp8266/StreamDev.h index 3e224420f9..1112ce73b3 100644 --- a/cores/esp8266/StreamDev.h +++ b/cores/esp8266/StreamDev.h @@ -20,6 +20,7 @@ #ifndef __STREAMDEV_H #define __STREAMDEV_H +#include #include #include @@ -47,7 +48,7 @@ class StreamNull: public Stream virtual int availableForWrite() override { - return 32767; + return std::numeric_limits::max(); } // Stream @@ -114,7 +115,7 @@ class StreamZero: public StreamNull // Stream virtual int available() override { - return 32767; + return std::numeric_limits::max(); } virtual int read() override @@ -141,7 +142,7 @@ class StreamZero: public StreamNull virtual ssize_t streamRemaining() override { - return 32767; + return std::numeric_limits::max(); } }; diff --git a/cores/esp8266/StreamString.h b/cores/esp8266/StreamString.h index 302ae25f95..a868f28d17 100644 --- a/cores/esp8266/StreamString.h +++ b/cores/esp8266/StreamString.h @@ -23,6 +23,7 @@ #ifndef __STREAMSTRING_H #define __STREAMSTRING_H +#include #include "WString.h" /////////////////////////////////////////////////////////////// @@ -50,7 +51,7 @@ class S2Stream: public Stream virtual int availableForWrite() override { - return 32767; + return std::numeric_limits::max(); } virtual int read() override From 908f980aa88c6ba8ffeede5847fb6344374f0de1 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 2 Feb 2021 01:20:08 +0100 Subject: [PATCH 187/207] remove comment --- cores/esp8266/debug.h | 1 - 1 file changed, 1 deletion(-) diff --git a/cores/esp8266/debug.h b/cores/esp8266/debug.h index c61aacf7ff..263d3e9149 100644 --- a/cores/esp8266/debug.h +++ b/cores/esp8266/debug.h @@ -33,7 +33,6 @@ extern void __iamslow(const char* what); static bool once = false; \ if (!once) { \ once = true; \ - /*if (__GXX_RTTI) DEBUGV(typeid(*this).name());*/ \ __iamslow((PGM_P)FPSTR(__FUNCTION__)); \ } \ } while (0) From b73a9ca8daa4274163fb7f14b2c6963510dc0243 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 2 Feb 2021 01:21:41 +0100 Subject: [PATCH 188/207] typo in doc --- doc/reference.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/reference.rst b/doc/reference.rst index 9a8c2c7a94..be87217b82 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -358,7 +358,7 @@ Arduino API } One will notice that in the network to serial direction, data are transfered - byte by byte while data are available. On the other direction, a temporary + byte by byte while data are available. In the other direction, a temporary buffer is created on stack, filled with available serial data, then transfered to network. From e0084f3933169792fab26d2448bd4c9e6a79e76e Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 2 Feb 2021 01:22:27 +0100 Subject: [PATCH 189/207] typo in doc --- doc/reference.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/reference.rst b/doc/reference.rst index be87217b82..32d474dbb8 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -360,7 +360,7 @@ Arduino API One will notice that in the network to serial direction, data are transfered byte by byte while data are available. In the other direction, a temporary buffer is created on stack, filled with available serial data, then - transfered to network. + transferred to network. The ``readBytes(buffer, length)`` method includes a timeout to ensure that all required bytes are received. The ``write(buffer, length)`` (inherited @@ -415,7 +415,7 @@ Stream extensions This method waits up to the given or default timeout to transfer data until the character ``delim`` is met. - Note: The delimiter is read but not transfered (like ``readBytesUntil``) + Note: The delimiter is read but not transferred (like ``readBytesUntil``) - ``Stream::sendAvailable(dest)`` @@ -446,7 +446,7 @@ Stream extensions ``F("some words in flash")`` or ``PROGMEM`` strings. This class makes no copy, even with data in flash. For flash content, byte-by-byte transfers is a consequence when "memcpy_P" cannot be used. Other - contents can be transfered at once when possible. + contents can be transferred at once when possible. .. code:: cpp From 44fb3783e978b17f1198f6993fd7224247632c11 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 2 Feb 2021 01:25:01 +0100 Subject: [PATCH 190/207] doc: insist on reference implementation --- doc/reference.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/reference.rst b/doc/reference.rst index 32d474dbb8..f279385d42 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -373,7 +373,8 @@ Arduino API len)`` method, which is similar to ``readBytes(buffer, len)`` without timeout: the returned value can be less than the requested size, so special care must be taken with this function, introduced in the Arduino - ``Client::`` class. This function has also been introduced in other classes + ``Client::`` class (cf. AVR reference implementation). + This function has also been introduced in other classes not derivating from ``Client::``, e.g. ``HardwareSerial::``. Stream extensions From 3be63d52e0b513a23ad0bc53cc17122b3fa594b3 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 2 Feb 2021 01:26:04 +0100 Subject: [PATCH 191/207] english --- doc/reference.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/reference.rst b/doc/reference.rst index f279385d42..3f7b33ee86 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -375,7 +375,7 @@ Arduino API care must be taken with this function, introduced in the Arduino ``Client::`` class (cf. AVR reference implementation). This function has also been introduced in other classes - not derivating from ``Client::``, e.g. ``HardwareSerial::``. + that don't derive from ``Client::``, e.g. ``HardwareSerial::``. Stream extensions From e29947495b0908545137842a4c212ebb0f90b8a2 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 2 Feb 2021 01:27:44 +0100 Subject: [PATCH 192/207] english --- doc/reference.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/reference.rst b/doc/reference.rst index 3f7b33ee86..146025558e 100644 --- a/doc/reference.rst +++ b/doc/reference.rst @@ -428,8 +428,8 @@ Stream extensions This method waits up to the given or default timeout to transfer all available data. It is useful when source is able to tell that no more - data will be available for this call, or when destination is able to - tell that it will no more be able to receive. + data will be available for this call, or when destination can tell + that it will no be able to receive anymore. For example, a source String will not grow during the transfer, or a particular network connection supposed to send a fixed amount of data From 085dbd22342ff298d74b5625bf6555706690ae0d Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 2 Feb 2021 01:31:29 +0100 Subject: [PATCH 193/207] SSEND2HTTPC() => StreamReportToHttpClientReport() --- libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 6a86e37f47..5cd50c144d 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -28,7 +28,7 @@ #include #include -static int SSEND2HTTPC (Stream::sendReport_e streamToError) +static int StreamReportToHttpClientReport (Stream::sendReport_e streamToError) { switch (streamToError) { @@ -630,7 +630,7 @@ int HTTPClient::writeToStream(Stream * stream) // do we have an error? if(_client->getLastSendResult() != Stream::STREAMSEND_SUCCESS) { - return returnError(SSEND2HTTPC(_client->getLastSendResult())); + return returnError(StreamReportToHttpClientReport(_client->getLastSendResult())); } } else if(_transferEncoding == HTTPC_TE_CHUNKED) { int size = 0; @@ -657,7 +657,7 @@ int HTTPClient::writeToStream(Stream * stream) int r = _client->sendSize(stream, len); if (_client->getLastSendResult() != Stream::STREAMSEND_SUCCESS) // not all data transferred - return returnError(SSEND2HTTPC(_client->getLastSendResult())); + return returnError(StreamReportToHttpClientReport(_client->getLastSendResult())); ret += r; } else { From 354288bb37ab723d1faf90a310bd3ab7e273db87 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 7 Feb 2021 23:48:19 +0100 Subject: [PATCH 194/207] changed the report enum to a class enum --- cores/esp8266/Stream.h | 29 ++++++++++--------- cores/esp8266/StreamDev.cpp | 14 ++++----- .../src/ESP8266HTTPClient.cpp | 22 +++++++------- .../examples/WiFiEcho/WiFiEcho.ino | 12 ++++---- 4 files changed, 40 insertions(+), 37 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index 4f1bc64007..b355009705 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -144,6 +144,7 @@ class Stream: public Print { // (outputCanTimeout() is defined in Print::) + //////////////////////// //////////////////// extensions: Streaming streams to streams // Stream::to*() // @@ -187,18 +188,16 @@ class Stream: public Print { // remaining size (-1 by default = unknown) virtual ssize_t streamRemaining () { return -1; } - typedef enum + enum class Report { - STREAMSEND_SUCCESS = 0, - STREAMSEND_TIMED_OUT, - STREAMSEND_READ_ERROR, - STREAMSEND_WRITE_ERROR, - STREAMSEND_SHORT, - } sendReport_e; + Success = 0, + TimedOut, + ReadError, + WriteError, + ShortOperation, + }; - sendReport_e getLastSendResult () const { return _sendResult; } - - //////////////////// + Report getLastSendReport () const { return _sendReport; } protected: size_t sendGeneric (Print* to, @@ -206,15 +205,19 @@ class Stream: public Print { const int readUntilChar = -1, oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* neverExpires=>getTimeout() */); - //////////////////// end of extensions + void setReport (Report report) { _sendReport = report; } + + private: + + Report _sendReport = Report::Success; + + //////////////////// end of extensions protected: long parseInt(char skipChar); // as parseInt() but the given skipChar is ignored // this allows format characters (typically commas) in values to be ignored float parseFloat(char skipChar); // as parseFloat() but the given skipChar is ignored - - sendReport_e _sendResult = STREAMSEND_SUCCESS; }; #endif diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamDev.cpp index 2dd359fe10..095bdef59d 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamDev.cpp @@ -30,7 +30,7 @@ size_t Stream::sendGeneric(Print* to, const int readUntilChar, const oneShotFastMs::timeType timeoutMs) { - _sendResult = STREAMSEND_SUCCESS; + setReport(Report::Success); if (len == 0) { @@ -156,7 +156,7 @@ size_t Stream::sendGeneric(Print* to, w = to->write(c); if (w != 1) { - _sendResult = STREAMSEND_WRITE_ERROR; + setReport(Report::WriteError); break; } written += 1; @@ -216,14 +216,14 @@ size_t Stream::sendGeneric(Print* to, ssize_t r = read(temp, w); if (r < 0) { - _sendResult = STREAMSEND_READ_ERROR; + setReport(Report::ReadError); break; } w = to->write(temp, r); written += w; if ((size_t)r != w) { - _sendResult = STREAMSEND_WRITE_ERROR; + setReport(Report::WriteError); break; } if (maxLen && w) @@ -249,11 +249,11 @@ size_t Stream::sendGeneric(Print* to, optimistic_yield(1000); } - if (getWriteError() == STREAMSEND_SUCCESS && maxLen > 0) + if (getLastSendReport() == Report::Success && maxLen > 0) { if (timeoutMs && timedOut) { - _sendResult = STREAMSEND_TIMED_OUT; + setReport(Report::TimedOut); } else if ((ssize_t)written != len) { @@ -263,7 +263,7 @@ size_t Stream::sendGeneric(Print* to, // // Mark it as an error because user usually wants to get what is // asked for. - _sendResult = STREAMSEND_SHORT; + setReport(Report::ShortOperation); } } return written; diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index 5cd50c144d..592a14d0cc 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -28,15 +28,15 @@ #include #include -static int StreamReportToHttpClientReport (Stream::sendReport_e streamToError) +static int StreamReportToHttpClientReport (Stream::Report streamSendError) { - switch (streamToError) + switch (streamSendError) { - case Stream::STREAMSEND_TIMED_OUT: return HTTPC_ERROR_READ_TIMEOUT; - case Stream::STREAMSEND_READ_ERROR: return HTTPC_ERROR_NO_STREAM; - case Stream::STREAMSEND_WRITE_ERROR: return HTTPC_ERROR_STREAM_WRITE; - case Stream::STREAMSEND_SHORT: return HTTPC_ERROR_STREAM_WRITE; - case Stream::STREAMSEND_SUCCESS: return 0; + case Stream::Report::TimedOut: return HTTPC_ERROR_READ_TIMEOUT; + case Stream::Report::ReadError: return HTTPC_ERROR_NO_STREAM; + case Stream::Report::WriteError: return HTTPC_ERROR_STREAM_WRITE; + case Stream::Report::ShortOperation: return HTTPC_ERROR_STREAM_WRITE; + case Stream::Report::Success: return 0; } return 0; // never reached, keep gcc quiet } @@ -629,8 +629,8 @@ int HTTPClient::writeToStream(Stream * stream) ret = _client->sendSize(stream, len); // do we have an error? - if(_client->getLastSendResult() != Stream::STREAMSEND_SUCCESS) { - return returnError(StreamReportToHttpClientReport(_client->getLastSendResult())); + if(_client->getLastSendReport() != Stream::Report::Success) { + return returnError(StreamReportToHttpClientReport(_client->getLastSendReport())); } } else if(_transferEncoding == HTTPC_TE_CHUNKED) { int size = 0; @@ -655,9 +655,9 @@ int HTTPClient::writeToStream(Stream * stream) if(len > 0) { // read len bytes with timeout int r = _client->sendSize(stream, len); - if (_client->getLastSendResult() != Stream::STREAMSEND_SUCCESS) + if (_client->getLastSendReport() != Stream::Report::Success) // not all data transferred - return returnError(StreamReportToHttpClientReport(_client->getLastSendResult())); + return returnError(StreamReportToHttpClientReport(_client->getLastSendReport())); ret += r; } else { diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino index 2bd5a2b910..5d38efaeb9 100644 --- a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino @@ -128,12 +128,12 @@ void loop() { } cnt++; - switch (client.getLastSendResult()) { - case Stream::STREAMSEND_SUCCESS: break; - case Stream::STREAMSEND_TIMED_OUT: Serial.println("Stream::to: timeout"); break; - case Stream::STREAMSEND_READ_ERROR: Serial.println("Stream::to: read error"); break; - case Stream::STREAMSEND_WRITE_ERROR: Serial.println("Stream::to: write error"); break; - case Stream::STREAMSEND_SHORT: Serial.println("Stream::to: short transfer"); break; + switch (client.getLastSendReport()) { + case Stream::Report::Success: break; + case Stream::Report::TimedOut: Serial.println("Stream::send: timeout"); break; + case Stream::Report::ReadError: Serial.println("Stream::send: read error"); break; + case Stream::Report::WriteError: Serial.println("Stream::send: write error"); break; + case Stream::Report::ShortOperation: Serial.println("Stream::send: short transfer"); break; } } From c2232651a4e65edef87dab74f2304684b4d00a4f Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sun, 7 Feb 2021 23:53:15 +0100 Subject: [PATCH 195/207] rename StreamDev.cpp -> StreamSend.cpp --- cores/esp8266/{StreamDev.cpp => StreamSend.cpp} | 2 +- tests/host/Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename cores/esp8266/{StreamDev.cpp => StreamSend.cpp} (99%) diff --git a/cores/esp8266/StreamDev.cpp b/cores/esp8266/StreamSend.cpp similarity index 99% rename from cores/esp8266/StreamDev.cpp rename to cores/esp8266/StreamSend.cpp index 095bdef59d..9e212fb549 100644 --- a/cores/esp8266/StreamDev.cpp +++ b/cores/esp8266/StreamSend.cpp @@ -1,5 +1,5 @@ /* - StreamDev.cpp - 1-copy transfer function + StreamDev.cpp - 1-copy transfer related methods Copyright (c) 2019 David Gauchard. All right reserved. This library is free software; you can redistribute it and/or diff --git a/tests/host/Makefile b/tests/host/Makefile index 0e2c18e25e..87303a9b3f 100644 --- a/tests/host/Makefile +++ b/tests/host/Makefile @@ -76,7 +76,7 @@ $(shell mkdir -p $(BINDIR)) CORE_CPP_FILES := \ $(addprefix $(abspath $(CORE_PATH))/,\ debug.cpp \ - StreamDev.cpp \ + StreamSend.cpp \ Stream.cpp \ WString.cpp \ Print.cpp \ From 756ca7224cd7e5a4b466ff0d1f9e5d49f7c572c9 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 8 Feb 2021 00:42:56 +0100 Subject: [PATCH 196/207] split Stream::StreamSend() --- cores/esp8266/Stream.h | 12 +- cores/esp8266/StreamSend.cpp | 367 ++++++++++++++++++++--------------- 2 files changed, 221 insertions(+), 158 deletions(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index b355009705..e824ff80c9 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -146,9 +146,9 @@ class Stream: public Print { //////////////////////// //////////////////// extensions: Streaming streams to streams - // Stream::to*() + // Stream::send*() // - // Stream::to*() uses 1-copy transfers when peekBuffer API is + // Stream::send*() uses 1-copy transfers when peekBuffer API is // available, or makes a regular transfer through a temporary buffer. // // - for efficiency, Stream classes should implement peekAPI when @@ -158,7 +158,7 @@ class Stream: public Print { using oneShotMs = esp8266::polledTimeout::oneShotFastMs; - // ::to*() methods: + // ::send*() methods: // - always stop before timeout when "no-more-input-possible-data" // or "no-more-output-possible-data" condition is met // - always return number of transfered bytes @@ -201,10 +201,14 @@ class Stream: public Print { protected: size_t sendGeneric (Print* to, - const ssize_t maxLen = -1, + const ssize_t len = -1, const int readUntilChar = -1, oneShotMs::timeType timeoutMs = oneShotMs::neverExpires /* neverExpires=>getTimeout() */); + size_t SendGenericPeekBuffer(Print* to, const ssize_t len, const int readUntilChar, const oneShotMs::timeType timeoutMs); + size_t SendGenericRegularUntil(Print* to, const ssize_t len, const int readUntilChar, const oneShotMs::timeType timeoutMs); + size_t SendGenericRegular(Print* to, const ssize_t len, const oneShotMs::timeType timeoutMs); + void setReport (Report report) { _sendReport = report; } private: diff --git a/cores/esp8266/StreamSend.cpp b/cores/esp8266/StreamSend.cpp index 9e212fb549..5121381cc5 100644 --- a/cores/esp8266/StreamSend.cpp +++ b/cores/esp8266/StreamSend.cpp @@ -23,12 +23,10 @@ #include #include -using esp8266::polledTimeout::oneShotFastMs; - size_t Stream::sendGeneric(Print* to, const ssize_t len, const int readUntilChar, - const oneShotFastMs::timeType timeoutMs) + const esp8266::polledTimeout::oneShotFastMs::timeType timeoutMs) { setReport(Report::Success); @@ -46,209 +44,269 @@ size_t Stream::sendGeneric(Print* to, // So we use getTimeout() for both, // (also when inputCanTimeout() is false) - // "neverExpires (default, impossible)" is translated to default timeout - oneShotFastMs timedOut(timeoutMs >= oneShotFastMs::neverExpires ? getTimeout() : timeoutMs); + if (hasPeekBufferAPI()) + return SendGenericPeekBuffer(to, len, readUntilChar, timeoutMs); - size_t written = 0; + if (readUntilChar >= 0) + return SendGenericRegularUntil(to, len, readUntilChar, timeoutMs); - // len==-1 => maxLen=0 <=> until starvation - size_t maxLen = std::max((ssize_t)0, len); + return SendGenericRegular(to, len, timeoutMs); +} - if (hasPeekBufferAPI()) - // peek-buffer API +size_t Stream::SendGenericPeekBuffer(Print* to, const ssize_t len, const int readUntilChar, const esp8266::polledTimeout::oneShotFastMs::timeType timeoutMs) +{ + // "neverExpires (default, impossible)" is translated to default timeout + esp8266::polledTimeout::oneShotFastMs timedOut(timeoutMs >= esp8266::polledTimeout::oneShotFastMs::neverExpires ? getTimeout() : timeoutMs); + // len==-1 => maxLen=0 <=> until starvation + const size_t maxLen = std::max((ssize_t)0, len); + size_t written = 0; - while (!maxLen || written < maxLen) + while (!maxLen || written < maxLen) + { + size_t avpk = peekAvailable(); + if (avpk == 0 && !inputCanTimeout()) { - size_t avpk = peekAvailable(); - if (avpk == 0 && !inputCanTimeout()) - { - // no more data to read, ever - break; - } + // no more data to read, ever + break; + } - size_t w = to->availableForWrite(); - if (w == 0 && !outputCanTimeout()) - { - // no more data can be written, ever - break; - } + size_t w = to->availableForWrite(); + if (w == 0 && !outputCanTimeout()) + { + // no more data can be written, ever + break; + } - w = std::min(w, avpk); - if (maxLen) - { - w = std::min(w, maxLen - written); - } - if (w) + w = std::min(w, avpk); + if (maxLen) + { + w = std::min(w, maxLen - written); + } + if (w) + { + const char* directbuf = peekBuffer(); + bool foundChar = false; + if (readUntilChar >= 0) { - const char* directbuf = peekBuffer(); - bool foundChar = false; - if (readUntilChar >= 0) + const char* last = (const char*)memchr(directbuf, readUntilChar, w); + if (last) { - const char* last = (const char*)memchr(directbuf, readUntilChar, w); - if (last) - { - w = std::min((size_t)(last - directbuf), w); - foundChar = true; - } + w = std::min((size_t)(last - directbuf), w); + foundChar = true; } - if (w && ((w = to->write(directbuf, w)))) - { - peekConsume(w); - written += w; - if (maxLen) - { - timedOut.reset(); - } - } - if (foundChar) + } + if (w && ((w = to->write(directbuf, w)))) + { + peekConsume(w); + written += w; + if (maxLen) { - peekConsume(1); - break; + timedOut.reset(); } } - - if (!w && !maxLen && readUntilChar < 0) + if (foundChar) { - // nothing has been transferred and no specific condition is requested + peekConsume(1); break; } + } - if (timedOut) - { - // either (maxLen>0) nothing has been transferred for too long - // or readUntilChar >= 0 but char is not encountered for too long - // or (maxLen=0) too much time has been spent here - break; - } + if (!w && !maxLen && readUntilChar < 0) + { + // nothing has been transferred and no specific condition is requested + break; + } + + if (timedOut) + { + // either (maxLen>0) nothing has been transferred for too long + // or readUntilChar >= 0 but char is not encountered for too long + // or (maxLen=0) too much time has been spent here + break; + } + + optimistic_yield(1000); + } - optimistic_yield(1000); + if (getLastSendReport() == Report::Success && maxLen > 0) + { + if (timeoutMs && timedOut) + { + setReport(Report::TimedOut); + } + else if ((ssize_t)written != len) + { + // This is happening when source cannot timeout (ex: a String) + // but has not enough data, or a dest has closed or cannot + // timeout but is too small (String, buffer...) + // + // Mark it as an error because user usually wants to get what is + // asked for. + setReport(Report::ShortOperation); } + } + + return written; +} + +size_t Stream::SendGenericRegularUntil(Print* to, const ssize_t len, const int readUntilChar, const esp8266::polledTimeout::oneShotFastMs::timeType timeoutMs) +{ + // regular Stream API + // no other choice than reading byte by byte + + // "neverExpires (default, impossible)" is translated to default timeout + esp8266::polledTimeout::oneShotFastMs timedOut(timeoutMs >= esp8266::polledTimeout::oneShotFastMs::neverExpires ? getTimeout() : timeoutMs); + // len==-1 => maxLen=0 <=> until starvation + const size_t maxLen = std::max((ssize_t)0, len); + size_t written = 0; - else if (readUntilChar >= 0) + while (!maxLen || written < maxLen) + { + size_t avpk = peekAvailable(); + if (avpk == 0 && !inputCanTimeout()) + { + // no more data to read, ever + break; + } - // regular Stream API - // no other choice than reading byte by byte + size_t w = to->availableForWrite(); + if (w == 0 && !outputCanTimeout()) + { + // no more data can be written, ever + break; + } - while (!maxLen || written < maxLen) + int c = read(); + if (c != -1) { - size_t avpk = peekAvailable(); - if (avpk == 0 && !inputCanTimeout()) + if (c == readUntilChar) { - // no more data to read, ever break; } - - size_t w = to->availableForWrite(); - if (w == 0 && !outputCanTimeout()) + w = to->write(c); + if (w != 1) { - // no more data can be written, ever + setReport(Report::WriteError); break; } - - int c = read(); - if (c != -1) + written += 1; + if (maxLen) { - if (c == readUntilChar) - { - break; - } - w = to->write(c); - if (w != 1) - { - setReport(Report::WriteError); - break; - } - written += 1; - if (maxLen) - { - timedOut.reset(); - } + timedOut.reset(); } + } - if (!w && !maxLen && readUntilChar < 0) - { - // nothing has been transferred and no specific condition is requested - break; - } + if (!w && !maxLen && readUntilChar < 0) + { + // nothing has been transferred and no specific condition is requested + break; + } - if (timedOut) - { - // either (maxLen>0) nothing has been transferred for too long - // or readUntilChar >= 0 but char is not encountered for too long - // or (maxLen=0) too much time has been spent here - break; - } + if (timedOut) + { + // either (maxLen>0) nothing has been transferred for too long + // or readUntilChar >= 0 but char is not encountered for too long + // or (maxLen=0) too much time has been spent here + break; + } + + optimistic_yield(1000); + } - optimistic_yield(1000); + if (getLastSendReport() == Report::Success && maxLen > 0) + { + if (timeoutMs && timedOut) + { + setReport(Report::TimedOut); + } + else if ((ssize_t)written != len) + { + // This is happening when source cannot timeout (ex: a String) + // but has not enough data, or a dest has closed or cannot + // timeout but is too small (String, buffer...) + // + // Mark it as an error because user usually wants to get what is + // asked for. + setReport(Report::ShortOperation); } + } - else + return written; +} + +size_t Stream::SendGenericRegular(Print* to, const ssize_t len, const esp8266::polledTimeout::oneShotFastMs::timeType timeoutMs) +{ + // regular Stream API + // use an intermediary buffer - // regular Stream API - // use an intermediary buffer + // "neverExpires (default, impossible)" is translated to default timeout + esp8266::polledTimeout::oneShotFastMs timedOut(timeoutMs >= esp8266::polledTimeout::oneShotFastMs::neverExpires ? getTimeout() : timeoutMs); + // len==-1 => maxLen=0 <=> until starvation + const size_t maxLen = std::max((ssize_t)0, len); + size_t written = 0; - while (!maxLen || written < maxLen) + while (!maxLen || written < maxLen) + { + size_t avr = available(); + if (avr == 0 && !inputCanTimeout()) { - size_t avr = available(); - if (avr == 0 && !inputCanTimeout()) - { - // no more data to read, ever - break; - } + // no more data to read, ever + break; + } - size_t w = to->availableForWrite(); - if (w == 0 && !to->outputCanTimeout()) - // no more data can be written, ever - { - break; - } + size_t w = to->availableForWrite(); + if (w == 0 && !to->outputCanTimeout()) + // no more data can be written, ever + { + break; + } - w = std::min(w, avr); - if (maxLen) - { - w = std::min(w, maxLen - written); - } - w = std::min(w, (decltype(w))64); //XXX FIXME 64 is a constant - if (w) + w = std::min(w, avr); + if (maxLen) + { + w = std::min(w, maxLen - written); + } + w = std::min(w, (decltype(w))64); //XXX FIXME 64 is a constant + if (w) + { + char temp[w]; + ssize_t r = read(temp, w); + if (r < 0) { - char temp[w]; - ssize_t r = read(temp, w); - if (r < 0) - { - setReport(Report::ReadError); - break; - } - w = to->write(temp, r); - written += w; - if ((size_t)r != w) - { - setReport(Report::WriteError); - break; - } - if (maxLen && w) - { - timedOut.reset(); - } + setReport(Report::ReadError); + break; } - - if (!w && !maxLen && readUntilChar < 0) + w = to->write(temp, r); + written += w; + if ((size_t)r != w) { - // nothing has been transferred and no specific condition is requested + setReport(Report::WriteError); break; } - - if (timedOut) + if (maxLen && w) { - // either (maxLen>0) nothing has been transferred for too long - // or readUntilChar >= 0 but char is not encountered for too long - // or (maxLen=0) too much time has been spent here - break; + timedOut.reset(); } + } + + if (!w && !maxLen) + { + // nothing has been transferred and no specific condition is requested + break; + } - optimistic_yield(1000); + if (timedOut) + { + // either (maxLen>0) nothing has been transferred for too long + // or readUntilChar >= 0 but char is not encountered for too long + // or (maxLen=0) too much time has been spent here + break; } + optimistic_yield(1000); + } + if (getLastSendReport() == Report::Success && maxLen > 0) { if (timeoutMs && timedOut) @@ -266,6 +324,7 @@ size_t Stream::sendGeneric(Print* to, setReport(Report::ShortOperation); } } + return written; } From 747a1a96e6bbe5412069da75eac947476cf14d11 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 8 Feb 2021 00:48:09 +0100 Subject: [PATCH 197/207] rename StreamDev.cpp -> StreamSend.cpp --- tests/restyle.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/restyle.sh b/tests/restyle.sh index 2b550cb49f..ea454069c5 100755 --- a/tests/restyle.sh +++ b/tests/restyle.sh @@ -18,7 +18,7 @@ cores/esp8266/Lwip* cores/esp8266/debug* cores/esp8266/core_esp8266_si2c.cpp cores/esp8266/StreamString.* -cores/esp8266/StreamDev.* +cores/esp8266/StreamSend.* libraries/Netdump " From bfe7ee7eee9bee03d15fd6d3427aaaaa5f96307f Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 8 Feb 2021 00:48:53 +0100 Subject: [PATCH 198/207] style --- cores/esp8266/StreamSend.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cores/esp8266/StreamSend.cpp b/cores/esp8266/StreamSend.cpp index 5121381cc5..cc1cb0d0bb 100644 --- a/cores/esp8266/StreamSend.cpp +++ b/cores/esp8266/StreamSend.cpp @@ -45,10 +45,14 @@ size_t Stream::sendGeneric(Print* to, // (also when inputCanTimeout() is false) if (hasPeekBufferAPI()) + { return SendGenericPeekBuffer(to, len, readUntilChar, timeoutMs); + } if (readUntilChar >= 0) + { return SendGenericRegularUntil(to, len, readUntilChar, timeoutMs); + } return SendGenericRegular(to, len, timeoutMs); } From 1188958dc88d5cfb464203b0f1ffecca9747cd64 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 15 Feb 2021 20:02:43 +0100 Subject: [PATCH 199/207] replace sendcontent()/template by proper sendcontent(ref) --- libraries/ESP8266WebServer/src/ESP8266WebServer.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer.h b/libraries/ESP8266WebServer/src/ESP8266WebServer.h index 91755e50a4..76c43cc822 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer.h @@ -179,10 +179,7 @@ class ESP8266WebServerTemplate void sendContent(const char *content, size_t size) { sendContent_P(content, size); } void sendContent(Stream* content, ssize_t content_length = 0); - template - void sendContent(T& content, ssize_t content_length = 0) { - sendContent(&content, content_length); - } + void sendContent(Stream& content, ssize_t content_length = 0) { sendContent(&content, content_length); } bool chunkedResponseModeStart_P (int code, PGM_P content_type) { if (_currentVersion == 0) From bfad4fc4b6a20ee2a2993299be7d4f6c05b39f78 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 15 Feb 2021 23:22:46 +0100 Subject: [PATCH 200/207] check for outputCanTimeout() on right stream --- cores/esp8266/StreamSend.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cores/esp8266/StreamSend.cpp b/cores/esp8266/StreamSend.cpp index cc1cb0d0bb..4c315adb78 100644 --- a/cores/esp8266/StreamSend.cpp +++ b/cores/esp8266/StreamSend.cpp @@ -76,7 +76,7 @@ size_t Stream::SendGenericPeekBuffer(Print* to, const ssize_t len, const int rea } size_t w = to->availableForWrite(); - if (w == 0 && !outputCanTimeout()) + if (w == 0 && !to->outputCanTimeout()) { // no more data can be written, ever break; @@ -175,7 +175,7 @@ size_t Stream::SendGenericRegularUntil(Print* to, const ssize_t len, const int r } size_t w = to->availableForWrite(); - if (w == 0 && !outputCanTimeout()) + if (w == 0 && !to->outputCanTimeout()) { // no more data can be written, ever break; From 70eb9e8a95487af5fda8a2cbb7201ef4418152ab Mon Sep 17 00:00:00 2001 From: david gauchard Date: Mon, 15 Feb 2021 23:36:00 +0100 Subject: [PATCH 201/207] webserver: check/print in debug mode --- libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index 441c7ed808..c6ee53350a 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -497,7 +497,10 @@ void ESP8266WebServerTemplate::sendContent(Stream* content, ssize_t _currentClient.printf("%zx\r\n", content_length); } size_t sent = content->sendSize(&_currentClient, content_length); - (void)sent; //XXXFIXME if (sent != content_length) print-error-on-console-and-return-false + if (sent != content_length) + { + DBGWS("HTTPServer: error: short send after timeout (%u<%d)\n", sent, content_length); + } if(_chunked) { _currentClient.printf_P(PSTR("\r\n")); if (content_length == 0) { From e88d21e0c6fb1a3ce51bcc714ffcc3f0bdb7fa3e Mon Sep 17 00:00:00 2001 From: david gauchard Date: Tue, 16 Feb 2021 04:25:49 +0100 Subject: [PATCH 202/207] fix variable type --- libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h index c6ee53350a..c48aa01905 100644 --- a/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h +++ b/libraries/ESP8266WebServer/src/ESP8266WebServer-impl.h @@ -496,10 +496,10 @@ void ESP8266WebServerTemplate::sendContent(Stream* content, ssize_t if(_chunked) { _currentClient.printf("%zx\r\n", content_length); } - size_t sent = content->sendSize(&_currentClient, content_length); + ssize_t sent = content->sendSize(&_currentClient, content_length); if (sent != content_length) { - DBGWS("HTTPServer: error: short send after timeout (%u<%d)\n", sent, content_length); + DBGWS("HTTPServer: error: short send after timeout (%d<%d)\n", sent, content_length); } if(_chunked) { _currentClient.printf_P(PSTR("\r\n")); From 594d77b34d6823d880117aef59ce3d4496a35cd3 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 10 Mar 2021 18:59:03 +0100 Subject: [PATCH 203/207] bugfix: avoid using peekBuffer API when not available --- cores/esp8266/StreamSend.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cores/esp8266/StreamSend.cpp b/cores/esp8266/StreamSend.cpp index 4c315adb78..3722a6478c 100644 --- a/cores/esp8266/StreamSend.cpp +++ b/cores/esp8266/StreamSend.cpp @@ -167,8 +167,8 @@ size_t Stream::SendGenericRegularUntil(Print* to, const ssize_t len, const int r while (!maxLen || written < maxLen) { - size_t avpk = peekAvailable(); - if (avpk == 0 && !inputCanTimeout()) + size_t avr = available(); + if (avr == 0 && !inputCanTimeout()) { // no more data to read, ever break; From b4ca0ddac1040c81bf8d385700805c0a5ca0c317 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 10 Mar 2021 19:02:32 +0100 Subject: [PATCH 204/207] use a constexpr for stack buffer size --- cores/esp8266/Stream.h | 1 + cores/esp8266/StreamSend.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index e824ff80c9..df8fd5411a 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -157,6 +157,7 @@ class Stream: public Print { // should implement {output,input}CanTimeout() using oneShotMs = esp8266::polledTimeout::oneShotFastMs; + constexpr int temporaryStackBufferSize = 64; // ::send*() methods: // - always stop before timeout when "no-more-input-possible-data" diff --git a/cores/esp8266/StreamSend.cpp b/cores/esp8266/StreamSend.cpp index 3722a6478c..f743889b62 100644 --- a/cores/esp8266/StreamSend.cpp +++ b/cores/esp8266/StreamSend.cpp @@ -271,7 +271,7 @@ size_t Stream::SendGenericRegular(Print* to, const ssize_t len, const esp8266::p { w = std::min(w, maxLen - written); } - w = std::min(w, (decltype(w))64); //XXX FIXME 64 is a constant + w = std::min(w, (decltype(w))temporaryStackBufferSize); if (w) { char temp[w]; From 81290b6bc8b0e1cd6faefbfa3f9a63985224afee Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 10 Mar 2021 22:15:00 +0100 Subject: [PATCH 205/207] update messages and python code in wifiecho example --- .../ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino | 15 ++++++++------- .../examples/WiFiEcho/echo-client.py | 18 +++++++++--------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino index 5d38efaeb9..0962b08365 100644 --- a/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/WiFiEcho.ino @@ -47,10 +47,11 @@ void setup() { MDNS.begin("echo23"); - Serial.print("Ready! Use 'telnet/nc "); - Serial.print(WiFi.localIP()); - Serial.printf(" %d' to try echo, use a bandwidth meter and try typing 1, 2 or 3 on console during transfer\n", port); - Serial.printf("Use 'python3 echo-client.py' to measure bandwidth and compare algorithms\n"); + Serial.printf("Ready!\n" + "- Use 'telnet/nc echo23.local %d' to try echo\n\n" + "- Use 'python3 echo-client.py' bandwidth meter to compare transfer APIs\n\n" + " and try typing 1, 1, 1, 2, 2, 2, 3, 3, 3 on console during transfers\n\n", + port); } @@ -83,9 +84,9 @@ void loop() { if (Serial.available()) { s = (s + 1) % (sizeof(sizes) / sizeof(sizes[0])); switch (Serial.read()) { - case '1': if (t != 1) s = 0; t = 1; Serial.println("byte-by-byte"); break; - case '2': if (t != 2) s = 1; t = 2; Serial.printf("through buffer\n"); break; - case '3': if (t != 3) s = 0; t = 3; Serial.printf("direct access\n"); break; + case '1': if (t != 1) s = 0; t = 1; Serial.println("byte-by-byte (watch then press 2 or 3)"); break; + case '2': if (t != 2) s = 1; t = 2; Serial.printf("through buffer (watch then press 2 again, or 1 or 3)\n"); break; + case '3': if (t != 3) s = 0; t = 3; Serial.printf("direct access (watch then press 3 again, or 1 or 2)\n"); break; } tot = cnt = 0; ESP.resetFreeContStack(); diff --git a/libraries/ESP8266WiFi/examples/WiFiEcho/echo-client.py b/libraries/ESP8266WiFi/examples/WiFiEcho/echo-client.py index 3e78b6af37..55c90073b7 100755 --- a/libraries/ESP8266WiFi/examples/WiFiEcho/echo-client.py +++ b/libraries/ESP8266WiFi/examples/WiFiEcho/echo-client.py @@ -11,16 +11,16 @@ global recv recv = 0 -async def tcp_echo_open (ip, port, loop): - return await asyncio.open_connection(ip, port, loop=loop) +async def tcp_echo_open (ip, port): + return await asyncio.open_connection(ip, port) -async def tcp_echo_sender(message, writer, loop): +async def tcp_echo_sender(message, writer): print('Writer started') while True: writer.write(message) await writer.drain() -async def tcp_echo_receiver(message, reader, loop): +async def tcp_echo_receiver(message, reader): global recv print('Reader started') while True: @@ -31,7 +31,7 @@ async def tcp_echo_receiver(message, reader, loop): if data != message: print('error') -async def tcp_stat(loop): +async def tcp_stat(): global recv dur = 0 loopsec = 2 @@ -42,8 +42,8 @@ async def tcp_stat(loop): print('BW=', (recv - last) * 2 * 8 / 1024 / loopsec, 'Kibits/s avg=', recv * 2 * 8 / 1024 / dur) loop = asyncio.get_event_loop() -reader, writer = loop.run_until_complete(tcp_echo_open('echo23.local', 23, loop)) -loop.create_task(tcp_echo_receiver(message, reader, loop)) -loop.create_task(tcp_echo_sender(message, writer, loop)) -loop.create_task(tcp_stat(loop)) +reader, writer = loop.run_until_complete(tcp_echo_open('echo23.local', 23)) +loop.create_task(tcp_echo_receiver(message, reader)) +loop.create_task(tcp_echo_sender(message, writer)) +loop.create_task(tcp_stat()) loop.run_forever() From 32eb9bfc24556579254a6df4e5d9ec392296400f Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 10 Mar 2021 22:15:37 +0100 Subject: [PATCH 206/207] constexpr are static in classes --- cores/esp8266/Stream.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cores/esp8266/Stream.h b/cores/esp8266/Stream.h index df8fd5411a..340d4f9c78 100644 --- a/cores/esp8266/Stream.h +++ b/cores/esp8266/Stream.h @@ -157,7 +157,7 @@ class Stream: public Print { // should implement {output,input}CanTimeout() using oneShotMs = esp8266::polledTimeout::oneShotFastMs; - constexpr int temporaryStackBufferSize = 64; + static constexpr int temporaryStackBufferSize = 64; // ::send*() methods: // - always stop before timeout when "no-more-input-possible-data" From f0498b57f1b6df110e1f6aabc712c9d50510c8af Mon Sep 17 00:00:00 2001 From: david gauchard Date: Sat, 13 Mar 2021 22:38:57 +0100 Subject: [PATCH 207/207] fix by @devyte's internal review: At this point it was found that wpos is above rpos, i.e.: the ringbuffer has a single contiguous block, with possible free mem at the bottom and at the top. If at this point there is an interrupt that causes wpos to wrap, the returned value below will be wrong, i.e.: some large number because of negative result returned as size_t type. Then, in SendGenericPeekBuffer(), the to->write() call will crash. --- cores/esp8266/uart.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cores/esp8266/uart.cpp b/cores/esp8266/uart.cpp index 55e15c9ec7..0b618e2b7f 100644 --- a/cores/esp8266/uart.cpp +++ b/cores/esp8266/uart.cpp @@ -250,10 +250,12 @@ size_t uart_peek_available (uart_t* uart) ETS_UART_INTR_DISABLE(); uart_rx_copy_fifo_to_buffer_unsafe(uart); + auto rpos = uart->rx_buffer->rpos; + auto wpos = uart->rx_buffer->wpos; ETS_UART_INTR_ENABLE(); - if(uart->rx_buffer->wpos < uart->rx_buffer->rpos) - return uart->rx_buffer->size - uart->rx_buffer->rpos; - return uart->rx_buffer->wpos - uart->rx_buffer->rpos; + if(wpos < rpos) + return uart->rx_buffer->size - rpos; + return wpos - rpos; } // return a pointer to available data buffer (size = available())