Merge branch 'master' into ethernet

Author: d-a-v

README.md

@@ -30,7 +30,7 @@ Starting with 1.6.4, Arduino allows installation of third-party platform package
 
 - Install the current upstream Arduino IDE at the 1.8.9 level or later. The current version is on the [Arduino website](https://www.arduino.cc/en/main/software).
 - Start Arduino and open the Preferences window.
-- Enter ```https://arduino.esp8266.com/stable/package_esp8266com_index.json``` into the *Additional Board Manager URLs* field. You can add multiple URLs, separating them with commas.
+- Enter ```https://arduino.esp8266.com/stable/package_esp8266com_index.json``` into the *File>Preferences>Additional Boards Manager URLs* field of the Arduino IDE. You can add multiple URLs, separating them with commas.
 - Open Boards Manager from Tools > Board menu and install *esp8266* platform (and don't forget to select your ESP8266 board from Tools > Board menu after installation).
 
 #### Latest release [![Latest release](https://img.shields.io/github/release/esp8266/Arduino.svg)](https://github.com/esp8266/Arduino/releases/latest/)

boards.txt

@@ -68,7 +68,9 @@ int load_app_from_flash_raw(const uint32_t flash_addr)
             load = true;
         }
 
-        if (address >= 0x40100000 && address < 0x40108000) {
+        // The final IRAM size, once boot has completed, can be either 32K or 48K.
+        // Allow for the higher in range testing.
+        if (address >= 0x40100000 && address < 0x4010C000) {
             load = true;
         }
 

bootloaders/eboot/eboot.c

@@ -225,6 +225,10 @@ void optimistic_yield(uint32_t interval_us);
 #include <cstdlib>
 #include <cmath>
 
+
+#include "mmu_iram.h"
+
+
 using std::min;
 using std::max;
 using std::round;

bootloaders/eboot/eboot.elf

@@ -30,14 +30,15 @@ void EspClass::getHeapStats(uint32_t* hfree, uint16_t* hmax, uint8_t* hfrag)
     // 100 * (1 - sqrt(sum(hole-size²)) / sum(hole-size))
 
     umm_info(NULL, false);
-    uint8_t block_size = umm_block_size();
+
+    uint32_t free_size = umm_free_heap_size_core(umm_get_current_heap());
     if (hfree)
-        *hfree = ummHeapInfo.freeBlocks * block_size;
+        *hfree = free_size;
     if (hmax)
-        *hmax = (uint16_t)ummHeapInfo.maxFreeContiguousBlocks * block_size;
+        *hmax = (uint16_t)umm_max_block_size_core(umm_get_current_heap());
     if (hfrag) {
-      if (ummHeapInfo.freeBlocks) {
-        *hfrag = 100 - (sqrt32(ummHeapInfo.freeBlocksSquared) * 100) / ummHeapInfo.freeBlocks;
+      if (free_size) {
+        *hfrag = umm_fragmentation_metric_core(umm_get_current_heap());
       } else {
         *hfrag = 0;
       }
@@ -46,11 +47,5 @@ void EspClass::getHeapStats(uint32_t* hfree, uint16_t* hmax, uint8_t* hfrag)
 
 uint8_t EspClass::getHeapFragmentation()
 {
-#ifdef UMM_INLINE_METRICS
-    return (uint8_t)umm_fragmentation_metric();
-#else
-    uint8_t hfrag;
-    getHeapStats(nullptr, nullptr, &hfrag);
-    return hfrag;
-#endif
+  return (uint8_t)umm_fragmentation_metric();
 }

cores/esp8266/Arduino.h

@@ -28,6 +28,8 @@
 #include "cont.h"
 
 #include "coredecls.h"
+#include "umm_malloc/umm_malloc.h"
+// #include "core_esp8266_vm.h"
 #include <pgmspace.h>
 
 extern "C" {
@@ -516,45 +518,45 @@ uint8_t *EspClass::random(uint8_t *resultArray, const size_t outputSizeBytes) co
 {
   /**
    * The ESP32 Technical Reference Manual v4.1 chapter 24 has the following to say about random number generation (no information found for ESP8266):
-   * 
+   *
    * "When used correctly, every 32-bit value the system reads from the RNG_DATA_REG register of the random number generator is a true random number.
    * These true random numbers are generated based on the noise in the Wi-Fi/BT RF system.
    * When Wi-Fi and BT are disabled, the random number generator will give out pseudo-random numbers.
-   * 
+   *
    * When Wi-Fi or BT is enabled, the random number generator is fed two bits of entropy every APB clock cycle (normally 80 MHz).
    * Thus, for the maximum amount of entropy, it is advisable to read the random register at a maximum rate of 5 MHz.
    * A data sample of 2 GB, read from the random number generator with Wi-Fi enabled and the random register read at 5 MHz,
    * has been tested using the Dieharder Random Number Testsuite (version 3.31.1).
    * The sample passed all tests."
-   * 
+   *
    * Since ESP32 is the sequal to ESP8266 it is unlikely that the ESP8266 is able to generate random numbers more quickly than 5 MHz when run at a 80 MHz frequency.
    * A maximum random number frequency of 0.5 MHz is used here to leave some margin for possibly inferior components in the ESP8266.
    * It should be noted that the ESP8266 has no Bluetooth functionality, so turning the WiFi off is likely to cause RANDOM_REG32 to use pseudo-random numbers.
-   * 
-   * It is possible that yield() must be called on the ESP8266 to properly feed the hardware random number generator new bits, since there is only one processor core available. 
+   *
+   * It is possible that yield() must be called on the ESP8266 to properly feed the hardware random number generator new bits, since there is only one processor core available.
    * However, no feeding requirements are mentioned in the ESP32 documentation, and using yield() could possibly cause extended delays during number generation.
    * Thus only delayMicroseconds() is used below.
-   */ 
+   */
 
   constexpr uint8_t cooldownMicros = 2;
   static uint32_t lastCalledMicros = micros() - cooldownMicros;
 
   uint32_t randomNumber = 0;
-  
+
   for(size_t byteIndex = 0; byteIndex < outputSizeBytes; ++byteIndex)
   {
     if(byteIndex % 4 == 0)
     {
       // Old random number has been used up (random number could be exactly 0, so we can't check for that)
-              
+
       uint32_t timeSinceLastCall = micros() - lastCalledMicros;
       if(timeSinceLastCall < cooldownMicros)
         delayMicroseconds(cooldownMicros - timeSinceLastCall);
-      
+
       randomNumber = RANDOM_REG32;
       lastCalledMicros = micros();
     }
-    
+
     resultArray[byteIndex] = randomNumber;
     randomNumber >>= 8;
   }
@@ -969,3 +971,57 @@ String EspClass::getSketchMD5()
     result = md5.toString();
     return result;
 }
+
+void EspClass::enableVM()
+{
+#ifdef UMM_HEAP_EXTERNAL
+    if (!vmEnabled)
+        install_vm_exception_handler();
+    vmEnabled = true;
+#endif
+}
+
+void EspClass::setExternalHeap()
+{
+#ifdef UMM_HEAP_EXTERNAL
+    if (vmEnabled)
+        umm_push_heap(UMM_HEAP_EXTERNAL);
+#endif
+}
+
+void EspClass::setIramHeap()
+{
+#ifdef UMM_HEAP_IRAM
+    umm_push_heap(UMM_HEAP_IRAM);
+#endif
+}
+
+void EspClass::setDramHeap()
+{
+#if defined(UMM_HEAP_EXTERNAL) && !defined(UMM_HEAP_IRAM)
+    if (vmEnabled) {
+        if (!umm_push_heap(UMM_HEAP_DRAM)) {
+            panic();
+        }
+    }
+#elif defined(UMM_HEAP_IRAM)
+    if (!umm_push_heap(UMM_HEAP_DRAM)) {
+        panic();
+    }
+#endif
+}
+
+void EspClass::resetHeap()
+{
+#if defined(UMM_HEAP_EXTERNAL) && !defined(UMM_HEAP_IRAM)
+    if (vmEnabled) {
+        if (!umm_pop_heap()) {
+            panic();
+        }
+    }
+#elif defined(UMM_HEAP_IRAM)
+    if (!umm_pop_heap()) {
+        panic();
+    }
+#endif
+}

cores/esp8266/Esp-frag.cpp

@@ -123,7 +123,7 @@ class EspClass {
         uint8_t getBootMode();
 
 #if defined(F_CPU) || defined(CORE_MOCK)
-        constexpr 
+        constexpr
 #endif
             inline uint8_t getCpuFreqMHz() const __attribute__((always_inline))
         {
@@ -215,7 +215,15 @@ class EspClass {
 #else
         uint32_t getCycleCount();
 #endif // !defined(CORE_MOCK)
+        void enableVM();
+        void setDramHeap();
+        void setIramHeap();
+        void setExternalHeap();
+        void resetHeap();
     private:
+#ifdef UMM_HEAP_EXTERNAL
+        bool vmEnabled = false;
+#endif
         /**
          * @brief Replaces @a byteCount bytes of a 4 byte block on flash
          *

cores/esp8266/Esp.cpp

@@ -46,6 +46,14 @@ int File::available() {
     return _p->size() - _p->position();
 }
 
+int File::availableForWrite() {
+    if (!_p)
+        return false;
+
+    return _p->availableForWrite();
+}
+
+
 int File::read() {
     if (!_p)
         return -1;

cores/esp8266/Esp.h

@@ -57,6 +57,7 @@ class File : public Stream
     // Print methods:
     size_t write(uint8_t) override;
     size_t write(const uint8_t *buf, size_t size) override;
+    int availableForWrite() override;
 
     // Stream methods:
     int available() override;

cores/esp8266/FS.cpp

@@ -35,6 +35,7 @@ class FileImpl {
     virtual bool seek(uint32_t pos, SeekMode mode) = 0;
     virtual size_t position() const = 0;
     virtual size_t size() const = 0;
+    virtual int availableForWrite() { return 0; }
     virtual bool truncate(uint32_t size) = 0;
     virtual void close() = 0;
     virtual const char* name() const = 0;

cores/esp8266/FS.h

@@ -150,7 +150,7 @@ class HardwareSerial: public Stream
     {
         return readBytes((char*)buffer, size);
     }
-    int availableForWrite(void)
+    int availableForWrite(void) override
     {
         return static_cast<int>(uart_tx_free(_uart));
     }

cores/esp8266/FSImpl.h

@@ -75,6 +75,10 @@ class Print {
         inline size_t write(char c) { return write((uint8_t) c); }
         inline size_t write(int8_t c) { return write((uint8_t) c); }
 
+        // default to zero, meaning "a single write may block"
+        // should be overriden by subclasses with buffering
+        virtual int availableForWrite() { return 0; }
+
         size_t printf(const char * format, ...)  __attribute__ ((format (printf, 2, 3)));
         size_t printf_P(PGM_P format, ...) __attribute__((format(printf, 2, 3)));
         size_t print(const __FlashStringHelper *);

cores/esp8266/HardwareSerial.h

@@ -102,6 +102,7 @@ bool schedule_function(const std::function<void(void)>& fn)
     return true;
 }
 
+IRAM_ATTR // (not only) called from ISR
 bool schedule_recurrent_function_us(const std::function<bool(void)>& fn,
     uint32_t repeat_us, const std::function<bool(void)>& alarm)
 {

cores/esp8266/Print.h

@@ -31,6 +31,8 @@
 #include "debug.h"
 #include "StackThunk.h"
 #include <ets_sys.h>
+#include <umm_malloc/umm_malloc.h>
+#include <umm_malloc/umm_heap_select.h>
 
 extern "C" {
 
@@ -48,7 +50,14 @@ void stack_thunk_add_ref()
 {
   stack_thunk_refcnt++;
   if (stack_thunk_refcnt == 1) {
+    DBG_MMU_PRINTF("\nStackThunk malloc(%u)\n", _stackSize * sizeof(uint32_t));
+    // The stack must be in DRAM, or an Soft WDT will follow. Not sure why,
+    // maybe too much time is consumed with the non32-bit exception handler.
+    // Also, interrupt handling on an IRAM stack would be very slow.
+    // Strings on the stack would be very slow to access as well.
+    HeapSelectDram ephemeral;
     stack_thunk_ptr = (uint32_t *)malloc(_stackSize * sizeof(uint32_t));
+    DBG_MMU_PRINTF("StackThunk stack_thunk_ptr: %p\n", stack_thunk_ptr);
     if (!stack_thunk_ptr) {
         // This is a fatal error, stop the sketch
         DEBUGV("Unable to allocate BearSSL stack\n");

cores/esp8266/Schedule.cpp

@@ -1,10 +1,11 @@
 
 // autogenerated from https://raw.githubusercontent.com/nayarsystems/posix_tz_db/master/zones.csv
 // by script <esp8266 arduino core>/tools/TZupdate.sh
-// Tue Jul  7 07:38:29 UTC 2020
+// Thu Nov 12 04:07:03 UTC 2020
 //
 // This database is autogenerated from IANA timezone database
-//    https://www.iana.org/time-zones
+//    https://raw.githubusercontent.com/nayarsystems/posix_tz_db/master/zones.csv
+//    (using https://www.iana.org/time-zones)
 // and can be updated on demand in this repository
 // or by yourself using the above script
 
@@ -211,10 +212,10 @@
 #define TZ_America_Winnipeg	PSTR("CST6CDT,M3.2.0,M11.1.0")
 #define TZ_America_Yakutat	PSTR("AKST9AKDT,M3.2.0,M11.1.0")
 #define TZ_America_Yellowknife	PSTR("MST7MDT,M3.2.0,M11.1.0")
-#define TZ_Antarctica_Casey	PSTR("<+08>-8")
+#define TZ_Antarctica_Casey	PSTR("<+11>-11")
 #define TZ_Antarctica_Davis	PSTR("<+07>-7")
 #define TZ_Antarctica_DumontDUrville	PSTR("<+10>-10")
-#define TZ_Antarctica_Macquarie	PSTR("<+11>-11")
+#define TZ_Antarctica_Macquarie	PSTR("AEST-10AEDT,M10.1.0,M4.1.0/3")
 #define TZ_Antarctica_Mawson	PSTR("<+05>-5")
 #define TZ_Antarctica_McMurdo	PSTR("NZST-12NZDT,M9.5.0,M4.1.0/3")
 #define TZ_Antarctica_Palmer	PSTR("<-03>3")
@@ -248,8 +249,8 @@
 #define TZ_Asia_Dubai	PSTR("<+04>-4")
 #define TZ_Asia_Dushanbe	PSTR("<+05>-5")
 #define TZ_Asia_Famagusta	PSTR("EET-2EEST,M3.5.0/3,M10.5.0/4")
-#define TZ_Asia_Gaza	PSTR("EET-2EEST,M3.5.5/0,M10.5.6/1")
-#define TZ_Asia_Hebron	PSTR("EET-2EEST,M3.5.5/0,M10.5.6/1")
+#define TZ_Asia_Gaza	PSTR("EET-2EEST,M3.4.4/48,M10.4.4/49")
+#define TZ_Asia_Hebron	PSTR("EET-2EEST,M3.4.4/48,M10.4.4/49")
 #define TZ_Asia_Ho_Chi_Minh	PSTR("<+07>-7")
 #define TZ_Asia_Hong_Kong	PSTR("HKT-8")
 #define TZ_Asia_Hovd	PSTR("<+07>-7")

cores/esp8266/StackThunk.cpp

@@ -25,15 +25,18 @@
 #include "core_esp8266_waveform.h"
 #include "user_interface.h"
 
-// Which pins have a tone running on them?
-static uint32_t _toneMap = 0;
-
-
 static void _startTone(uint8_t _pin, uint32_t high, uint32_t low, uint32_t duration) {
   if (_pin > 16) {
     return;
   }
 
+#ifndef WAVEFORM_LOCKED_PHASE
+  // Stop any analogWrites (PWM) because they are a different generator
+  _stopPWM(_pin);
+#endif
+  // If there's another Tone or startWaveform on this pin
+  // it will be changed on-the-fly (no need to stop it)
+
   pinMode(_pin, OUTPUT);
 
   high = std::max(high, (uint32_t)microsecondsToClockCycles(25));  // new 20KHz maximum tone frequency,
@@ -42,9 +45,7 @@ static void _startTone(uint8_t _pin, uint32_t high, uint32_t low, uint32_t durat
   duration = microsecondsToClockCycles(duration * 1000UL);
   duration += high + low - 1;
   duration -= duration % (high + low);
-  if (startWaveformClockCycles(_pin, high, low, duration)) {
-    _toneMap |= 1 << _pin;
-  }
+  startWaveformClockCycles(_pin, high, low, duration);
 }
 
 
@@ -86,6 +87,5 @@ void noTone(uint8_t _pin) {
     return;
   }
   stopWaveform(_pin);
-  _toneMap &= ~(1 << _pin);
   digitalWrite(_pin, 0);
 }

cores/esp8266/TZ.h

@@ -203,6 +203,11 @@ bool UpdaterClass::end(bool evenIfRemaining){
     return false;
   }
 
+  // Updating w/o any data is an error we detect here
+  if (!progress()) {
+    _setError(UPDATE_ERROR_NO_DATA);
+  }
+
   if(hasError() || (!isFinished() && !evenIfRemaining)){
 #ifdef DEBUG_UPDATER
     DEBUG_UPDATER.printf_P(PSTR("premature end: res:%u, pos:%zu/%zu\n"), getError(), progress(), _size);
@@ -551,6 +556,8 @@ void UpdaterClass::printError(Print &out){
     out.println(F("Bad Size Given"));
   } else if(_error == UPDATE_ERROR_STREAM){
     out.println(F("Stream Read Timeout"));
+  } else if(_error == UPDATE_ERROR_NO_DATA){
+    out.println(F("No data supplied"));
   } else if(_error == UPDATE_ERROR_MD5){
     out.printf_P(PSTR("MD5 Failed: expected:%s, calculated:%s\n"), _target_md5.c_str(), _md5.toString().c_str());
   } else if(_error == UPDATE_ERROR_SIGN){