Fix device test environment variables

tests/device/Makefile

@@ -23,6 +23,7 @@ TEST_REPORT_HTML := test_report.html
 
 ifneq ("$(V)","1")
 	SILENT = @
+	REDIR = >& /dev/null
 else
 	BUILDER_DEBUG_FLAG = -verbose
 	RUNNER_DEBUG_FLAG = -d
@@ -50,10 +51,11 @@ $(TEST_LIST):
 ifneq ("$(NO_BUILD)","1")
 	@test -n "$(ARDUINO_IDE_PATH)" || (echo "Please export ARDUINO_IDE_PATH" && exit 1)
 	@echo Compiling $(notdir $@)
+	@rm -f $(LOCAL_BUILD_DIR)/build.options.json
 	$(SILENT)$(BUILD_TOOL) -compile -logger=human \
 		-libraries "$(PWD)/libraries" \
 		-core-api-version="10608" \
-		-warnings=none \
+		-warnings=all \
 		$(BUILDER_DEBUG_FLAG) \
 		-build-path $(LOCAL_BUILD_DIR) \
 		-tools $(ARDUINO_IDE_PATH)/tools-builder \
@@ -64,24 +66,25 @@ ifneq ("$(NO_BUILD)","1")
 endif
 ifneq ("$(NO_UPLOAD)","1")
 	@test -n "$(UPLOAD_PORT)" || (echo "Failed to detect upload port, please export UPLOAD_PORT manually" && exit 1)
-	@test -e $(dir $@)/make_spiffs.py && (echo "Generating and uploading SPIFFS" && \
-		(cd $(dir $@) && $(PYTHON) ./make_spiffs.py) && \
+	@test -e $(dir $@)/make_spiffs.py && ( \
+		echo "Generating and uploading SPIFFS" && \
+		(cd $(dir $@) && $(PYTHON) ./make_spiffs.py $(REDIR) ) && \
 		$(MKSPIFFS) --create $(dir $@)data/ --size 0xFB000 \
-			--block 8192 --page 256 $(LOCAL_BUILD_DIR)/spiffs.img && \
+			--block 8192 --page 256 $(LOCAL_BUILD_DIR)/spiffs.img $(REDIR) && \
 		$(ESPTOOL) $(UPLOAD_VERBOSE_FLAG) \
 			--chip esp8266 \
 			--port $(UPLOAD_PORT) \
 			--baud $(UPLOAD_BAUD) \
 			--after no_reset \
-			write_flash 0x300000 $(LOCAL_BUILD_DIR)/spiffs.img ) \
+			write_flash 0x300000 $(LOCAL_BUILD_DIR)/spiffs.img $(REDIR) ) \
 			|| (echo "No SPIFFS to upload")
 	@echo Uploading binary
 	$(SILENT)$(ESPTOOL) $(UPLOAD_VERBOSE_FLAG)  \
 		--chip esp8266 \
 		--port $(UPLOAD_PORT) \
 		--baud $(UPLOAD_BAUD) \
 		--after no_reset \
-		write_flash 0x0 $(LOCAL_BUILD_DIR)/$(notdir $@).bin # no reset
+		write_flash 0x0 $(LOCAL_BUILD_DIR)/$(notdir $@).bin $(REDIR) # no reset
 endif
 ifneq ("$(NO_RUN)","1")
 	@test -n "$(UPLOAD_PORT)" || (echo "Failed to detect upload port, please export UPLOAD_PORT manually" && exit 1)
@@ -90,7 +93,7 @@ ifneq ("$(NO_RUN)","1")
 		--chip esp8266 \
 		--port $(UPLOAD_PORT) \
 		--baud $(UPLOAD_BAUD) \
-		read_flash_status # reset
+		read_flash_status $(REDIR) # reset
 	$(SILENT)source $(BS_DIR)/virtualenv/bin/activate && \
 		$(PYTHON) $(BS_DIR)/runner.py \
 			$(RUNNER_DEBUG_FLAG) \

tests/device/libraries/BSTest/runner.py

@@ -104,6 +104,10 @@ def run_tests(self):
                     if res != BSTestRunner.SUCCESS:
                         print('failed to set environment variables')
                         break;
+                    res = self.pretest()
+                    if res != BSTestRunner.SUCCESS:
+                        print('failed to run pretest init')
+                        break;
                     should_update_env = False
                 if name in self.mocks:
                     debug_print('setting up mocks')
@@ -198,6 +202,21 @@ def update_env(self, env_to_set):
                 return BSTestRunner.TIMEOUT
         return BSTestRunner.SUCCESS
 
+    def pretest(self):
+        # Environment now set up, call the pretest init (wifi connect, etc.)
+        self.sp.sendline('pretest');
+        timeout = 10
+        while timeout > 0:
+            res = self.sp.expect(['>>>>>bs_test_pretest result=1', EOF, TIMEOUT]) # Only expect a pass, abort testing if failure
+            if res == 0:
+                break
+            time.sleep(0.1)
+            timeout -= 0.1
+        if res != 0:
+            return BSTestRunner.TIMEOUT
+        else:
+            return BSTestRunner.SUCCESS
+
     def request_env(self, key):
         self.sp.sendline('getenv "{}"'.format(key))
         timeout = 10

tests/device/libraries/BSTest/src/BSArgs.h

@@ -69,7 +69,7 @@ inline size_t split_args(char *line, char **argv, size_t argv_size)
     const int ESCAPE = '\\';
     const int SPACE = ' ';
     split_state_t state = SS_SPACE;
-    int argc = 0;
+    size_t argc = 0;
     char *next_arg_start = line;
     char *out_ptr = line;
     for (char *in_ptr = line; argc < argv_size - 1; ++in_ptr) {
@@ -148,4 +148,4 @@ inline size_t split_args(char *line, char **argv, size_t argv_size)
 
 } // namespace protocol
 
-#endif //BS_ARGS_H
+#endif //BS_ARGS_H

tests/device/libraries/BSTest/src/BSProtocol.h

@@ -5,6 +5,8 @@
 
 #define BS_LINE_PREFIX ">>>>>bs_test_"
 
+extern bool pretest();
+
 namespace bs
 {
 namespace protocol
@@ -58,6 +60,12 @@ void output_getenv_result(IO& io, const char* key, const char* value)
     io.printf(BS_LINE_PREFIX "getenv value=\"%s\"\n", value);
 }
 
+template<typename IO>
+void output_pretest_result(IO& io, bool res)
+{
+    io.printf(BS_LINE_PREFIX "pretest result=%d\n", res?1:0);
+}
+
 template<typename IO>
 bool input_handle(IO& io, char* line_buf, size_t line_buf_size, int& test_num)
 {
@@ -87,6 +95,14 @@ bool input_handle(IO& io, char* line_buf, size_t line_buf_size, int& test_num)
         output_getenv_result(io, argv[1], (value != NULL) ? value : "");
         return false;
     }
+    if (strcmp(argv[0], "pretest") == 0) {
+        if (argc != 1) {
+            return false;
+        }
+        bool res = ::pretest();
+        output_pretest_result(io, res);
+        return false;
+    }
     /* not one of the commands, try to parse as test number */
     char* endptr;
     test_num = (int) strtol(argv[0], &endptr, 10);

tests/device/test_BearSSL/test_BearSSL.ino

@@ -33,6 +33,11 @@ void setup()
 {
     Serial.begin(115200);
     Serial.setDebugOutput(true);
+    BS_RUN(Serial);
+}
+
+bool pretest()
+{
     WiFi.persistent(false);
     WiFi.mode(WIFI_STA);
     WiFi.begin(getenv("STA_SSID"), getenv("STA_PASS"));
@@ -45,22 +50,11 @@ void setup()
     Serial.printf("Number of CA certs read: %d\n", numCerts);
     if (numCerts == 0) {
         Serial.printf("No certs found. Did you run certs-from-mozill.py and upload the SPIFFS directory before running?\n");
-        REQUIRE(1==0);
+        return false;
     }
-    BS_RUN(Serial);
-}
-
-static void stopAll()
-{
-    WiFiClient::stopAll();
-}
-
-static void disconnectWiFI()
-{
-    wifi_station_disconnect();
+    return true;
 }
 
-
 // Set time via NTP, as required for x.509 validation
 void setClock() {
   configTime(3 * 3600, 0, "pool.ntp.org", "time.nist.gov");
@@ -143,7 +137,7 @@ int run(const char *str)
     return maxUsage;
 }
 
-#define TC(x) TEST_CASE("BearSSL - Maximum stack usage < 5600 bytes @ "x".badssl.org", "[bearssl]") { REQUIRE(run(x) < 5600); }
+#define TC(x) TEST_CASE("BearSSL - Maximum stack usage < 5600 bytes @ " x ".badssl.org", "[bearssl]") { REQUIRE(run(x) < 5600); }
 
 TC("expired")
 TC("wrong.host")

tests/device/test_ClientContext/test_ClientContext.ino

@@ -23,13 +23,18 @@ void setup()
 {
     Serial.begin(115200);
     Serial.setDebugOutput(true);
+    BS_RUN(Serial);
+}
+
+bool pretest()
+{
     WiFi.persistent(false);
     WiFi.mode(WIFI_STA);
     WiFi.begin(getenv("STA_SSID"), getenv("STA_PASS"));
     while (WiFi.status() != WL_CONNECTED) {
         delay(500);
     }
-    BS_RUN(Serial);
+    return true;
 }
 
 TEST_CASE("WiFi release ClientContext", "[clientcontext]")

tests/device/test_FS/test_FS.ino

@@ -10,6 +10,10 @@ void setup()
     BS_RUN(Serial);
 }
 
+bool pretest()
+{
+    return true;
+}
 
 
 TEST_CASE("read-write test","[fs]")

tests/device/test_Print_printf/test_Print_printf.ino

@@ -9,13 +9,18 @@ void setup()
     BS_RUN(Serial);
 }
 
+bool pretest()
+{
+    return true;
+}
+
 TEST_CASE("Print::printf works for any reasonable output length", "[Print]")
 {
 
     auto test_printf = [](size_t size) {
         StreamString str;
         auto buf = new char[size + 1];
-        for (int i = 0; i < size; ++i) {
+        for (size_t i = 0; i < size; ++i) {
             buf[i] = 'a';
         }
         buf[size] = 0;

tests/device/test_WiFiClient/test_WiFiClient.ino

@@ -13,13 +13,18 @@ void setup()
 {
     Serial.begin(115200);
     Serial.setDebugOutput(true);
+    BS_RUN(Serial);
+}
+
+bool pretest()
+{
     WiFi.persistent(false);
     WiFi.mode(WIFI_STA);
     WiFi.begin(getenv("STA_SSID"), getenv("STA_PASS"));
     while (WiFi.status() != WL_CONNECTED) {
         delay(500);
     }
-    BS_RUN(Serial);
+    return true;
 }
 
 static void stopAll()
@@ -53,4 +58,4 @@ TEST_CASE("WiFi disconnect during WiFiClient::connect", "[wificlient]")
 
 void loop()
 {
-}
+}

tests/device/test_WiFiServer/test_WiFiServer.ino

@@ -10,13 +10,18 @@ BS_ENV_DECLARE();
 void setup()
 {
     Serial.begin(115200);
+    BS_RUN(Serial);
+}
+
+bool pretest()
+{
     WiFi.persistent(false);
     WiFi.begin(getenv("STA_SSID"), getenv("STA_PASS"));
     while (WiFi.status() != WL_CONNECTED) {
         delay(500);
     }
     MDNS.begin("esp8266-wfs-test");
-    BS_RUN(Serial);
+    return true;
 }
 
 TEST_CASE("Simple echo server", "[WiFiServer]")

tests/device/test_WiFi_events/test_WiFi_events.ino

@@ -11,9 +11,14 @@ void setup()
 {
     Serial.begin(115200);
     Serial.setDebugOutput(false);
+    BS_RUN(Serial);
+}
+
+bool pretest()
+{
     WiFi.persistent(false);
     WiFi.mode(WIFI_OFF);
-    BS_RUN(Serial);
+    return true;
 }
 
 static std::map<WiFiEvent_t, int> sEventsReceived;
@@ -29,10 +34,13 @@ TEST_CASE("WiFi.onEvent is called for specific events", "[wifi][events]")
     sEventsReceived[WIFI_EVENT_STAMODE_DISCONNECTED] = 0;
     sEventsReceived[WIFI_EVENT_STAMODE_GOT_IP] = 0;
 
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
     WiFi.onEvent(onWiFiEvent, WIFI_EVENT_STAMODE_CONNECTED);
     WiFi.onEvent(onWiFiEvent, WIFI_EVENT_STAMODE_DISCONNECTED);
     WiFi.onEvent(onWiFiEvent, WIFI_EVENT_STAMODE_GOT_IP);
     WiFi.onEvent(onWiFiEvent, WIFI_EVENT_ANY);
+#pragma GCC diagnostic pop
 
     WiFi.mode(WIFI_STA);
     WiFi.begin(getenv("STA_SSID"), getenv("STA_PASS"));
@@ -44,24 +52,30 @@ TEST_CASE("WiFi.onEvent is called for specific events", "[wifi][events]")
     WiFi.disconnect();
     delay(100);
     WiFi.mode(WIFI_OFF);
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
     REQUIRE(sEventsReceived[WIFI_EVENT_STAMODE_CONNECTED] == 2);
     REQUIRE(sEventsReceived[WIFI_EVENT_STAMODE_DISCONNECTED] >= 2 && sEventsReceived[WIFI_EVENT_STAMODE_DISCONNECTED] % 2 == 0);
     REQUIRE(sEventsReceived[WIFI_EVENT_STAMODE_GOT_IP] == 2);
+#pragma GCC diagnostic pop
 }
 
 TEST_CASE("STA mode events are called both when using DHCP and static config", "[wifi][events]")
 {
     String events;
 
     auto handler1 = WiFi.onStationModeConnected([&](const WiFiEventStationModeConnected& evt){
+        (void) evt;
         events += "connected,";
     });  
     auto handler2 = WiFi.onStationModeDisconnected([&](const WiFiEventStationModeDisconnected& evt){
+        (void) evt;
         if (events.length()) {
             events += "disconnected,";
         }
     });
     auto handler3 = WiFi.onStationModeGotIP([&](const WiFiEventStationModeGotIP& evt){
+        (void) evt;
         events += "got_ip,";
     });
 
@@ -104,12 +118,15 @@ TEST_CASE("Events are not called if handler is deleted", "[wifi][events]")
     String events;
 
     WiFi.onStationModeConnected([&](const WiFiEventStationModeConnected& evt){
+        (void) evt;
         events += "connected,";
     });  
     WiFi.onStationModeDisconnected([&](const WiFiEventStationModeDisconnected& evt){
+        (void) evt;
         events += "disconnected,";
     });
     WiFi.onStationModeGotIP([&](const WiFiEventStationModeGotIP& evt){
+        (void) evt;
         events += "got_ip,";
     });