Support for STARTLS/STARTSSL in-band transport upgrades/renegotation (#9100)

* Split start_ssl_client into two phases; to allow the implementation of protocols that use some sort of in-band STARTTLS or STARTSSL signal to upgrade a plaint text connection to SSL/TLS. Examples of these protocols are XMPP, SMTP and various database TCP connections.

* Remove removed setTimeout that was accidentally included (was removed for IDF >=5), bring timeout inline with the other timeouts (ints), fix cert/key checks to look if there is actually something there (all issues caught by the CI/CD on windows-latest

* Quell compiler warning; use the right timeout

* Newer versions of MBEDTLS make the client key struct private (and most of the x509 struct too), so absent of a non-null pointer we cannot check wether it is populated. Solve this by looking at the version (as 0 is not a valid x509 version).

* Fix another \(rightfull\) compiler warning iwth the version pointer

* Quell CI/CD runs on non-WiFi supporting hardare

* Quell CI/CD runs on non-WiFi supporting hardare

* Fix typo in directory name

* Apply suggestions from code review

Co-authored-by: Jan Procházka <[email protected]>

* Rename Files

* Remove leftover file

---------

Co-authored-by: Me No Dev <[email protected]>
Co-authored-by: Jan Procházka <[email protected]>
Co-authored-by: Lucas Saavedra Vaz <[email protected]>

Author: 4 people

Diff for: libraries/WiFiClientSecure/examples/WiFiClientSecureProtocolUpgrade/.skip.esp32h2

@@ -0,0 +1,176 @@
+/* STARTSSL example
+
+   Inline upgrading from a clear-text connection to an SSL/TLS connection.
+
+   Some protocols such as SMTP, XMPP, Mysql, Postgress and others allow, or require,
+   that you start the connection without encryption; and then send a command to switch
+   over to encryption.
+
+   E.g. a typical SMTP submission would entail a dialogue such as this:
+
+   1. client connects to server in the clear
+   2. server says hello
+   3. client sents a EHLO
+   4. server tells the client that it supports SSL/TLS
+   5. client sends a 'STARTTLS' to make use of this faciltiy
+   6. client/server negiotiate a SSL or TLS connection.
+   7. client sends another EHLO
+   8. server now tells the client what (else) is supported; such as additional authentication options.
+   ... conversation continues encrypted.
+
+   This can be enabled in WiFiClientSecure by telling it to start in plaintext:
+
+          client.setPlainStart();
+
+   and client is than a plain, TCP, connection (just as WiFiClient would be); until the client calls
+   the method:
+
+          client.startTLS(); // returns zero on error; non zero on success.
+
+    After which things switch to TLS/SSL.
+*/
+
+#include <WiFiClientSecure.h>
+
+#ifndef WIFI_NETWORK
+#define WIFI_NETWORK "YOUR Wifi SSID"
+#endif
+
+#ifndef WIFI_PASSWD
+#define WIFI_PASSWD "your-secret-password"
+#endif
+
+#ifndef SMTP_HOST
+#define SMTP_HOST "smtp.gmail.com"
+#endif
+
+#ifndef SMTP_PORT
+#define SMTP_PORT (587) // Standard (plaintext) submission port
+#endif
+
+const char* ssid = WIFI_NETWORK;       // your network SSID (name of wifi network)
+const char* password = WIFI_PASSWD;    // your network password
+const char* server = SMTP_HOST;        // Server URL
+const int submission_port = SMTP_PORT; // submission port.
+
+WiFiClientSecure client;
+
+static bool readAllSMTPLines();
+
+void setup() {
+  int ret;
+  //Initialize serial and wait for port to open:
+  Serial.begin(115200);
+  delay(100);
+
+  Serial.print("Attempting to connect to SSID: ");
+  Serial.print(ssid);
+  WiFi.begin(ssid, password);
+
+  // attempt to connect to Wifi network:
+  while (WiFi.status() != WL_CONNECTED) {
+    Serial.print(".");
+    // wait 1 second for re-trying
+    delay(1000);
+  }
+
+  Serial.print("Connected to ");
+  Serial.println(ssid);
+
+  Serial.printf("\nStarting connection to server: %s:%d\n", server, submission_port);
+
+
+  // skip verification for this demo. In production one should at the very least
+  // enable TOFU; or ideally hardcode a (CA) certificate that is trusted.
+  client.setInsecure();
+
+  // Enable a plain-test start.
+  client.setPlainStart();
+
+  if (!client.connect(server, SMTP_PORT)) {
+    Serial.println("Connection failed!");
+    return;
+  };
+
+  Serial.println("Connected to server (in the clear, in plaintest)");
+
+  if (!readAllSMTPLines()) goto err;
+
+  Serial.println("Sending  : EHLO\t\tin the clear");
+  client.print("EHLO there\r\n");
+
+  if (!readAllSMTPLines()) goto err;
+
+  Serial.println("Sending  : STARTTLS\t\tin the clear");
+  client.print("STARTTLS\r\n");
+
+  if (!readAllSMTPLines()) goto err;
+
+  Serial.println("Upgrading connection to TLS");
+  if ((ret=client.startTLS()) <= 0) {
+    Serial.printf("Upgrade connection failed: err %d\n", ret);
+    goto err;
+  }
+
+  Serial.println("Sending  : EHLO again\t\tover the now encrypted connection");
+  client.print("EHLO again\r\n");
+
+  if (!readAllSMTPLines()) goto err;
+
+  // normally, as this point - we'd be authenticating and then be submitting
+  // an email. This has been left out of this example.
+  
+  Serial.println("Sending  : QUIT\t\t\tover the now encrypted connection");
+  client.print("QUIT\r\n");
+
+  if (!readAllSMTPLines()) goto err;
+
+  Serial.println("Completed OK\n");
+err:
+  Serial.println("Closing connection");
+  client.stop();
+}
+
+// SMTP command repsponse start with three digits and a space;
+// or, for continuation, with three digits and a '-'.
+static bool readAllSMTPLines() {
+  String s = "";
+  int i;
+
+  // blocking read; we cannot rely on a timeout
+  // of a WiFiClientSecure read; as it is non
+  // blocking.
+  const unsigned long timeout = 15 * 1000;
+  unsigned long start = millis(); // the timeout is for the entire CMD block response; not per character/line.
+  while (1) {
+    while ((i = client.available()) == 0 && millis() - start < timeout) {
+      /* .. wait */
+    };
+    if (i == 0) {
+      Serial.println("Timeout reading SMTP response");
+      return false;
+    };
+    if (i < 0)
+      break;
+
+    i = client.read();
+    if (i < 0)
+      break;
+
+    if (i > 31 && i < 128) s += (char)i;
+    if (i == 0x0A) {
+      Serial.print("Receiving: ");
+      Serial.println(s);
+      if (s.charAt(3) == ' ')
+        return true;
+      s = "";
+    }
+  }
+  Serial.printf("Error reading SMTP command response line: %d\n", i);
+  return false;
+}
+
+void loop() {
+  // do nothing
+}
+

Diff for: libraries/WiFiClientSecure/examples/WiFiClientSecureProtocolUpgrade/WiFiClientSecureProtocolUpgrade.ino

@@ -140,16 +140,40 @@ int WiFiClientSecure::connect(const char *host, uint16_t port, const char *CA_ce
 int WiFiClientSecure::connect(IPAddress ip, uint16_t port, const char *host, const char *CA_cert, const char *cert, const char *private_key)
 {
     int ret = start_ssl_client(sslclient, ip, port, host, _timeout, CA_cert, _use_ca_bundle, cert, private_key, NULL, NULL, _use_insecure, _alpn_protos);
+
+    if (ret >=0 && ! _stillinPlainStart)
+        ret = ssl_starttls_handshake(sslclient);
+    else
+        log_i("Actual TLS start posponed.");
+
     _lastError = ret;
+
     if (ret < 0) {
-        log_e("start_ssl_client: %d", ret);
+        log_e("start_ssl_client: connect failed: %d", ret);
         stop();
         return 0;
     }
     _connected = true;
     return 1;
 }
 
+int WiFiClientSecure::startTLS()
+{
+    int ret = 1;
+    if (_stillinPlainStart) {
+        log_i("startTLS: starting TLS/SSL on this dplain connection");
+        ret = ssl_starttls_handshake(sslclient);
+        if (ret < 0) {
+                log_e("startTLS: %d", ret);
+                stop();
+                return 0;
+        };
+        _stillinPlainStart = false;
+    } else 
+        log_i("startTLS: ignoring StartTLS - as we should be secure already");
+    return 1;
+}
+
 int WiFiClientSecure::connect(IPAddress ip, uint16_t port, const char *pskIdent, const char *psKey) {
     return connect(ip.toString().c_str(), port, pskIdent, psKey);
 }
@@ -164,7 +188,7 @@ int WiFiClientSecure::connect(const char *host, uint16_t port, const char *pskId
     int ret = start_ssl_client(sslclient, address, port, host, _timeout, NULL, false, NULL, NULL, pskIdent, psKey, _use_insecure, _alpn_protos);
     _lastError = ret;
     if (ret < 0) {
-        log_e("start_ssl_client: %d", ret);
+        log_e("start_ssl_client: connect failed %d", ret);
         stop();
         return 0;
     }
@@ -189,17 +213,18 @@ int WiFiClientSecure::read()
 {
     uint8_t data = -1;
     int res = read(&data, 1);
-    if (res < 0) {
-        return res;
-    }
-    return data;
+    return res < 0 ? res: data;
 }
 
 size_t WiFiClientSecure::write(const uint8_t *buf, size_t size)
 {
     if (!_connected) {
         return 0;
     }
+
+    if (_stillinPlainStart) 
+        return send_net_data(sslclient, buf, size);
+
     if(_lastWriteTimeout != _timeout){
         struct timeval timeout_tv;
         timeout_tv.tv_sec = _timeout / 1000;
@@ -209,9 +234,9 @@ size_t WiFiClientSecure::write(const uint8_t *buf, size_t size)
             _lastWriteTimeout = _timeout;
         }
     }
-
     int res = send_ssl_data(sslclient, buf, size);
     if (res < 0) {
+        log_e("Closing connection on failed write");
         stop();
         res = 0;
     }
@@ -220,6 +245,9 @@ size_t WiFiClientSecure::write(const uint8_t *buf, size_t size)
 
 int WiFiClientSecure::read(uint8_t *buf, size_t size)
 {
+    if(_stillinPlainStart) 
+        return  get_net_receive(sslclient, buf, size);
+
     if(_lastReadTimeout != _timeout){
         if(fd() >= 0){
             struct timeval timeout_tv;
@@ -232,7 +260,7 @@ int WiFiClientSecure::read(uint8_t *buf, size_t size)
         }
     }
 
-    int peeked = 0;
+    int peeked = 0, res = -1;
     int avail = available();
     if ((!buf && size) || avail <= 0) {
         return -1;
@@ -251,9 +279,10 @@ int WiFiClientSecure::read(uint8_t *buf, size_t size)
         buf++;
         peeked = 1;
     }
-    
-    int res = get_ssl_receive(sslclient, buf, size);
+        res = get_ssl_receive(sslclient, buf, size);
+
     if (res < 0) {
+        log_e("Closing connection on failed read");
         stop();
         return peeked?peeked:res;
     }
@@ -262,12 +291,17 @@ int WiFiClientSecure::read(uint8_t *buf, size_t size)
 
 int WiFiClientSecure::available()
 {
-    int peeked = (_peek >= 0);
+    if (_stillinPlainStart) 
+        return peek_net_receive(sslclient,0);
+
+    int peeked = (_peek >= 0), res = -1;
     if (!_connected) {
         return peeked;
     }
-    int res = data_to_read(sslclient);
-    if (res < 0) {
+    res = data_to_read(sslclient);
+
+    if (res < 0 && !_stillinPlainStart) {
+        log_e("Closing connection on failed available check");
         stop();
         return peeked?peeked:res;
     }
@@ -403,3 +437,4 @@ int WiFiClientSecure::fd() const
 {
     return sslclient->socket;
 }
+

Diff for: libraries/WiFiClientSecure/src/WiFiClientSecure.cpp

@@ -34,6 +34,7 @@ class WiFiClientSecure : public WiFiClient
     int _peek = -1;
     int _timeout;
     bool _use_insecure;
+    bool _stillinPlainStart = false;
     const char *_CA_cert;
     const char *_cert;
     const char *_private_key;
@@ -78,6 +79,17 @@ class WiFiClientSecure : public WiFiClient
     bool verify(const char* fingerprint, const char* domain_name);
     void setHandshakeTimeout(unsigned long handshake_timeout);
     void setAlpnProtocols(const char **alpn_protos);
+
+    // Certain protocols start in plain-text; and then have the client
+    // give some STARTSSL command to `upgrade' the connection to TLS
+    // or SSL. Setting PlainStart to true (the default is false) enables
+    // this. It is up to the application code to then call 'startTLS()'
+    // at the right point to initialise the SSL or TLS upgrade.
+    
+    void setPlainStart() { _stillinPlainStart = true; };
+    bool stillInPlainStart() { return _stillinPlainStart; };
+    int startTLS();
+
     const mbedtls_x509_crt* getPeerCertificate() { return mbedtls_ssl_get_peer_cert(&sslclient->ssl_ctx); };
     bool getFingerprintSHA256(uint8_t sha256_result[32]) { return get_peer_fingerprint(sslclient, sha256_result); };
     int fd() const;