Merge pull request #66 from arduino/m2crypto_fix

ussl: Clean up SSL and m2crypto code.

Author: iabdalkader

.github/workflows/client-test.yml

@@ -76,7 +76,15 @@ jobs:
         DEVICE_ID: ${{ secrets.DEVICE_ID1 }}
         SECRET_KEY: ${{ secrets.SECRET_KEY }}
       run: |
-        python tests/ci.py
+        python tests/ci.py --basic-auth
+
+    - name: '☁️  Connect to IoT cloud (CPython / Key/Cert Auth)'
+      env:
+        DEVICE_ID: ${{ secrets.DEVICE_ID2 }}
+        SECRET_KEY: ${{ secrets.SECRET_KEY }}
+      run: |
+        python tests/ci.py --file-auth
+
 
     - name: '☁️  Connect to IoT cloud (CPython / Crypto Auth)'
       env:
@@ -93,4 +101,4 @@ jobs:
       run: |
         export PATH="${HOME}/cache/bin:${PATH}"
         micropython -c "import sys; print(sys.path)"
-        micropython tests/ci.py
+        micropython tests/ci.py --basic-auth

examples/example.py

@@ -11,7 +11,7 @@
 from arduino_iot_cloud import Task
 from random import uniform
 import argparse
-import arduino_iot_cloud.ussl as ssl
+import ssl
 
 from secrets import DEVICE_ID
 from secrets import SECRET_KEY  # noqa

examples/micropython.py

@@ -2,7 +2,7 @@
 # Any copyright is dedicated to the Public Domain.
 # https://creativecommons.org/publicdomain/zero/1.0/
 import time
-import ussl
+import ssl
 import network
 import logging
 from time import strftime
@@ -73,7 +73,7 @@ def wifi_connect():
     client = ArduinoCloudClient(
         device_id=DEVICE_ID,
         ssl_params={
-            "keyfile": KEY_PATH, "certfile": CERT_PATH, "cadata": CADATA, "cert_reqs": ussl.CERT_REQUIRED
+            "keyfile": KEY_PATH, "certfile": CERT_PATH, "cadata": CADATA, "cert_reqs": ssl.CERT_REQUIRED
         }
     )
 

pyproject.toml

@@ -22,7 +22,6 @@ classifiers = [
 ]
 dependencies = [
   'cbor2 >= 5.4.6',
-  'M2Crypto >= 0.38.0',
   'micropython-senml >= 0.1.0',
 ]
 

src/arduino_iot_cloud/ucloud.py

@@ -5,6 +5,7 @@
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
 import time
+import sys
 import logging
 from senml import SenmlPack
 from senml import SenmlRecord
@@ -175,6 +176,14 @@ def __init__(
         self.senmlpack = SenmlPack("", self.senml_generic_callback)
         self.started = False
 
+        if "pin" in ssl_params:
+            try:
+                # Use M2Crypto to load key and cert from HSM.
+                import M2Crypto  # noqa
+            except (ImportError, AttributeError):
+                logging.error("The m2crypto module is required to use HSM.")
+                sys.exit(1)
+
         # Convert args to bytes if they are passed as strings.
         if isinstance(device_id, str):
             device_id = bytes(device_id, "utf-8")
@@ -188,19 +197,6 @@ def __init__(
         # Update RTC from NTP server on MicroPython.
         self.update_systime(ntp_server, ntp_timeout)
 
-        # MicroPython does not support secure elements yet, and key/cert
-        # must be loaded from DER files and passed as binary blobs.
-        if "keyfile" in ssl_params and "der" in ssl_params["keyfile"]:
-            with open(ssl_params.pop("keyfile"), "rb") as f:
-                ssl_params["key"] = f.read()
-        if "certfile" in ssl_params and "der" in ssl_params["certfile"]:
-            with open(ssl_params.pop("certfile"), "rb") as f:
-                ssl_params["cert"] = f.read()
-
-        if "ca_certs" in ssl_params and "der" in ssl_params["ca_certs"]:
-            with open(ssl_params.pop("ca_certs"), "rb") as f:
-                ssl_params["cadata"] = f.read()
-
         # If no server/port were passed in args, set the default server/port
         # based on authentication type.
         if server is None:

src/arduino_iot_cloud/umqtt.py

@@ -26,11 +26,7 @@
 import struct
 import select
 import logging
-
-try:
-    from ussl import wrap_socket
-except ImportError:
-    from arduino_iot_cloud.ussl import wrap_socket
+import arduino_iot_cloud.ussl as ssl
 
 
 class MQTTException(Exception):
@@ -99,14 +95,14 @@ def connect(self, clean_session=True, timeout=5.0):
         try:
             self.sock = socket.socket()
             self.sock.settimeout(timeout)
-            self.sock = wrap_socket(self.sock, **self.ssl_params)
+            self.sock = ssl.wrap_socket(self.sock, self.ssl_params)
             self.sock.connect(addr)
         except Exception:
             self.sock.close()
             self.sock = socket.socket()
             self.sock.settimeout(timeout)
             self.sock.connect(addr)
-            self.sock = wrap_socket(self.sock, **self.ssl_params)
+            self.sock = ssl.wrap_socket(self.sock, self.ssl_params)
 
         premsg = bytearray(b"\x10\0\0\0\0\0")
         msg = bytearray(b"\x04MQTT\x04\x02\0\0")

src/arduino_iot_cloud/ussl.py

@@ -6,64 +6,72 @@
 #
 # SSL module with m2crypto backend for HSM support.
 
-from M2Crypto import Engine, m2, SSL
+import sys
+import ssl
 
-CERT_NONE = SSL.verify_none
-CERT_REQUIRED = SSL.verify_peer
-
-_key = None
-_cert = None
+pkcs11 = None
 
 # Default engine and provider.
 _ENGINE_PATH = "/usr/lib/engines-3/libpkcs11.so"
 _MODULE_PATH = "/usr/lib/softhsm/libsofthsm2.so"
 
 
-def init(pin, certfile, keyfile, engine_path, module_path):
-    global _key, _cert
-    Engine.load_dynamic_engine("pkcs11", engine_path)
-    pkcs11 = Engine.Engine("pkcs11")
-    pkcs11.ctrl_cmd_string("MODULE_PATH", module_path)
-    pkcs11.ctrl_cmd_string("PIN", pin)
-    pkcs11.init()
-    _key = pkcs11.load_private_key(keyfile)
-    _cert = pkcs11.load_certificate(certfile)
-
-
 def wrap_socket(
-    sock_in,
-    pin=None,
-    certfile=None,
-    keyfile=None,
-    ca_certs=None,
-    cert_reqs=CERT_NONE,
-    ciphers=None,
-    engine_path=_ENGINE_PATH,
-    module_path=_MODULE_PATH,
+    sock,
+    ssl_params={},
 ):
-    if certfile is None or keyfile is None:
-        # Fallback to Python's SSL
-        import ssl
-        return ssl.wrap_socket(sock_in)
+    if any(k not in ssl_params for k in ("keyfile", "certfile", "pin")):
+        # Use Micro/CPython's SSL
+        if sys.implementation.name == "micropython":
+            # Load key, cert and CA from DER files, and pass them as binary blobs.
+            mpargs = {"keyfile": "key", "certfile": "cert", "ca_certs": "cadata"}
+            for k, v in mpargs.items():
+                if k in ssl_params and "der" in ssl_params[k]:
+                    with open(ssl_params.pop(k), "rb") as f:
+                        ssl_params[v] = f.read()
+        return ssl.wrap_socket(sock, **ssl_params)
+
+    # Use M2Crypto to load key and cert from HSM.
+    from M2Crypto import m2, SSL, Engine
 
-    if _key is None or _cert is None:
-        init(pin, certfile, keyfile, engine_path, module_path)
+    global pkcs11
+    if pkcs11 is None:
+        pkcs11 = Engine.load_dynamic_engine(
+            "pkcs11", ssl_params.get("engine_path", _ENGINE_PATH)
+        )
+        pkcs11.ctrl_cmd_string(
+            "MODULE_PATH", ssl_params.get("module_path", _MODULE_PATH)
+        )
+        pkcs11.ctrl_cmd_string("PIN", ssl_params["pin"])
+        pkcs11.init()
 
-    # Create SSL context
+    # Create and configure SSL context
     ctx = SSL.Context("tls")
     ctx.set_default_verify_paths()
     ctx.set_allow_unknown_ca(False)
 
+    ciphers = ssl_params.get("ciphers", None)
     if ciphers is not None:
         ctx.set_cipher_list(ciphers)
 
-    if ca_certs is not None and cert_reqs is not CERT_NONE:
+    ca_certs = ssl_params.get("ca_certs", None)
+    if ca_certs is not None:
         if ctx.load_verify_locations(ca_certs) != 1:
             raise Exception("Failed to load CA certs")
-        ctx.set_verify(SSL.verify_peer, depth=9)
+
+    cert_reqs = ssl_params.get("cert_reqs", ssl.CERT_NONE)
+    if cert_reqs == ssl.CERT_NONE:
+        cert_reqs = SSL.verify_none
+    else:
+        cert_reqs = SSL.verify_peer
+    ctx.set_verify(cert_reqs, depth=9)
 
     # Set key/cert
-    m2.ssl_ctx_use_x509(ctx.ctx, _cert.x509)
-    m2.ssl_ctx_use_pkey_privkey(ctx.ctx, _key.pkey)
+    key = pkcs11.load_private_key(ssl_params["keyfile"])
+    m2.ssl_ctx_use_pkey_privkey(ctx.ctx, key.pkey)
+
+    cert = pkcs11.load_certificate(ssl_params["certfile"])
+    m2.ssl_ctx_use_x509(ctx.ctx, cert.x509)
+
     SSL.Connection.postConnectionCheck = None
-    return SSL.Connection(ctx, sock=sock_in)
+    return SSL.Connection(ctx, sock=sock)

tests/ci.py

@@ -27,7 +27,13 @@ def on_value_changed(client, value):
         "-d", "--debug", action="store_true", help="Enable debugging messages"
     )
     parser.add_argument(
-        "-c", "--crypto-device", action="store_true", help="Use crypto device"
+        "-b", "--basic-auth", action="store_true", help="Username and password auth",
+    )
+    parser.add_argument(
+        "-c", "--crypto-device", action="store_true", help="Use soft-hsm/crypto device",
+    )
+    parser.add_argument(
+        "-f", "--file-auth", action="store_true", help="Use key/cert files"
     )
     args = parser.parse_args()
 
@@ -45,8 +51,25 @@ def on_value_changed(client, value):
     # the CA certificate (if any) in "ssl_params". Alternatively, a username and password can
     # be used to authenticate, for example:
     #   client = ArduinoCloudClient(device_id=DEVICE_ID, username=DEVICE_ID, password=SECRET_KEY)
-    if args.crypto_device:
-        import arduino_iot_cloud.ussl as ssl
+    if args.basic_auth:
+        client = ArduinoCloudClient(
+            device_id=os.getenv("DEVICE_ID"),
+            username=os.getenv("DEVICE_ID"),
+            password=os.getenv("SECRET_KEY"),
+        )
+    elif args.file_auth:
+        import ssl
+        client = ArduinoCloudClient(
+            device_id=os.getenv("DEVICE_ID"),
+            ssl_params={
+                "keyfile": "key.pem",
+                "certfile": "cert.pem",
+                "ca_certs": "ca-root.pem",
+                "cert_reqs": ssl.CERT_REQUIRED,
+            },
+        )
+    elif args.crypto_device:
+        import ssl
         client = ArduinoCloudClient(
             device_id=os.getenv("DEVICE_ID"),
             ssl_params={
@@ -60,11 +83,8 @@ def on_value_changed(client, value):
             },
         )
     else:
-        client = ArduinoCloudClient(
-            device_id=os.getenv("DEVICE_ID"),
-            username=os.getenv("DEVICE_ID"),
-            password=os.getenv("SECRET_KEY"),
-        )
+        parser.print_help()
+        sys.exit(1)
 
     # Register cloud objects.
     # Note: The following objects must be created first in the dashboard and linked to the device.

tests/ci.sh

@@ -9,17 +9,19 @@ ci_install_micropython() {
 	git clone --depth=1 https://github.com/micropython/micropython.git
 
 	cat > micropython/ports/unix/manifest.py <<-EOF
-	include("\$(PORT_DIR)/variants/manifest.py")
-	include("\$(MPY_DIR)/extmod/asyncio")
+	include("\$(PORT_DIR)/variants/standard/manifest.py")
 	require("bundle-networking")
 	require("time")
 	require("senml")
 	require("logging")
 	EOF
 
-	make -C micropython/mpy-cross/
-	make -C micropython/ports/unix/ submodules
-	make -C micropython/ports/unix/ FROZEN_MANIFEST=manifest.py CFLAGS_EXTRA="-DMICROPY_PY_SELECT=1"
+    echo "#undef MICROPY_PY_SELECT_SELECT" >> micropython/ports/unix/variants/mpconfigvariant_common.h
+    echo "#undef MICROPY_PY_SELECT_POSIX_OPTIMISATIONS" >> micropython/ports/unix/variants/mpconfigvariant_common.h
+
+	make -j12 -C micropython/mpy-cross/
+	make -j12 -C micropython/ports/unix/ submodules
+	make -j12 -C micropython/ports/unix/ FROZEN_MANIFEST=manifest.py CFLAGS_EXTRA="-DMICROPY_PY_SELECT=1"
 	cp micropython/ports/unix/build-standard/micropython ${CACHE_DIR}
 }