From 579c0d2a68a0385c65f598c9477039dd4e6b1482 Mon Sep 17 00:00:00 2001 From: iabdalkader Date: Fri, 25 Aug 2023 15:00:56 +0200 Subject: [PATCH 1/2] ussl: Clean up SSL and m2crypto code. - Move all SSL code to ussl module. - Allow the client to be used without installing m2crypto if it's not needed. --- examples/example.py | 2 +- examples/micropython.py | 4 +- pyproject.toml | 1 - src/arduino_iot_cloud/ucloud.py | 22 ++++----- src/arduino_iot_cloud/umqtt.py | 10 ++-- src/arduino_iot_cloud/ussl.py | 84 ++++++++++++++++++--------------- 6 files changed, 61 insertions(+), 62 deletions(-) diff --git a/examples/example.py b/examples/example.py index 9a15077..bb8397d 100644 --- a/examples/example.py +++ b/examples/example.py @@ -11,7 +11,7 @@ from arduino_iot_cloud import Task from random import uniform import argparse -import arduino_iot_cloud.ussl as ssl +import ssl from secrets import DEVICE_ID from secrets import SECRET_KEY # noqa diff --git a/examples/micropython.py b/examples/micropython.py index 77aed73..82927d9 100644 --- a/examples/micropython.py +++ b/examples/micropython.py @@ -2,7 +2,7 @@ # Any copyright is dedicated to the Public Domain. # https://creativecommons.org/publicdomain/zero/1.0/ import time -import ussl +import ssl import network import logging from time import strftime @@ -73,7 +73,7 @@ def wifi_connect(): client = ArduinoCloudClient( device_id=DEVICE_ID, ssl_params={ - "keyfile": KEY_PATH, "certfile": CERT_PATH, "cadata": CADATA, "cert_reqs": ussl.CERT_REQUIRED + "keyfile": KEY_PATH, "certfile": CERT_PATH, "cadata": CADATA, "cert_reqs": ssl.CERT_REQUIRED } ) diff --git a/pyproject.toml b/pyproject.toml index c895d15..1d1eb45 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,6 @@ classifiers = [ ] dependencies = [ 'cbor2 >= 5.4.6', - 'M2Crypto >= 0.38.0', 'micropython-senml >= 0.1.0', ] diff --git a/src/arduino_iot_cloud/ucloud.py b/src/arduino_iot_cloud/ucloud.py index a8365b6..a72ee19 100644 --- a/src/arduino_iot_cloud/ucloud.py +++ b/src/arduino_iot_cloud/ucloud.py @@ -5,6 +5,7 @@ # file, You can obtain one at https://mozilla.org/MPL/2.0/. import time +import sys import logging from senml import SenmlPack from senml import SenmlRecord @@ -175,6 +176,14 @@ def __init__( self.senmlpack = SenmlPack("", self.senml_generic_callback) self.started = False + if "pin" in ssl_params: + try: + # Use M2Crypto to load key and cert from HSM. + import M2Crypto # noqa + except (ImportError, AttributeError): + logging.error("The m2crypto module is required to use HSM.") + sys.exit(1) + # Convert args to bytes if they are passed as strings. if isinstance(device_id, str): device_id = bytes(device_id, "utf-8") @@ -188,19 +197,6 @@ def __init__( # Update RTC from NTP server on MicroPython. self.update_systime(ntp_server, ntp_timeout) - # MicroPython does not support secure elements yet, and key/cert - # must be loaded from DER files and passed as binary blobs. - if "keyfile" in ssl_params and "der" in ssl_params["keyfile"]: - with open(ssl_params.pop("keyfile"), "rb") as f: - ssl_params["key"] = f.read() - if "certfile" in ssl_params and "der" in ssl_params["certfile"]: - with open(ssl_params.pop("certfile"), "rb") as f: - ssl_params["cert"] = f.read() - - if "ca_certs" in ssl_params and "der" in ssl_params["ca_certs"]: - with open(ssl_params.pop("ca_certs"), "rb") as f: - ssl_params["cadata"] = f.read() - # If no server/port were passed in args, set the default server/port # based on authentication type. if server is None: diff --git a/src/arduino_iot_cloud/umqtt.py b/src/arduino_iot_cloud/umqtt.py index 1c727e6..cea39b0 100644 --- a/src/arduino_iot_cloud/umqtt.py +++ b/src/arduino_iot_cloud/umqtt.py @@ -26,11 +26,7 @@ import struct import select import logging - -try: - from ussl import wrap_socket -except ImportError: - from arduino_iot_cloud.ussl import wrap_socket +import arduino_iot_cloud.ussl as ssl class MQTTException(Exception): @@ -99,14 +95,14 @@ def connect(self, clean_session=True, timeout=5.0): try: self.sock = socket.socket() self.sock.settimeout(timeout) - self.sock = wrap_socket(self.sock, **self.ssl_params) + self.sock = ssl.wrap_socket(self.sock, self.ssl_params) self.sock.connect(addr) except Exception: self.sock.close() self.sock = socket.socket() self.sock.settimeout(timeout) self.sock.connect(addr) - self.sock = wrap_socket(self.sock, **self.ssl_params) + self.sock = ssl.wrap_socket(self.sock, self.ssl_params) premsg = bytearray(b"\x10\0\0\0\0\0") msg = bytearray(b"\x04MQTT\x04\x02\0\0") diff --git a/src/arduino_iot_cloud/ussl.py b/src/arduino_iot_cloud/ussl.py index 1c5a667..d9c4427 100644 --- a/src/arduino_iot_cloud/ussl.py +++ b/src/arduino_iot_cloud/ussl.py @@ -6,64 +6,72 @@ # # SSL module with m2crypto backend for HSM support. -from M2Crypto import Engine, m2, SSL +import sys +import ssl -CERT_NONE = SSL.verify_none -CERT_REQUIRED = SSL.verify_peer - -_key = None -_cert = None +pkcs11 = None # Default engine and provider. _ENGINE_PATH = "/usr/lib/engines-3/libpkcs11.so" _MODULE_PATH = "/usr/lib/softhsm/libsofthsm2.so" -def init(pin, certfile, keyfile, engine_path, module_path): - global _key, _cert - Engine.load_dynamic_engine("pkcs11", engine_path) - pkcs11 = Engine.Engine("pkcs11") - pkcs11.ctrl_cmd_string("MODULE_PATH", module_path) - pkcs11.ctrl_cmd_string("PIN", pin) - pkcs11.init() - _key = pkcs11.load_private_key(keyfile) - _cert = pkcs11.load_certificate(certfile) - - def wrap_socket( - sock_in, - pin=None, - certfile=None, - keyfile=None, - ca_certs=None, - cert_reqs=CERT_NONE, - ciphers=None, - engine_path=_ENGINE_PATH, - module_path=_MODULE_PATH, + sock, + ssl_params={}, ): - if certfile is None or keyfile is None: - # Fallback to Python's SSL - import ssl - return ssl.wrap_socket(sock_in) + if any(k not in ssl_params for k in ("keyfile", "certfile", "pin")): + # Use Micro/CPython's SSL + if sys.implementation.name == "micropython": + # Load key, cert and CA from DER files, and pass them as binary blobs. + mpargs = {"keyfile": "key", "certfile": "cert", "ca_certs": "cadata"} + for k, v in mpargs.items(): + if k in ssl_params and "der" in ssl_params[k]: + with open(ssl_params.pop(k), "rb") as f: + ssl_params[v] = f.read() + return ssl.wrap_socket(sock, **ssl_params) + + # Use M2Crypto to load key and cert from HSM. + from M2Crypto import m2, SSL, Engine - if _key is None or _cert is None: - init(pin, certfile, keyfile, engine_path, module_path) + global pkcs11 + if pkcs11 is None: + pkcs11 = Engine.load_dynamic_engine( + "pkcs11", ssl_params.get("engine_path", _ENGINE_PATH) + ) + pkcs11.ctrl_cmd_string( + "MODULE_PATH", ssl_params.get("module_path", _MODULE_PATH) + ) + pkcs11.ctrl_cmd_string("PIN", ssl_params["pin"]) + pkcs11.init() - # Create SSL context + # Create and configure SSL context ctx = SSL.Context("tls") ctx.set_default_verify_paths() ctx.set_allow_unknown_ca(False) + ciphers = ssl_params.get("ciphers", None) if ciphers is not None: ctx.set_cipher_list(ciphers) - if ca_certs is not None and cert_reqs is not CERT_NONE: + ca_certs = ssl_params.get("ca_certs", None) + if ca_certs is not None: if ctx.load_verify_locations(ca_certs) != 1: raise Exception("Failed to load CA certs") - ctx.set_verify(SSL.verify_peer, depth=9) + + cert_reqs = ssl_params.get("cert_reqs", ssl.CERT_NONE) + if cert_reqs == ssl.CERT_NONE: + cert_reqs = SSL.verify_none + else: + cert_reqs = SSL.verify_peer + ctx.set_verify(cert_reqs, depth=9) # Set key/cert - m2.ssl_ctx_use_x509(ctx.ctx, _cert.x509) - m2.ssl_ctx_use_pkey_privkey(ctx.ctx, _key.pkey) + key = pkcs11.load_private_key(ssl_params["keyfile"]) + m2.ssl_ctx_use_pkey_privkey(ctx.ctx, key.pkey) + + cert = pkcs11.load_certificate(ssl_params["certfile"]) + m2.ssl_ctx_use_x509(ctx.ctx, cert.x509) + SSL.Connection.postConnectionCheck = None - return SSL.Connection(ctx, sock=sock_in) + return SSL.Connection(ctx, sock=sock) From 7b315b6da15c8fa3060da36303b94fdef55625a4 Mon Sep 17 00:00:00 2001 From: iabdalkader Date: Fri, 25 Aug 2023 15:03:44 +0200 Subject: [PATCH 2/2] misc: Update workflow and CI test. --- .github/workflows/client-test.yml | 12 +++++++++-- tests/ci.py | 36 ++++++++++++++++++++++++------- tests/ci.sh | 12 ++++++----- 3 files changed, 45 insertions(+), 15 deletions(-) diff --git a/.github/workflows/client-test.yml b/.github/workflows/client-test.yml index e838988..18118c4 100644 --- a/.github/workflows/client-test.yml +++ b/.github/workflows/client-test.yml @@ -76,7 +76,15 @@ jobs: DEVICE_ID: ${{ secrets.DEVICE_ID1 }} SECRET_KEY: ${{ secrets.SECRET_KEY }} run: | - python tests/ci.py + python tests/ci.py --basic-auth + + - name: '☁️ Connect to IoT cloud (CPython / Key/Cert Auth)' + env: + DEVICE_ID: ${{ secrets.DEVICE_ID2 }} + SECRET_KEY: ${{ secrets.SECRET_KEY }} + run: | + python tests/ci.py --file-auth + - name: '☁️ Connect to IoT cloud (CPython / Crypto Auth)' env: @@ -93,4 +101,4 @@ jobs: run: | export PATH="${HOME}/cache/bin:${PATH}" micropython -c "import sys; print(sys.path)" - micropython tests/ci.py + micropython tests/ci.py --basic-auth diff --git a/tests/ci.py b/tests/ci.py index 45c277e..9bf3288 100644 --- a/tests/ci.py +++ b/tests/ci.py @@ -27,7 +27,13 @@ def on_value_changed(client, value): "-d", "--debug", action="store_true", help="Enable debugging messages" ) parser.add_argument( - "-c", "--crypto-device", action="store_true", help="Use crypto device" + "-b", "--basic-auth", action="store_true", help="Username and password auth", + ) + parser.add_argument( + "-c", "--crypto-device", action="store_true", help="Use soft-hsm/crypto device", + ) + parser.add_argument( + "-f", "--file-auth", action="store_true", help="Use key/cert files" ) args = parser.parse_args() @@ -45,8 +51,25 @@ def on_value_changed(client, value): # the CA certificate (if any) in "ssl_params". Alternatively, a username and password can # be used to authenticate, for example: # client = ArduinoCloudClient(device_id=DEVICE_ID, username=DEVICE_ID, password=SECRET_KEY) - if args.crypto_device: - import arduino_iot_cloud.ussl as ssl + if args.basic_auth: + client = ArduinoCloudClient( + device_id=os.getenv("DEVICE_ID"), + username=os.getenv("DEVICE_ID"), + password=os.getenv("SECRET_KEY"), + ) + elif args.file_auth: + import ssl + client = ArduinoCloudClient( + device_id=os.getenv("DEVICE_ID"), + ssl_params={ + "keyfile": "key.pem", + "certfile": "cert.pem", + "ca_certs": "ca-root.pem", + "cert_reqs": ssl.CERT_REQUIRED, + }, + ) + elif args.crypto_device: + import ssl client = ArduinoCloudClient( device_id=os.getenv("DEVICE_ID"), ssl_params={ @@ -60,11 +83,8 @@ def on_value_changed(client, value): }, ) else: - client = ArduinoCloudClient( - device_id=os.getenv("DEVICE_ID"), - username=os.getenv("DEVICE_ID"), - password=os.getenv("SECRET_KEY"), - ) + parser.print_help() + sys.exit(1) # Register cloud objects. # Note: The following objects must be created first in the dashboard and linked to the device. diff --git a/tests/ci.sh b/tests/ci.sh index b300947..77793a7 100755 --- a/tests/ci.sh +++ b/tests/ci.sh @@ -9,17 +9,19 @@ ci_install_micropython() { git clone --depth=1 https://github.com/micropython/micropython.git cat > micropython/ports/unix/manifest.py <<-EOF - include("\$(PORT_DIR)/variants/manifest.py") - include("\$(MPY_DIR)/extmod/asyncio") + include("\$(PORT_DIR)/variants/standard/manifest.py") require("bundle-networking") require("time") require("senml") require("logging") EOF - make -C micropython/mpy-cross/ - make -C micropython/ports/unix/ submodules - make -C micropython/ports/unix/ FROZEN_MANIFEST=manifest.py CFLAGS_EXTRA="-DMICROPY_PY_SELECT=1" + echo "#undef MICROPY_PY_SELECT_SELECT" >> micropython/ports/unix/variants/mpconfigvariant_common.h + echo "#undef MICROPY_PY_SELECT_POSIX_OPTIMISATIONS" >> micropython/ports/unix/variants/mpconfigvariant_common.h + + make -j12 -C micropython/mpy-cross/ + make -j12 -C micropython/ports/unix/ submodules + make -j12 -C micropython/ports/unix/ FROZEN_MANIFEST=manifest.py CFLAGS_EXTRA="-DMICROPY_PY_SELECT=1" cp micropython/ports/unix/build-standard/micropython ${CACHE_DIR} }