From 47b72d8dcea9962563dc14bd0c41c866e273184d Mon Sep 17 00:00:00 2001 From: Bret Ambrose Date: Tue, 13 Feb 2024 13:04:55 -0800 Subject: [PATCH 1/8] Use ssl context wrap socket in Python3.12+ --- AWSIoTPythonSDK/core/protocol/paho/client.py | 39 +++++++++++++++----- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/AWSIoTPythonSDK/core/protocol/paho/client.py b/AWSIoTPythonSDK/core/protocol/paho/client.py index 4216829..cb4fa3c 100755 --- a/AWSIoTPythonSDK/core/protocol/paho/client.py +++ b/AWSIoTPythonSDK/core/protocol/paho/client.py @@ -793,11 +793,22 @@ def reconnect(self): verify_hostname = self._tls_insecure is False # Decide whether we need to verify hostname + # To keep the SSL Context update minimal, only apply forced ssl context to python3.12+ + force_ssl_context = sys.version_info[0] > 3 or (sys.version_info[0] == 3 and sys.version_info[1] >= 12) + if self._tls_ca_certs is not None: if self._useSecuredWebsocket: # Never assign to ._ssl before wss handshake is finished # Non-None value for ._ssl will allow ops before wss-MQTT connection is established - rawSSL = ssl.wrap_socket(sock, ca_certs=self._tls_ca_certs, cert_reqs=ssl.CERT_REQUIRED) # Add server certificate verification + if force_ssl_context: + ssl_context = ssl.SSLContext() + ssl_context.load_verify_locations(self._tls_ca_certs) + ssl_context.verify_mode = ssl.CERT_REQUIRED + + rawSSL = ssl_context.wrap_socket(sock) + else: + rawSSL = ssl.wrap_socket(sock, ca_certs=self._tls_ca_certs, cert_reqs=ssl.CERT_REQUIRED) # Add server certificate verification + rawSSL.setblocking(0) # Non-blocking socket self._ssl = SecuredWebSocketCore(rawSSL, self._host, self._port, self._AWSAccessKeyIDCustomConfig, self._AWSSecretAccessKeyCustomConfig, self._AWSSessionTokenCustomConfig) # Override the _ssl socket # self._ssl.enableDebug() @@ -816,14 +827,24 @@ def reconnect(self): verify_hostname = False # Since check_hostname in SSLContext is already set to True, no need to verify it again self._ssl.do_handshake() else: - self._ssl = ssl.wrap_socket( - sock, - certfile=self._tls_certfile, - keyfile=self._tls_keyfile, - ca_certs=self._tls_ca_certs, - cert_reqs=self._tls_cert_reqs, - ssl_version=self._tls_version, - ciphers=self._tls_ciphers) + if force_ssl_context: + ssl_context = ssl.SSLContext(self._tls_version) + ssl_context.load_cert_chain(self._tls_certfile, self._tls_keyfile) + ssl_context.load_verify_locations(self._tls_ca_certs) + ssl_context.verify_mode = self._tls_cert_reqs + if self._tls_ciphers is not None: + ssl_context.set_ciphers(self._tls_ciphers) + + self._ssl = ssl_context.wrap_socket(sock) + else: + self._ssl = ssl.wrap_socket( + sock, + certfile=self._tls_certfile, + keyfile=self._tls_keyfile, + ca_certs=self._tls_ca_certs, + cert_reqs=self._tls_cert_reqs, + ssl_version=self._tls_version, + ciphers=self._tls_ciphers) if verify_hostname: if sys.version_info[0] < 3 or (sys.version_info[0] == 3 and sys.version_info[1] < 5): # No IP host match before 3.5.x From 14ae74202123254ee2011c2ead6062d1a17495fa Mon Sep 17 00:00:00 2001 From: Bret Ambrose Date: Tue, 13 Feb 2024 13:49:57 -0800 Subject: [PATCH 2/8] What version of python are we using? --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ff9adb..46b125d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,4 +49,5 @@ jobs: pip install pytest pip install mock pip install boto3 + python --version ./test-integration/run/run.sh ${{ matrix.test-type }} 1000 100 7 From c090dcfaa9d07dc13940c20fb706d1cb52c34069 Mon Sep 17 00:00:00 2001 From: Bret Ambrose Date: Tue, 13 Feb 2024 14:14:37 -0800 Subject: [PATCH 3/8] Another removed API --- AWSIoTPythonSDK/core/protocol/paho/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AWSIoTPythonSDK/core/protocol/paho/client.py b/AWSIoTPythonSDK/core/protocol/paho/client.py index cb4fa3c..0b637c5 100755 --- a/AWSIoTPythonSDK/core/protocol/paho/client.py +++ b/AWSIoTPythonSDK/core/protocol/paho/client.py @@ -849,7 +849,8 @@ def reconnect(self): if verify_hostname: if sys.version_info[0] < 3 or (sys.version_info[0] == 3 and sys.version_info[1] < 5): # No IP host match before 3.5.x self._tls_match_hostname() - else: + elif sys.version_info[0] == 3 and sys.version_info[1] < 7: + # host name verification is handled internally in Python3.7+ ssl.match_hostname(self._ssl.getpeercert(), self._host) self._sock = sock From 8f94bc5826116fabb9538c77447c01098644ad6b Mon Sep 17 00:00:00 2001 From: Bret Ambrose Date: Tue, 13 Feb 2024 14:33:53 -0800 Subject: [PATCH 4/8] Use a workaround to install 2.7; update GG discovery with SSL context branch --- .github/workflows/ci.yml | 7 +++--- .../core/greengrass/discovery/providers.py | 22 ++++++++++++++----- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46b125d..be31750 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,11 +37,10 @@ jobs: fail-fast: false matrix: test-type: [ MutualAuth, MutualAuthT , Websocket, ALPN, ALPNT] - python-version: [ '2.x', '3.x' ] - #[MutualAuth, Websocket, ALPN] + python-version: [ '2.7', '3.5', '3.7', '3.12' ] steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: LizardByte/setup-python-action@v2024.202.205923 with: python-version: ${{ matrix.python-version }} - name: Integration tests diff --git a/AWSIoTPythonSDK/core/greengrass/discovery/providers.py b/AWSIoTPythonSDK/core/greengrass/discovery/providers.py index b37b086..ad8ac22 100644 --- a/AWSIoTPythonSDK/core/greengrass/discovery/providers.py +++ b/AWSIoTPythonSDK/core/greengrass/discovery/providers.py @@ -261,12 +261,22 @@ def _create_ssl_connection(self, sock): ssl_sock = ssl_context.wrap_socket(sock, server_hostname=self._host, do_handshake_on_connect=False) ssl_sock.do_handshake() else: - ssl_sock = ssl.wrap_socket(sock, - certfile=self._cert_path, - keyfile=self._key_path, - ca_certs=self._ca_path, - cert_reqs=ssl.CERT_REQUIRED, - ssl_version=ssl_protocol_version) + # To keep the SSL Context update minimal, only apply forced ssl context to python3.12+ + force_ssl_context = sys.version_info[0] > 3 or (sys.version_info[0] == 3 and sys.version_info[1] >= 12) + if force_ssl_context: + ssl_context = ssl.SSLContext(ssl_protocol_version) + ssl_context.load_cert_chain(self._cert_path, self._key_path) + ssl_context.load_verify_locations(self._ca_path) + ssl_context.verify_mode = ssl.CERT_REQUIRED + + ssl_sock = ssl_context.wrap_socket(sock) + else: + ssl_sock = ssl.wrap_socket(sock, + certfile=self._cert_path, + keyfile=self._key_path, + ca_certs=self._ca_path, + cert_reqs=ssl.CERT_REQUIRED, + ssl_version=ssl_protocol_version) self._logger.debug("Matching host name...") if sys.version_info[0] < 3 or (sys.version_info[0] == 3 and sys.version_info[1] < 2): From ba91f5ae245a19030a3de3222b1bd8fc2f50bab3 Mon Sep 17 00:00:00 2001 From: Bret Ambrose Date: Tue, 13 Feb 2024 14:39:08 -0800 Subject: [PATCH 5/8] Not allowed to do that apparently --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be31750..b05f552 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,10 +37,10 @@ jobs: fail-fast: false matrix: test-type: [ MutualAuth, MutualAuthT , Websocket, ALPN, ALPNT] - python-version: [ '2.7', '3.5', '3.7', '3.12' ] + python-version: [ '3.5', '3.7', '3.12' ] steps: - uses: actions/checkout@v4 - - uses: LizardByte/setup-python-action@v2024.202.205923 + - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Integration tests From e3ca6749e59dd41caed84a7cee9349886a73a01d Mon Sep 17 00:00:00 2001 From: Bret Ambrose Date: Tue, 13 Feb 2024 14:46:25 -0800 Subject: [PATCH 6/8] Remove 3.5 and guard other call to match_hostname --- .github/workflows/ci.yml | 2 +- AWSIoTPythonSDK/core/greengrass/discovery/providers.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b05f552..b1b1cf7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,7 @@ jobs: fail-fast: false matrix: test-type: [ MutualAuth, MutualAuthT , Websocket, ALPN, ALPNT] - python-version: [ '3.5', '3.7', '3.12' ] + python-version: [ '3.7', '3.12' ] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v2 diff --git a/AWSIoTPythonSDK/core/greengrass/discovery/providers.py b/AWSIoTPythonSDK/core/greengrass/discovery/providers.py index ad8ac22..f9677ed 100644 --- a/AWSIoTPythonSDK/core/greengrass/discovery/providers.py +++ b/AWSIoTPythonSDK/core/greengrass/discovery/providers.py @@ -281,7 +281,7 @@ def _create_ssl_connection(self, sock): self._logger.debug("Matching host name...") if sys.version_info[0] < 3 or (sys.version_info[0] == 3 and sys.version_info[1] < 2): self._tls_match_hostname(ssl_sock) - else: + elif sys.version_info[0] == 3 and sys.version_info[1] < 7: ssl.match_hostname(ssl_sock.getpeercert(), self._host) return ssl_sock From c9a756eda57e97c9e163320e08dd8d43a6e5b62f Mon Sep 17 00:00:00 2001 From: Bret Ambrose Date: Tue, 13 Feb 2024 15:01:00 -0800 Subject: [PATCH 7/8] Remove redundant test modes --- .github/workflows/ci.yml | 2 +- test-integration/run/run.sh | 59 +++++++++++++------------------------ 2 files changed, 22 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1b1cf7..fd09c7e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: strategy: fail-fast: false matrix: - test-type: [ MutualAuth, MutualAuthT , Websocket, ALPN, ALPNT] + test-type: [ MutualAuth, Websocket, ALPN ] python-version: [ '3.7', '3.12' ] steps: - uses: actions/checkout@v4 diff --git a/test-integration/run/run.sh b/test-integration/run/run.sh index f420f73..0eb933b 100755 --- a/test-integration/run/run.sh +++ b/test-integration/run/run.sh @@ -35,16 +35,13 @@ USAGE="usage: run.sh ${CREDENTIAL_DIR}certificate.pem.crt - python ${RetrieveAWSKeys} ${AWSSetName_privatekey} > ${CREDENTIAL_DIR}privateKey.pem.key + python ${RetrieveAWSKeys} ${AWSSetName_certificate} > ${CREDENTIAL_DIR}certificate.pem.crt + python ${RetrieveAWSKeys} ${AWSSetName_privatekey} > ${CREDENTIAL_DIR}privateKey.pem.key curl -s "${CA_CERT_URL}" > ${CA_CERT_PATH} - echo -e "URL retrieved certificate data:\n$(cat ${CA_CERT_PATH})\n" - python ${RetrieveAWSKeys} ${AWSDRSName_certificate} > ${CREDENTIAL_DIR}certificate_drs.pem.crt - python ${RetrieveAWSKeys} ${AWSDRSName_privatekey} > ${CREDENTIAL_DIR}privateKey_drs.pem.key - elif [ "$1"x == "Websocket"x -o "$1"x == "WebsocketT"x ]; then - ACCESS_KEY_ID_ARN=$(python ${RetrieveAWSKeys} ${AWSSecretForWebsocket_TodWorker_KeyId}) + echo -e "URL retrieved certificate data\n" + python ${RetrieveAWSKeys} ${AWSDRSName_certificate} > ${CREDENTIAL_DIR}certificate_drs.pem.crt + python ${RetrieveAWSKeys} ${AWSDRSName_privatekey} > ${CREDENTIAL_DIR}privateKey_drs.pem.key + elif [ "$1"x == "Websocket"x ]; then + ACCESS_KEY_ID_ARN=$(python ${RetrieveAWSKeys} ${AWSSecretForWebsocket_TodWorker_KeyId}) ACCESS_SECRET_KEY_ARN=$(python ${RetrieveAWSKeys} ${AWSSecretForWebsocket_TodWorker_SecretKey}) TestMode="Websocket" - if [ "$1"x == "WebsocketT"x ]; then - ACCESS_KEY_ID_ARN=$(python ${RetrieveAWSKeys} ${AWSSecretForWebsocket_Desktop_KeyId}) - ACCESS_SECRET_KEY_ARN=$(python ${RetrieveAWSKeys} ${AWSSecretForWebsocket_Desktop_SecretKey}) - fi - echo ${ACCESS_KEY_ID_ARN} - echo ${ACCESS_SECRET_KEY_ARN} export AWS_ACCESS_KEY_ID=${ACCESS_KEY_ID_ARN} export AWS_SECRET_ACCESS_KEY=${ACCESS_SECRET_KEY_ARN} curl -s "${CA_CERT_URL}" > ${CA_CERT_PATH} - echo -e "URL retrieved certificate data:\n$(cat ${CA_CERT_PATH})\n" - elif [ "$1"x == "ALPN"x -o "$1"x == "ALPNT"x ]; then + echo -e "URL retrieved certificate data\n" + elif [ "$1"x == "ALPN"x ]; then AWSSetName_privatekey=${AWSMutualAuth_TodWorker_private_key} - AWSSetName_certificate=${AWSMutualAuth_TodWorker_certificate} - AWSDRSName_privatekey=${AWSGGDiscovery_TodWorker_private_key} + AWSSetName_certificate=${AWSMutualAuth_TodWorker_certificate} + AWSDRSName_privatekey=${AWSGGDiscovery_TodWorker_private_key} AWSDRSName_certificate=${AWSGGDiscovery_TodWorker_certificate} TestMode="ALPN" - if [ "$1"x == "ALPNT"x ]; then - AWSSetName_privatekey=${AWSMutualAuth_Desktop_private_key} - AWSSetName_certificate=${AWSMutualAuth_Desktop_certificate} - fi python ${RetrieveAWSKeys} ${AWSSetName_certificate} > ${CREDENTIAL_DIR}certificate.pem.crt - python ${RetrieveAWSKeys} ${AWSSetName_privatekey} > ${CREDENTIAL_DIR}privateKey.pem.key + python ${RetrieveAWSKeys} ${AWSSetName_privatekey} > ${CREDENTIAL_DIR}privateKey.pem.key curl -s "${CA_CERT_URL}" > ${CA_CERT_PATH} - echo -e "URL retrieved certificate data:\n$(cat ${CA_CERT_PATH})\n" - python ${RetrieveAWSKeys} ${AWSDRSName_certificate} > ${CREDENTIAL_DIR}certificate_drs.pem.crt - python ${RetrieveAWSKeys} ${AWSDRSName_privatekey} > ${CREDENTIAL_DIR}privateKey_drs.pem.key + echo -e "URL retrieved certificate data\n" + python ${RetrieveAWSKeys} ${AWSDRSName_certificate} > ${CREDENTIAL_DIR}certificate_drs.pem.crt + python ${RetrieveAWSKeys} ${AWSDRSName_privatekey} > ${CREDENTIAL_DIR}privateKey_drs.pem.key else - echo "Mode not supported" - exit 1 + echo "Mode not supported" + exit 1 fi # Obtain ZIP package and unzip it locally echo ${TestMode} From fa8c56ca2f25d914da7303ac3c6e79871831355e Mon Sep 17 00:00:00 2001 From: Bret Ambrose Date: Tue, 13 Feb 2024 15:02:58 -0800 Subject: [PATCH 8/8] Explanatory comment --- AWSIoTPythonSDK/core/greengrass/discovery/providers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/AWSIoTPythonSDK/core/greengrass/discovery/providers.py b/AWSIoTPythonSDK/core/greengrass/discovery/providers.py index f9677ed..192f71a 100644 --- a/AWSIoTPythonSDK/core/greengrass/discovery/providers.py +++ b/AWSIoTPythonSDK/core/greengrass/discovery/providers.py @@ -282,6 +282,7 @@ def _create_ssl_connection(self, sock): if sys.version_info[0] < 3 or (sys.version_info[0] == 3 and sys.version_info[1] < 2): self._tls_match_hostname(ssl_sock) elif sys.version_info[0] == 3 and sys.version_info[1] < 7: + # host name verification is handled internally in Python3.7+ ssl.match_hostname(ssl_sock.getpeercert(), self._host) return ssl_sock