Skip to content

Commit df18fe9

Browse files
bretambrosebretambrose
authored
Custom auth fixes, tests, and CI framework (#422)
* Remove CI jobs that used CRT containers. We should not use CRT credentials in any way in SDK repos. * A variety of custom auth bug fixes, primarily around websockets and mqtt311 * Adds full support for signed custom authorizers to both mqtt5 and mqtt311 builders * Restores unit tests runs when built with aws-crt-builder * Adds matrix of custom auth tests for both mqtt311 and mqtt5
1 parent 0af8307 commit df18fe9

File tree

7 files changed

+364
-118
lines changed

7 files changed

+364
-118
lines changed

‎.builder/actions/crt-ci-test.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import Builder
2+
import re
3+
4+
class CiTest(Builder.Action):
5+
6+
def _write_environment_script_secret_to_env(self, env, secret_name):
7+
mqtt5_ci_environment_script = env.shell.get_secret(secret_name)
8+
env_line = re.compile('^export\s+(\w+)=(.+)')
9+
10+
lines = mqtt5_ci_environment_script.splitlines()
11+
for line in lines:
12+
env_pair_match = env_line.match(line)
13+
if env_pair_match.group(1) and env_pair_match.group(2):
14+
env.shell.setenv(env_pair_match.group(1), env_pair_match.group(2), quiet=True)
15+
16+
17+
def run(self, env):
18+
19+
actions = []
20+
21+
try:
22+
self._write_environment_script_secret_to_env(env, "ci/sdk-unit-testing")
23+
24+
env.shell.exec(["python3", "-m", "unittest", "discover", "--verbose"], check=True)
25+
except:
26+
print(f'Failure while running tests')
27+
actions.append("exit 1")
28+
29+
return Builder.Script(actions, name='ci-test')

‎.github/workflows/ci.yml

Lines changed: 33 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ env:
2121
CI_UTILS_FOLDER: "./aws-iot-device-sdk-python-v2/utils"
2222
CI_SAMPLES_CFG_FOLDER: "./aws-iot-device-sdk-python-v2/.github/workflows"
2323
CI_SAMPLES_FOLDER: "./aws-iot-device-sdk-python-v2/samples"
24-
CI_IOT_CONTAINERS_ROLE: ${{ secrets.AWS_CI_IOT_CONTAINERS }}
2524
CI_PUBSUB_ROLE: ${{ secrets.AWS_CI_PUBSUB_ROLE }}
2625
CI_COGNITO_ROLE: ${{ secrets.AWS_CI_COGNITO_ROLE }}
2726
CI_X509_ROLE: ${{ secrets.AWS_CI_X509_ROLE }}
@@ -31,77 +30,27 @@ env:
3130
CI_FLEET_PROVISIONING_ROLE: ${{ secrets.AWS_CI_FLEET_PROVISIONING_ROLE }}
3231
CI_DEVICE_ADVISOR: ${{ secrets.AWS_CI_DEVICE_ADVISOR_ROLE }}
3332
CI_MQTT5_ROLE: ${{ secrets.AWS_CI_MQTT5_ROLE }}
33+
CI_BUILD_AND_TEST_ROLE: arn:aws:iam::180635532705:role/V2_SDK_Unit_Testing
3434

3535
jobs:
36-
all-python-versions:
37-
runs-on: ubuntu-latest
38-
permissions:
39-
id-token: write # This is required for requesting the JWT
40-
steps:
41-
- name: Build ${{ env.PACKAGE_NAME }}
42-
env:
43-
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_DA_ROLE_KEY }}
44-
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_DA_ROLE_PRIVATE_KEY }}
45-
# There's hackery in builder.json so that when we run on manylinux
46-
# we build and test using every version of python that we support.
47-
run: |
48-
aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
49-
./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-x64 build -p ${{ env.PACKAGE_NAME }}
5036

51-
52-
al2:
53-
runs-on: ubuntu-latest
37+
windows:
38+
runs-on: windows-latest
5439
permissions:
5540
id-token: write # This is required for requesting the JWT
5641
steps:
57-
- name: configure AWS credentials (containers)
58-
uses: aws-actions/configure-aws-credentials@v1
59-
with:
60-
role-to-assume: ${{ env.CI_IOT_CONTAINERS_ROLE }}
61-
aws-region: ${{ env.AWS_DEFAULT_REGION }}
62-
# We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages
63-
- name: Build ${{ env.PACKAGE_NAME }}
42+
- name: Install boto3
6443
run: |
65-
aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
66-
./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-al2-x64 build -p ${{ env.PACKAGE_NAME }}
67-
68-
69-
raspberry:
70-
runs-on: ubuntu-latest # latest
71-
permissions:
72-
id-token: write # This is required for requesting the JWT
73-
strategy:
74-
fail-fast: false
75-
matrix:
76-
image:
77-
- raspbian-bullseye
78-
steps:
44+
python -m pip install boto3
7945
- name: configure AWS credentials (containers)
80-
uses: aws-actions/configure-aws-credentials@v1
46+
uses: aws-actions/configure-aws-credentials@v2
8147
with:
82-
role-to-assume: ${{ env.CI_IOT_CONTAINERS_ROLE }}
48+
role-to-assume: ${{ env.CI_BUILD_AND_TEST_ROLE }}
8349
aws-region: ${{ env.AWS_DEFAULT_REGION }}
84-
# set arm arch
85-
- name: Install qemu/docker
86-
run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
87-
- name: Build ${{ env.PACKAGE_NAME }}
88-
run: |
89-
aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
90-
./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }}
91-
92-
93-
windows:
94-
runs-on: windows-latest
95-
permissions:
96-
id-token: write # This is required for requesting the JWT
97-
steps:
9850
- name: Build ${{ env.PACKAGE_NAME }}
9951
run: |
10052
python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
10153
python builder.pyz build -p ${{ env.PACKAGE_NAME }}
102-
- name: Running samples in CI setup
103-
run: |
104-
python -m pip install boto3
10554
- name: configure AWS credentials (PubSub)
10655
uses: aws-actions/configure-aws-credentials@v1
10756
with:
@@ -136,6 +85,14 @@ jobs:
13685
permissions:
13786
id-token: write # This is required for requesting the JWT
13887
steps:
88+
- name: Install boto3
89+
run: |
90+
python3 -m pip install boto3
91+
- name: configure AWS credentials (containers)
92+
uses: aws-actions/configure-aws-credentials@v2
93+
with:
94+
role-to-assume: ${{ env.CI_BUILD_AND_TEST_ROLE }}
95+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
13996
- name: Build ${{ env.PACKAGE_NAME }}
14097
run: |
14198
python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
@@ -175,6 +132,14 @@ jobs:
175132
permissions:
176133
id-token: write # This is required for requesting the JWT
177134
steps:
135+
- name: Running samples in CI setup
136+
run: |
137+
python -m pip install boto3
138+
- name: configure AWS credentials (containers)
139+
uses: aws-actions/configure-aws-credentials@v2
140+
with:
141+
role-to-assume: ${{ env.CI_BUILD_AND_TEST_ROLE }}
142+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
178143
- name: Build ${{ env.PACKAGE_NAME }}
179144
run: |
180145
python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
@@ -215,17 +180,22 @@ jobs:
215180
permissions:
216181
id-token: write # This is required for requesting the JWT
217182
steps:
218-
- name: Build ${{ env.PACKAGE_NAME }}
219-
run: |
220-
python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
221-
chmod a+x builder
222-
./builder build -p ${{ env.PACKAGE_NAME }}
223183
- name: Running samples in CI setup
224184
run: |
225185
python3 -m pip install boto3
226186
sudo apt-get update -y
227187
sudo apt-get install softhsm -y
228188
softhsm2-util --version
189+
- name: configure AWS credentials (containers)
190+
uses: aws-actions/configure-aws-credentials@v2
191+
with:
192+
role-to-assume: ${{ env.CI_BUILD_AND_TEST_ROLE }}
193+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
194+
- name: Build ${{ env.PACKAGE_NAME }}
195+
run: |
196+
python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
197+
chmod a+x builder
198+
./builder build -p ${{ env.PACKAGE_NAME }}
229199
- name: configure AWS credentials (Connect and PubSub)
230200
uses: aws-actions/configure-aws-credentials@v1
231201
with:

‎awsiot/mqtt5_client_builder.py

Lines changed: 55 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -569,6 +569,8 @@ def direct_with_custom_authorizer(
569569
auth_authorizer_name=None,
570570
auth_authorizer_signature=None,
571571
auth_password=None,
572+
auth_token_key_name=None,
573+
auth_token_value=None,
572574
**kwargs) -> awscrt.mqtt5.Client:
573575
"""
574576
This builder creates an :class:`awscrt.mqtt5.Client`, configured for an MQTT5 Client using a custom
@@ -581,17 +583,30 @@ def direct_with_custom_authorizer(
581583
auth_username (`str`): The username to use with the custom authorizer.
582584
If provided, the username given will be passed when connecting to the custom authorizer.
583585
If not provided, it will check to see if a username has already been set (via username="example")
584-
and will use that instead.
585-
If no username has been set then no username will be sent with the MQTT connection.
586-
587-
auth_authorizer_name (`str`): The name of the custom authorizer.
588-
If not provided, then "x-amz-customauthorizer-name" will not be added with the MQTT connection.
589-
590-
auth_authorizer_signature (`str`): The signature of the custom authorizer.
591-
If not provided, then "x-amz-customauthorizer-name" will not be added with the MQTT connection.
586+
and will use that instead. Custom authentication parameters will be appended as appropriate
587+
to any supplied username value.
592588
593589
auth_password (`str`): The password to use with the custom authorizer.
594-
If not provided, then no passord will be set.
590+
If not provided, then no password will be sent in the initial CONNECT packet.
591+
592+
auth_authorizer_name (`str`): Name of the custom authorizer to use.
593+
Required if the endpoint does not have a default custom authorizer associated with it. It is strongly
594+
suggested to URL-encode this value; the SDK will not do so for you.
595+
596+
auth_authorizer_signature (`str`): The digital signature of the token value in the `auth_token_value`
597+
parameter. The signature must be based on the private key associated with the custom authorizer. The
598+
signature must be base64 encoded.
599+
Required if the custom authorizer has signing enabled. It is strongly suggested to URL-encode this value;
600+
the SDK will not do so for you.
601+
602+
auth_token_key_name (`str`): Key used to extract the custom authorizer token from MQTT username query-string
603+
properties.
604+
Required if the custom authorizer has signing enabled. It is strongly suggested to URL-encode
605+
this value; the SDK will not do so for you.
606+
607+
auth_token_value (`str`): An opaque token value. This value must be signed by the private key associated with
608+
the custom authorizer and the result passed in via the `auth_authorizer_signature` parameter.
609+
Required if the custom authorizer has signing enabled.
595610
"""
596611

597612
_check_required_kwargs(**kwargs)
@@ -606,10 +621,14 @@ def direct_with_custom_authorizer(
606621
if auth_authorizer_name is not None:
607622
username_string = _add_to_username_parameter(
608623
username_string, auth_authorizer_name, "x-amz-customauthorizer-name=")
624+
609625
if auth_authorizer_signature is not None:
610626
username_string = _add_to_username_parameter(
611627
username_string, auth_authorizer_signature, "x-amz-customauthorizer-signature=")
612628

629+
if auth_token_key_name is not None and auth_token_value is not None:
630+
username_string = _add_to_username_parameter(username_string, auth_token_value, auth_token_key_name + "=")
631+
613632
kwargs["username"] = username_string
614633
kwargs["password"] = auth_password
615634

@@ -628,6 +647,8 @@ def websockets_with_custom_authorizer(
628647
auth_authorizer_signature=None,
629648
auth_password=None,
630649
websocket_proxy_options=None,
650+
auth_token_key_name=None,
651+
auth_token_value=None,
631652
**kwargs) -> awscrt.mqtt5.Client:
632653
"""
633654
This builder creates an :class:`awscrt.mqtt5.Client`, configured for an MQTT5 Client using a custom
@@ -640,17 +661,30 @@ def websockets_with_custom_authorizer(
640661
auth_username (`str`): The username to use with the custom authorizer.
641662
If provided, the username given will be passed when connecting to the custom authorizer.
642663
If not provided, it will check to see if a username has already been set (via username="example")
643-
and will use that instead.
644-
If no username has been set then no username will be sent with the MQTT connection.
664+
and will use that instead. Custom authentication parameters will be appended as appropriate
665+
to any supplied username value.
666+
667+
auth_password (`str`): The password to use with the custom authorizer.
668+
If not provided, then no password will be sent in the initial CONNECT packet.
645669
646-
auth_authorizer_name (`str`): The name of the custom authorizer.
647-
If not provided, then "x-amz-customauthorizer-name" will not be added with the MQTT connection.
670+
auth_authorizer_name (`str`): Name of the custom authorizer to use.
671+
Required if the endpoint does not have a default custom authorizer associated with it. It is strongly
672+
suggested to URL-encode this value; the SDK will not do so for you.
648673
649-
auth_authorizer_signature (`str`): The signature of the custom authorizer.
650-
If not provided, then "x-amz-customauthorizer-name" will not be added with the MQTT connection.
674+
auth_authorizer_signature (`str`): The digital signature of the token value in the `auth_token_value`
675+
parameter. The signature must be based on the private key associated with the custom authorizer. The
676+
signature must be base64 encoded.
677+
Required if the custom authorizer has signing enabled. It is strongly suggested to URL-encode this value;
678+
the SDK will not do so for you.
651679
652-
auth_password (`str`): The password to use with the custom authorizer.
653-
If not provided, then no passord will be set.
680+
auth_token_key_name (`str`): Key used to extract the custom authorizer token from MQTT username query-string
681+
properties.
682+
Required if the custom authorizer has signing enabled. It is strongly suggested to URL-encode
683+
this value; the SDK will not do so for you.
684+
685+
auth_token_value (`str`): An opaque token value. This value must be signed by the private key associated with
686+
the custom authorizer and the result passed in via the `auth_authorizer_signature` parameter.
687+
Required if the custom authorizer has signing enabled.
654688
655689
websocket_proxy_options (awscrt.http.HttpProxyOptions): Deprecated,
656690
for proxy settings use `http_proxy_options` (described in
@@ -669,10 +703,14 @@ def websockets_with_custom_authorizer(
669703
if auth_authorizer_name is not None:
670704
username_string = _add_to_username_parameter(
671705
username_string, auth_authorizer_name, "x-amz-customauthorizer-name=")
706+
672707
if auth_authorizer_signature is not None:
673708
username_string = _add_to_username_parameter(
674709
username_string, auth_authorizer_signature, "x-amz-customauthorizer-signature=")
675710

711+
if auth_token_key_name is not None and auth_token_value is not None:
712+
username_string = _add_to_username_parameter(username_string, auth_token_value, auth_token_key_name + "=")
713+
676714
kwargs["username"] = username_string
677715
kwargs["password"] = auth_password
678716

0 commit comments

Comments
 (0)