Skip to content

Commit 399c6a3

Browse files
TwistedTwiglegTwistedTwigleg
authored
CI Adjustment (#357)
Adjusts how CI runs so it uses OpenID. Adjusts the README for the samples so the policies shown are minimal and working in all cases, adjusts all samples so they know when they are running in CI, and adds a GitHub action that runs all samples, including Shadow, Jobs, and Fleet Provisioning. It also modifies the jobs sample to display the number of pending jobs before trying to execute/resolve them, making it like the Java V2 Jobs sample.
1 parent 86f2503 commit 399c6a3

26 files changed

+1073
-232
lines changed

‎.github/workflows/ci.yml

Lines changed: 172 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,37 +13,206 @@ env:
1313
PACKAGE_NAME: aws-iot-device-sdk-python-v2
1414
LINUX_BASE_IMAGE: ubuntu-16-x64
1515
RUN: ${{ github.run_id }}-${{ github.run_number }}
16-
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_DATEST_ACCESS_KEY_ID }}
17-
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_DATEST_SECRET_ACCESS_KEY }}
1816
AWS_DEFAULT_REGION: us-east-1
17+
DA_TOPIC: test/da
18+
DA_SHADOW_PROPERTY: datest
19+
DA_SHADOW_VALUE_SET: ON
20+
DA_SHADOW_VALUE_DEFAULT: OFF
21+
CI_UTILS_FOLDER: "./aws-iot-device-sdk-python-v2/utils"
22+
CI_SAMPLES_FOLDER: "./aws-iot-device-sdk-python-v2/samples"
23+
CI_IOT_CONTAINERS: ${{ secrets.AWS_CI_IOT_CONTAINERS }}
24+
CI_PUBSUB_ROLE: ${{ secrets.AWS_CI_PUBSUB_ROLE }}
25+
CI_CUSTOM_AUTHORIZER_ROLE: ${{ secrets.AWS_CI_CUSTOM_AUTHORIZER_ROLE }}
26+
CI_SHADOW_ROLE: ${{ secrets.AWS_CI_SHADOW_ROLE }}
27+
CI_JOBS_ROLE: ${{ secrets.AWS_CI_JOBS_ROLE }}
28+
CI_FLEET_PROVISIONING_ROLE: ${{ secrets.AWS_CI_FLEET_PROVISIONING_ROLE }}
29+
CI_DEVICE_ADVISOR: ${{ secrets.AWS_CI_DEVICE_ADVISOR_ROLE }}
1930

2031
jobs:
2132

2233
al2:
2334
runs-on: ubuntu-latest
35+
permissions:
36+
id-token: write # This is required for requesting the JWT
2437
steps:
25-
# We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages
38+
- name: configure AWS credentials (containers)
39+
uses: aws-actions/configure-aws-credentials@v1
40+
with:
41+
role-to-assume: ${{ env.CI_IOT_CONTAINERS }}
42+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
43+
# We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages
2644
- name: Build ${{ env.PACKAGE_NAME }}
2745
run: |
2846
aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
2947
./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-al2-x64 build -p ${{ env.PACKAGE_NAME }}
3048
3149
windows:
3250
runs-on: windows-latest
51+
permissions:
52+
id-token: write # This is required for requesting the JWT
3353
steps:
3454
- name: Build ${{ env.PACKAGE_NAME }}
3555
run: |
3656
python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
3757
python builder.pyz build -p ${{ env.PACKAGE_NAME }}
58+
- name: Running samples in CI setup
59+
run: |
60+
python -m pip install boto3
61+
- name: configure AWS credentials (PubSub)
62+
uses: aws-actions/configure-aws-credentials@v1
63+
with:
64+
role-to-assume: ${{ env.CI_PUBSUB_ROLE }}
65+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
66+
- name: run PubSub sample
67+
run: |
68+
python ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key'
69+
- name: run Windows Certificate Connect sample
70+
run: |
71+
python ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/windows_cert_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' --sample_run_certutil true
72+
- name: configure AWS credentials (Device Advisor)
73+
uses: aws-actions/configure-aws-credentials@v1
74+
with:
75+
role-to-assume: ${{ env.CI_DEVICE_ADVISOR }}
76+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
77+
- name: run DeviceAdvisor
78+
run: |
79+
cd ./aws-iot-device-sdk-python-v2
80+
python ./deviceadvisor/script/DATestRun.py
3881
3982
osx:
4083
runs-on: macos-latest
84+
permissions:
85+
id-token: write # This is required for requesting the JWT
86+
steps:
87+
- name: Build ${{ env.PACKAGE_NAME }}
88+
run: |
89+
python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
90+
chmod a+x builder
91+
./builder build -p ${{ env.PACKAGE_NAME }}
92+
- name: Running samples in CI setup
93+
run: |
94+
python3 -m pip install boto3
95+
- name: configure AWS credentials (PubSub)
96+
uses: aws-actions/configure-aws-credentials@v1
97+
with:
98+
role-to-assume: ${{ env.CI_PUBSUB_ROLE }}
99+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
100+
- name: run PubSub sample
101+
run: |
102+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key'
103+
- name: configure AWS credentials (Device Advisor)
104+
uses: aws-actions/configure-aws-credentials@v1
105+
with:
106+
role-to-assume: ${{ env.CI_DEVICE_ADVISOR }}
107+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
108+
- name: run DeviceAdvisor
109+
run: |
110+
cd ./aws-iot-device-sdk-python-v2
111+
python3 ./deviceadvisor/script/DATestRun.py
112+
113+
linux:
114+
runs-on: ubuntu-20.04 # latest
115+
permissions:
116+
id-token: write # This is required for requesting the JWT
41117
steps:
42118
- name: Build ${{ env.PACKAGE_NAME }}
43119
run: |
44120
python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
45121
chmod a+x builder
46122
./builder build -p ${{ env.PACKAGE_NAME }}
123+
- name: Running samples in CI setup
124+
run: |
125+
python3 -m pip install boto3
126+
- name: configure AWS credentials (PubSub)
127+
uses: aws-actions/configure-aws-credentials@v1
128+
with:
129+
role-to-assume: ${{ env.CI_PUBSUB_ROLE }}
130+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
131+
- name: run PubSub sample
132+
run: |
133+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key'
134+
- name: configure AWS credentials (Device Advisor)
135+
uses: aws-actions/configure-aws-credentials@v1
136+
with:
137+
role-to-assume: ${{ env.CI_DEVICE_ADVISOR }}
138+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
139+
- name: run DeviceAdvisor
140+
run: |
141+
cd ./aws-iot-device-sdk-python-v2
142+
python3 ./deviceadvisor/script/DATestRun.py
143+
144+
# Runs the samples and ensures that everything is working
145+
linux-smoke-tests:
146+
runs-on: ubuntu-latest
147+
permissions:
148+
id-token: write # This is required for requesting the JWT
149+
steps:
150+
- name: Build ${{ env.PACKAGE_NAME }}
151+
run: |
152+
python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
153+
chmod a+x builder
154+
./builder build -p ${{ env.PACKAGE_NAME }}
155+
- name: Running samples in CI setup
156+
run: |
157+
python3 -m pip install boto3
158+
sudo apt-get update -y
159+
sudo apt-get install softhsm -y
160+
softhsm2-util --version
161+
- name: configure AWS credentials (Connect and PubSub)
162+
uses: aws-actions/configure-aws-credentials@v1
163+
with:
164+
role-to-assume: ${{ env.CI_PUBSUB_ROLE }}
165+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
166+
- name: run Basic Connect sample
167+
run: |
168+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/basic_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key'
169+
- name: run Websocket Connect sample
170+
run: |
171+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/websocket_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_arguments '--signing_region us-east-1'
172+
- name: run PubSub sample
173+
run: |
174+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key'
175+
- name: run PKCS11 Connect sample
176+
run: |
177+
mkdir -p /tmp/tokens
178+
export SOFTHSM2_CONF=/tmp/softhsm2.conf
179+
echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf
180+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key'
181+
- name: configure AWS credentials (Custom Authorizer)
182+
uses: aws-actions/configure-aws-credentials@v1
183+
with:
184+
role-to-assume: ${{ env.CI_CUSTOM_AUTHORIZER_ROLE }}
185+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
186+
- name: run CustomAuthorizerConnect sample
187+
run: |
188+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/custom_authorizer_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_custom_authorizer_name 'ci/CustomAuthorizer/name' --sample_secret_custom_authorizer_password 'ci/CustomAuthorizer/password'
189+
- name: configure AWS credentials (Shadow)
190+
uses: aws-actions/configure-aws-credentials@v1
191+
with:
192+
role-to-assume: ${{ env.CI_SHADOW_ROLE }}
193+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
194+
- name: run Shadow sample
195+
run: |
196+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/shadow.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/Shadow/cert' --sample_secret_private_key 'ci/Shadow/key' --sample_arguments '--thing_name CI_Shadow_Thing'
197+
- name: configure AWS credentials (Jobs)
198+
uses: aws-actions/configure-aws-credentials@v1
199+
with:
200+
role-to-assume: ${{ env.CI_JOBS_ROLE }}
201+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
202+
- name: run Jobs sample
203+
run: |
204+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/jobs.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/Jobs/cert' --sample_secret_private_key 'ci/Jobs/key' --sample_arguments '--thing_name CI_Jobs_Thing'
205+
- name: configure AWS credentials (Fleet provisioning)
206+
uses: aws-actions/configure-aws-credentials@v1
207+
with:
208+
role-to-assume: ${{ env.CI_FLEET_PROVISIONING_ROLE }}
209+
aws-region: ${{ env.AWS_DEFAULT_REGION }}
210+
- name: run Fleet Provisioning sample
211+
run: |
212+
echo "Generating UUID for IoT thing"
213+
Sample_UUID=$(python3 -c "import uuid; print (uuid.uuid4())")
214+
python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/fleetprovisioning.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/FleetProvisioning/cert' --sample_secret_private_key 'ci/FleetProvisioning/key' --sample_arguments "--template_name CI_FleetProvisioning_Template --template_parameters {\"SerialNumber\":\"${Sample_UUID}\"}"
215+
python3 ${{ env.CI_UTILS_FOLDER }}/delete_iot_thing_ci.py --thing_name "Fleet_Thing_${Sample_UUID}" --region "us-east-1"
47216
48217
# check that docs can still build
49218
check-docs:

‎README.md

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,18 +36,27 @@ to Python by the `awscrt` package ([PyPI](https://pypi.org/project/awscrt/)) ([G
3636
[Step-by-step instructions](./documents/PREREQUISITES.md)
3737

3838
### Install from PyPI
39+
40+
#### MacOS and Linux:
41+
3942
```
4043
python3 -m pip install awsiotsdk
4144
```
4245

46+
#### Windows:
47+
48+
```
49+
python -m pip install awsiotsdk
50+
```
51+
4352
### Install from source
4453
```
4554
# Create a workspace directory to hold all the SDK files
4655
mkdir sdk-workspace
4756
cd sdk-workspace
4857
# Clone the repository
4958
git clone https://github.com/aws/aws-iot-device-sdk-python-v2.git
50-
# Install using Pip
59+
# Install using Pip (use 'python' instead of 'python3' on Windows)
5160
python3 -m pip install ./aws-iot-device-sdk-python-v2
5261
```
5362

‎builder.json

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -89,14 +89,7 @@
8989
"python3 -m pip install ."
9090
],
9191
"test_steps": [
92-
"python3 -m pip install boto3",
93-
"python3 deviceadvisor/script/DATestRun.py"
9492
],
9593
"env": {
96-
"DA_TOPIC": "test/da",
97-
"DA_SHADOW_PROPERTY": "datest",
98-
"DA_SHADOW_VALUE_SET": "ON",
99-
"DA_SHADOW_VALUE_DEFAULT": "OFF",
100-
"DA_S3_NAME": "aws-iot-sdk-deviceadvisor-logs"
10194
}
10295
}

‎codebuild/samples/connect-auth-linux.sh

Lines changed: 0 additions & 16 deletions
This file was deleted.

‎codebuild/samples/connect-linux.sh

Lines changed: 0 additions & 17 deletions
This file was deleted.

‎codebuild/samples/linux-smoke-tests.yml

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,7 @@ phases:
99
commands:
1010
- echo Build started on `date`
1111
- $CODEBUILD_SRC_DIR/codebuild/samples/setup-linux.sh
12-
- $CODEBUILD_SRC_DIR/codebuild/samples/connect-linux.sh
13-
- $CODEBUILD_SRC_DIR/codebuild/samples/pkcs11-connect-linux.sh
1412
- $CODEBUILD_SRC_DIR/codebuild/samples/pubsub-linux.sh
15-
- $CODEBUILD_SRC_DIR/codebuild/samples/connect-auth-linux.sh
1613
post_build:
1714
commands:
1815
- echo Build completed on `date`

‎codebuild/samples/pkcs11-connect-linux.sh

Lines changed: 0 additions & 37 deletions
This file was deleted.

‎deviceadvisor/script/DATestConfig.json

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@
22
"tests" :["MQTT Connect", "MQTT Publish", "MQTT Subscribe", "Shadow Publish", "Shadow Update"],
33
"test_suite_ids" :
44
{
5-
"MQTT Connect" : "ejbdzmo3hf3v",
6-
"MQTT Publish" : "euw7favf6an4",
7-
"MQTT Subscribe" : "01o8vo6no7sd",
8-
"Shadow Publish" : "elztm2jebc1q",
9-
"Shadow Update" : "vuydgrbbbfce"
5+
"MQTT Connect" : "mxn32qkm8npn",
6+
"MQTT Publish" : "gcjhujhhz50p",
7+
"MQTT Subscribe" : "nyiuiwx5yxtj",
8+
"Shadow Publish" : "fttdr8ufljnf",
9+
"Shadow Update" : "ng9t8am2jnry"
1010
},
1111
"test_exe_path" :
1212
{

0 commit comments

Comments
 (0)