aws · TwistedTwigleg · Oct 5, 2022 · Sep 29, 2022 · Sep 29, 2022 · Sep 29, 2022
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -13,37 +13,206 @@ env:
   PACKAGE_NAME: aws-iot-device-sdk-python-v2
   LINUX_BASE_IMAGE: ubuntu-16-x64
   RUN: ${{ github.run_id }}-${{ github.run_number }}
-  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_DATEST_ACCESS_KEY_ID }}
-  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_DATEST_SECRET_ACCESS_KEY }}
   AWS_DEFAULT_REGION: us-east-1
+  DA_TOPIC: test/da
+  DA_SHADOW_PROPERTY: datest
+  DA_SHADOW_VALUE_SET: ON
+  DA_SHADOW_VALUE_DEFAULT: OFF
+  CI_UTILS_FOLDER: "./aws-iot-device-sdk-python-v2/utils"
+  CI_SAMPLES_FOLDER: "./aws-iot-device-sdk-python-v2/samples"
+  CI_IOT_CONTAINERS: ${{ secrets.AWS_CI_IOT_CONTAINERS }}
+  CI_PUBSUB_ROLE: ${{ secrets.AWS_CI_PUBSUB_ROLE }}
+  CI_CUSTOM_AUTHORIZER_ROLE: ${{ secrets.AWS_CI_CUSTOM_AUTHORIZER_ROLE }}
+  CI_SHADOW_ROLE: ${{ secrets.AWS_CI_SHADOW_ROLE }}
+  CI_JOBS_ROLE: ${{ secrets.AWS_CI_JOBS_ROLE }}
+  CI_FLEET_PROVISIONING_ROLE: ${{ secrets.AWS_CI_FLEET_PROVISIONING_ROLE }}
+  CI_DEVICE_ADVISOR: ${{ secrets.AWS_CI_DEVICE_ADVISOR_ROLE }}
 
 jobs:
 
   al2:
     runs-on: ubuntu-latest
+    permissions:
+      id-token: write # This is required for requesting the JWT
     steps:
-        # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages
+    - name: configure AWS credentials (containers)
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        role-to-assume: ${{ env.CI_IOT_CONTAINERS }}
+        aws-region: ${{ env.AWS_DEFAULT_REGION }}
+      # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages
     - name: Build ${{ env.PACKAGE_NAME }}
       run: |
         aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
         ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-al2-x64 build -p ${{ env.PACKAGE_NAME }}
 
   windows:
     runs-on: windows-latest
+    permissions:
+      id-token: write # This is required for requesting the JWT
     steps:
     - name: Build ${{ env.PACKAGE_NAME }}
       run: |
         python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
         python builder.pyz build -p ${{ env.PACKAGE_NAME }}
+    - name: Running samples in CI setup
+      run: |
+        python -m pip install boto3
+    - name: configure AWS credentials (PubSub)
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        role-to-assume: ${{ env.CI_PUBSUB_ROLE }}
+        aws-region: ${{ env.AWS_DEFAULT_REGION }}
+    - name: run PubSub sample
+      run: |
+        python ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key'
+    - name: run Windows Certificate Connect sample
+      run: |
+        python ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/windows_cert_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key' --sample_run_certutil true
+    - name: configure AWS credentials (Device Advisor)
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        role-to-assume: ${{ env.CI_DEVICE_ADVISOR }}
+        aws-region: ${{ env.AWS_DEFAULT_REGION }}
+    - name: run DeviceAdvisor
+      run: |
+        cd ./aws-iot-device-sdk-python-v2
+        python ./deviceadvisor/script/DATestRun.py
 
   osx:
     runs-on: macos-latest
+    permissions:
+      id-token: write # This is required for requesting the JWT
+    steps:
+    - name: Build ${{ env.PACKAGE_NAME }}
+      run: |
+        python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
+        chmod a+x builder
+        ./builder build -p ${{ env.PACKAGE_NAME }}
+    - name: Running samples in CI setup
+      run: |
+        python3 -m pip install boto3
+    - name: configure AWS credentials (PubSub)
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        role-to-assume: ${{ env.CI_PUBSUB_ROLE }}
+        aws-region: ${{ env.AWS_DEFAULT_REGION }}
+    - name: run PubSub sample
+      run: |
+        python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key'
+    - name: configure AWS credentials (Device Advisor)
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        role-to-assume: ${{ env.CI_DEVICE_ADVISOR }}
+        aws-region: ${{ env.AWS_DEFAULT_REGION }}
+    - name: run DeviceAdvisor
+      run: |
+        cd ./aws-iot-device-sdk-python-v2
+        python3 ./deviceadvisor/script/DATestRun.py
+
+  linux:
+    runs-on: ubuntu-20.04 # latest
+    permissions:
+      id-token: write # This is required for requesting the JWT
     steps:
     - name: Build ${{ env.PACKAGE_NAME }}
       run: |
         python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
         chmod a+x builder
         ./builder build -p ${{ env.PACKAGE_NAME }}
+    - name: Running samples in CI setup
+      run: |
+        python3 -m pip install boto3
+    - name: configure AWS credentials (PubSub)
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        role-to-assume: ${{ env.CI_PUBSUB_ROLE }}
+        aws-region: ${{ env.AWS_DEFAULT_REGION }}
+    - name: run PubSub sample
+      run: |
+        python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key'
+    - name: configure AWS credentials (Device Advisor)
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        role-to-assume: ${{ env.CI_DEVICE_ADVISOR }}
+        aws-region: ${{ env.AWS_DEFAULT_REGION }}
+    - name: run DeviceAdvisor
+      run: |
+        cd ./aws-iot-device-sdk-python-v2
+        python3 ./deviceadvisor/script/DATestRun.py
+
+  # Runs the samples and ensures that everything is working
+  linux-smoke-tests:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write # This is required for requesting the JWT
+    steps:
+      - name: Build ${{ env.PACKAGE_NAME }}
+        run: |
+          python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
+          chmod a+x builder
+          ./builder build -p ${{ env.PACKAGE_NAME }}
+      - name: Running samples in CI setup
+        run: |
+          python3 -m pip install boto3
+          sudo apt-get update -y
+          sudo apt-get install softhsm -y
+          softhsm2-util --version
+      - name: configure AWS credentials (Connect and PubSub)
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ env.CI_PUBSUB_ROLE }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
+      - name: run Basic Connect sample
+        run: |
+          python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/basic_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key'
+      - name: run Websocket Connect sample
+        run: |
+          python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/websocket_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_arguments '--signing_region us-east-1'
+      - name: run PubSub sample
+        run: |
+          python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pubsub.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/key'
+      - name: run PKCS11 Connect sample
+        run: |
+          mkdir -p /tmp/tokens
+          export SOFTHSM2_CONF=/tmp/softhsm2.conf
+          echo "directories.tokendir = /tmp/tokens" > /tmp/softhsm2.conf
+          python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/pkcs11_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/PubSub/cert' --sample_secret_private_key 'ci/PubSub/keyp8' --sample_run_softhsm 'true' --sample_arguments '--pkcs11_lib /usr/lib/softhsm/libsofthsm2.so --pin 0000 --token_label my-token --key_label my-key'
+      - name: configure AWS credentials (Custom Authorizer)
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ env.CI_CUSTOM_AUTHORIZER_ROLE }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
+      - name: run CustomAuthorizerConnect sample
+        run: |
+          python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/custom_authorizer_connect.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_custom_authorizer_name 'ci/CustomAuthorizer/name' --sample_secret_custom_authorizer_password 'ci/CustomAuthorizer/password'
+      - name: configure AWS credentials (Shadow)
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ env.CI_SHADOW_ROLE }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
+      - name: run Shadow sample
+        run: |
+          python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/shadow.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/Shadow/cert' --sample_secret_private_key 'ci/Shadow/key' --sample_arguments '--thing_name CI_Shadow_Thing'
+      - name: configure AWS credentials (Jobs)
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ env.CI_JOBS_ROLE }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
+      - name: run Jobs sample
+        run: |
+          python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/jobs.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/Jobs/cert' --sample_secret_private_key 'ci/Jobs/key' --sample_arguments '--thing_name CI_Jobs_Thing'
+      - name: configure AWS credentials (Fleet provisioning)
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ env.CI_FLEET_PROVISIONING_ROLE }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
+      - name: run Fleet Provisioning sample
+        run: |
+          echo "Generating UUID for IoT thing"
+          Sample_UUID=$(python3  -c "import uuid; print (uuid.uuid4())")
+          python3 ${{ env.CI_UTILS_FOLDER }}/run_sample_ci.py --language Python --sample_file "${{ env.CI_SAMPLES_FOLDER }}/fleetprovisioning.py" --sample_region ${{ env.AWS_DEFAULT_REGION }} --sample_secret_endpoint 'ci/endpoint' --sample_secret_certificate 'ci/FleetProvisioning/cert' --sample_secret_private_key 'ci/FleetProvisioning/key' --sample_arguments "--template_name CI_FleetProvisioning_Template --template_parameters {\"SerialNumber\":\"${Sample_UUID}\"}"
+          python3 ${{ env.CI_UTILS_FOLDER }}/delete_iot_thing_ci.py --thing_name "Fleet_Thing_${Sample_UUID}" --region "us-east-1"
 
   # check that docs can still build
   check-docs:

diff --git a/README.md b/README.md
@@ -36,18 +36,27 @@ to Python by the `awscrt` package ([PyPI](https://pypi.org/project/awscrt/)) ([G
 [Step-by-step instructions](./documents/PREREQUISITES.md)
 
 ### Install from PyPI
+
+#### MacOS and Linux:
+
 ```
 python3 -m pip install awsiotsdk
 ```
 
+#### Windows:
+
+```
+python -m pip install awsiotsdk
+```
+
 ### Install from source
 ```
 # Create a workspace directory to hold all the SDK files
 mkdir sdk-workspace
 cd sdk-workspace
 # Clone the repository
 git clone https://github.com/aws/aws-iot-device-sdk-python-v2.git
-# Install using Pip
+# Install using Pip (use 'python' instead of 'python3' on Windows)
 python3 -m pip install ./aws-iot-device-sdk-python-v2
 ```
 

diff --git a/builder.json b/builder.json
@@ -89,14 +89,7 @@
         "python3 -m pip install ."
     ],
     "test_steps": [
-        "python3 -m pip install boto3",
-        "python3 deviceadvisor/script/DATestRun.py"
     ],
     "env": {
-        "DA_TOPIC": "test/da",
-        "DA_SHADOW_PROPERTY": "datest",
-        "DA_SHADOW_VALUE_SET": "ON",
-        "DA_SHADOW_VALUE_DEFAULT": "OFF",
-        "DA_S3_NAME": "aws-iot-sdk-deviceadvisor-logs"
     }
 }
diff --git a/codebuild/samples/connect-auth-linux.sh b/codebuild/samples/connect-auth-linux.sh
diff --git a/codebuild/samples/connect-linux.sh b/codebuild/samples/connect-linux.sh
diff --git a/codebuild/samples/linux-smoke-tests.yml b/codebuild/samples/linux-smoke-tests.yml
@@ -9,10 +9,7 @@ phases:
     commands:
       - echo Build started on `date`
       - $CODEBUILD_SRC_DIR/codebuild/samples/setup-linux.sh
-      - $CODEBUILD_SRC_DIR/codebuild/samples/connect-linux.sh
-      - $CODEBUILD_SRC_DIR/codebuild/samples/pkcs11-connect-linux.sh
       - $CODEBUILD_SRC_DIR/codebuild/samples/pubsub-linux.sh
-      - $CODEBUILD_SRC_DIR/codebuild/samples/connect-auth-linux.sh
   post_build:
     commands:
       - echo Build completed on `date`
diff --git a/codebuild/samples/pkcs11-connect-linux.sh b/codebuild/samples/pkcs11-connect-linux.sh
diff --git a/deviceadvisor/script/DATestConfig.json b/deviceadvisor/script/DATestConfig.json
@@ -2,11 +2,11 @@
     "tests" :["MQTT Connect", "MQTT Publish", "MQTT Subscribe", "Shadow Publish", "Shadow Update"],
     "test_suite_ids" :
     {
-        "MQTT Connect" : "ejbdzmo3hf3v",
-        "MQTT Publish" : "euw7favf6an4",
-        "MQTT Subscribe" : "01o8vo6no7sd",
-        "Shadow Publish" : "elztm2jebc1q",
-        "Shadow Update" : "vuydgrbbbfce"
+        "MQTT Connect" : "mxn32qkm8npn",
+        "MQTT Publish" : "gcjhujhhz50p",
+        "MQTT Subscribe" : "nyiuiwx5yxtj",
+        "Shadow Publish" : "fttdr8ufljnf",
+        "Shadow Update" : "ng9t8am2jnry"
     },
     "test_exe_path" :
     {