New CD workflow

.github/workflows/release.yml

@@ -0,0 +1,58 @@
+name: Release
+
+env:
+  GH_TOKEN: ${{ github.token }}
+
+on:
+  workflow_dispatch:
+    inputs:
+      release_type:
+        type: choice
+        description: "Release type:"
+        required: true
+        options:
+          - bug fix (PATCH)
+          - new feature (MINOR)
+      release_title:
+        description: "The title of the release"
+        required: true
+
+jobs:
+  update-version:
+    runs-on: ubuntu-20.04 # latest
+    permissions:
+      contents: write # allow push
+      pull-requests: write # allow making PR
+
+    steps:
+    - name: Checkout Sources
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Make new release
+      env:
+        Title: ${{ github.event.inputs.release_title }}
+      run: |
+        # Escape special characters
+        Title=$(echo ${Title//[\"]\\\"})
+        Title=$(echo ${Title//[\']\\\'})
+        Title=$(echo ${Title//[\$]})
+
+        ./utils/publish-release.sh "${{ github.event.inputs.release_type }}" "$Title"
+
+    - name: configure AWS credentials (Release)
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        role-to-assume: ${{ secrets.AWS_CI_RELEASE_ROLE }}
+        aws-region: us-east-1
+
+    - name: "Create VERSION file and trigger release"
+      run: |
+        version=$(git describe --tags --abbrev=0)
+        version_without_v=$(echo ${version} | cut -f2 -dv)
+        echo "${version_without_v}" > VERSION
+
+        zip VERSION.zip VERSION
+        export S3_URL=$(aws secretsmanager get-secret-value --secret-id ci/python_v2_version --query "SecretString" | cut -f2 -d\")
+        aws s3 cp VERSION.zip $S3_URL

README.md

@@ -99,3 +99,5 @@ We need your help in making this SDK great. Please participate in the community
 ## License
 
 This library is licensed under the [Apache 2.0 License](./documents/LICENSE).
+
+Latest released version: v1.12.0

continuous-delivery/publish_to_prod_pypi.yml → codebuild/cd/publish-to-prod-pypi.yml

@@ -1,5 +1,9 @@
+# Assumes are running using the Ubuntu Codebuild standard image
+# Makes a release deployment to production PyPi repo.
+#
+# NOTE: This script assumes that the AWS CLI-V2 is pre-installed!
+# - AWS CLI-V2 is a requirement to run this script.
 version: 0.2
-# this image assumes Ubuntu base image
 phases:
   install:
     commands:
@@ -8,11 +12,13 @@ phases:
       - export PATH=$PATH:$HOME/.local/bin
       - python3 -m pip install --user --upgrade pip
       - python3 -m pip install --user --upgrade twine setuptools wheel awscli PyOpenSSL six
+      - echo "\nBuild version data:"
+      - echo "\nPython Version:"; python3 --version
   pre_build:
     commands:
-      - cd aws-iot-device-sdk-python-v2
-      - pypirc=$(aws secretsmanager get-secret-value --secret-id "prod/aws-sdk-python-v2/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc
-      - export PKG_VERSION=$(git describe --tags | cut -f2 -dv)
+      - cd $CODEBUILD_SRC_DIR/aws-iot-device-sdk-python-v2
+      - pypirc=$(aws secretsmanager get-secret-value --secret-id "cd/aws-sdk-python-v2-prod/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc
+      - export PKG_VERSION=$(cat $CODEBUILD_SRC_DIR/VERSION)
       - echo "Updating package version to ${PKG_VERSION}"
       - sed --in-place -E "s/__version__ = '.+'/__version__ = '${PKG_VERSION}'/" awsiot/__init__.py
   build:

continuous-delivery/publish_to_test_pypi.yml → codebuild/cd/publish-to-test-pypi.yml

@@ -1,5 +1,9 @@
+# Assumes are running using the Ubuntu Codebuild standard image
+# Makes a release deployment to test PyPi repo.
+#
+# NOTE: This script assumes that the AWS CLI-V2 is pre-installed!
+# - AWS CLI-V2 is a requirement to run this script.
 version: 0.2
-# this image assumes Ubuntu base image
 phases:
   install:
     commands:
@@ -8,11 +12,13 @@ phases:
       - export PATH=$PATH:$HOME/.local/bin
       - python3 -m pip install --user --upgrade pip
       - python3 -m pip install --user --upgrade twine setuptools wheel awscli PyOpenSSL six
+      - echo "\nBuild version data:"
+      - echo "\nPython Version:"; python3 --version
   pre_build:
     commands:
-      - pypirc=$(aws secretsmanager get-secret-value --secret-id "alpha/aws-crt-python/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc
-      - cd aws-iot-device-sdk-python-v2
-      - export PKG_VERSION=$(git describe --tags | cut -f2 -dv)
+      - cd $CODEBUILD_SRC_DIR/aws-iot-device-sdk-python-v2
+      - pypirc=$(aws secretsmanager get-secret-value --secret-id "cd/aws-sdk-python-v2-test/.pypirc" --query "SecretString" | cut -f2 -d\") && echo "$pypirc" > ~/.pypirc
+      - export PKG_VERSION=$(cat $CODEBUILD_SRC_DIR/VERSION)
       - echo "Updating package version to ${PKG_VERSION}"
       - sed --in-place -E "s/__version__ = '.+'/__version__ = '${PKG_VERSION}'/" awsiot/__init__.py
   build:

codebuild/cd/test-prod-pypi.yml

@@ -0,0 +1,36 @@
+# Assumes are running using the Ubuntu Codebuild standard image
+# Makes sure the release to test PyPi repository worked
+#
+# NOTE: This script assumes that the AWS CLI-V2 is pre-installed!
+# - AWS CLI-V2 is a requirement to run this script.
+version: 0.2
+phases:
+  install:
+    commands:
+      - sudo apt-get update -y
+      - sudo apt-get install python3 python3-pip -y
+      - python3 -m pip install --upgrade pip
+      - python3 -m pip install --upgrade setuptools
+      - echo "\nBuild version data:"
+      - echo "\nPython Version:"; python3 --version
+
+  pre_build:
+    commands:
+      # Material for PubSub sample
+      - curl https://www.amazontrust.com/repository/AmazonRootCA1.pem --output /tmp/AmazonRootCA1.pem
+      - cert=$(aws secretsmanager get-secret-value --secret-id "ci/PubSub/cert" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$cert" > /tmp/certificate.pem
+      - key=$(aws secretsmanager get-secret-value --secret-id "ci/PubSub/key" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$key" > /tmp/privatekey.pem
+      - ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "ci/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
+  build:
+    commands:
+      - echo Build started on `date`
+      - cd $CODEBUILD_SRC_DIR/aws-iot-device-sdk-python-v2
+      - CURRENT_TAG_VERSION=$(cat $CODEBUILD_SRC_DIR/VERSION)
+      - python3 codebuild/cd/pip-install-with-retry.py --no-cache-dir --user awsiotsdk==$CURRENT_TAG_VERSION
+      # Run PubSub sample
+      - python3 samples/pubsub.py --endpoint ${ENDPOINT} --cert /tmp/certificate.pem --key /tmp/privatekey.pem --ca_file /tmp/AmazonRootCA1.pem --verbosity Trace
+
+  post_build:
+    commands:
+      - echo Build completed on `date`
+

codebuild/cd/test-test-pypi.yml

@@ -0,0 +1,36 @@
+# Assumes are running using the Ubuntu Codebuild standard image
+# Makes sure the release to test PyPi repository worked
+#
+# NOTE: This script assumes that the AWS CLI-V2 is pre-installed!
+# - AWS CLI-V2 is a requirement to run this script.
+version: 0.2
+phases:
+  install:
+    commands:
+      - sudo apt-get update -y
+      - sudo apt-get install python3 python3-pip -y
+      - python3 -m pip install --upgrade pip
+      - python3 -m pip install --upgrade setuptools
+      - echo "\nBuild version data:"
+      - echo "\nPython Version:"; python3 --version
+
+  pre_build:
+    commands:
+      - curl https://www.amazontrust.com/repository/AmazonRootCA1.pem --output /tmp/AmazonRootCA1.pem
+      - cert=$(aws secretsmanager get-secret-value --secret-id "ci/PubSub/cert" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$cert" > /tmp/certificate.pem
+      - key=$(aws secretsmanager get-secret-value --secret-id "ci/PubSub/key" --query "SecretString" | cut -f2 -d":" | cut -f2 -d\") && echo "$key" > /tmp/privatekey.pem
+      - ENDPOINT=$(aws secretsmanager get-secret-value --secret-id "ci/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
+  build:
+    commands:
+      - echo Build started on `date`
+      - cd $CODEBUILD_SRC_DIR/aws-iot-device-sdk-python-v2
+      - CURRENT_TAG_VERSION=$(cat $CODEBUILD_SRC_DIR/VERSION)
+      # this is here because typing isn't in test-pypi, so pull it from prod instead
+      - python3 -m pip install typing
+      - python3 codebuild/cd/pip-install-with-retry.py -i https://testpypi.python.org/simple --user awsiotsdk==$CURRENT_TAG_VERSION
+      # Run PubSub sample
+      - python3 samples/pubsub.py --endpoint ${ENDPOINT} --cert /tmp/certificate.pem --key /tmp/privatekey.pem --ca_file /tmp/AmazonRootCA1.pem --verbosity Trace
+
+  post_build:
+    commands:
+      - echo Build completed on `date`

codebuild/cd/test-version-exists.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+VERSION_FILE_PATH=$1
+if [ ! readlink -e "$VERSION_FILE_PATH" ]; then
+    echo "No VERSION file found! Cannot make release!"
+    exit 1
+else
+    echo "VERSION file found..."
+fi
+VERSION=$(cat $VERSION_FILE_PATH)
+
+# Make sure the version variable is populated
+if [ -z "${VERSION}" ]; then
+    echo "VERSION file is empty!"
+    exit 1
+else
+    echo "VERSION file contains: ${VERSION}"
+fi
+
+# Make sure the version follows the correct format: major.minor.patch
+LENGTH_CHECK="${VERSION//[^.]}"
+if [ ${#LENGTH_CHECK} != 2 ]; then
+    echo "VERSION file contains invalid version (not in format major.minor.patch)"
+    exit 1
+fi
+# Use RegX to ensure it only contains numbers and periods
+REGX_CHECK='^([0-9]+\.){0,2}(\*|[0-9]+)$'
+if [[ $VERSION =~ $REGX_CHECK ]]; then
+    echo "VERSION file contains valid version"
+else
+    echo "VERSION file contains invalid version (RegX validator failed)"
+    exit 1
+fi
+
+# Does PyPi have the version? If so, do not allow it!
+if python3 -m pip install --no-cache-dir -vvv awsiotsdk==$VERSION; then
+    echo "$VERSION is already in pypi, cut a new tag if you want to upload another version."
+    exit 1
+fi
+
+echo "$VERSION currently does not exist in pypi, allowing pipeline to continue."
+exit 0

codebuild/cd/test-version-exists.yml

@@ -0,0 +1,19 @@
+# Assumes are running using the Ubuntu Codebuild standard image
+# Makes sure the version in CD has not already been released.
+# Will fail the build and stop the pipeline if the version has already been released.
+#
+# NOTE: This script assumes that the AWS CLI-V2 is pre-installed!
+# - AWS CLI-V2 is a requirement to run this script.
+version: 0.2
+phases:
+  install:
+    commands:
+      - sudo apt-get update -y
+      - sudo apt-get install python3 python3-pip -y
+      - pip3 install --upgrade setuptools
+      - echo "\nBuild version data:"
+      - echo "\nPython Version:"; python3 --version
+  build:
+    commands:
+      - cd $CODEBUILD_SRC_DIR/aws-iot-device-sdk-python-v2
+      - bash ./codebuild/cd/test-version-exists.sh $CODEBUILD_SRC_DIR/VERSION