chore: fix all CI and start migration to GitHub Actions (#139)

* chore: force tox to update pip

* chore: update isort configuration to 5.0.0

* chore: autoformat

* chore: hypothesis.HealthCheck.hung_test is deprecated

* chore: flake8 linting

* chore: update pylint and flake8 configs

* chore: linting fixes

* chore: update default Python envlist
- remove 3.4
- add 3.8

* chore: Python 2 lists do not have copy()

* chore: address all pylint issues aside from TODO references

* chore: unlock mypy version

* chore: fix type annotation syntax errors

* chore: move TODOs into GitHub issues

* chore: move test TODOs to GitHub issues

* chore: autoformat

* chore: rework moto use
- fixes issues with multiple service mocks in Python 2
- module scope avoids resetting the mocked service for tests that use multiple mocked tables

* chore: force nocmk environment to black all environment variables

* chore: add GitHub Actions workflows

* chore: move sourcebuildcheck and nocmk into upstream-py3 job

* chore: add ci-requirements.txt

* chore: work around bug in Python 2 Hypothesis behavior by only running fast tests for Python 2

* chore: fix sourcebuildcheck script
- The ls command was getting a relative path when the script ran but a bare filename in manual checks.

* chore: pruning known runs from Travis that fail due to known infrastructure issues

Author: mattsb42-aws

.github/workflows/ci_static-analysis.yaml

@@ -0,0 +1,44 @@
+# This workflow runs static analysis checks on pull requests.
+name: static analysis
+
+on:
+  pull_request:
+  push:
+  # Run once a day
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  analysis:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        category:
+# Disabled pending completion of integration
+# https://github.com/aws/aws-dynamodb-encryption-python/issues/66
+#          - mypy-py2
+#          - mypy-py3
+          - bandit
+          - doc8
+          - readme
+          - docs
+          - flake8
+          - pylint
+          - flake8-tests
+          - flake8-examples
+          - pylint-tests
+          - pylint-examples
+          - black-check
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 3.x
+      - run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade -r ci-requirements.txt
+      - name: check
+        env:
+          TOXENV: ${{ matrix.category }}
+        run: tox -- -vv

.github/workflows/ci_tests.yaml

@@ -0,0 +1,131 @@
+# This workflow runs tests on pull requests.
+name: tests
+
+on:
+  pull_request:
+  push:
+  # Run once a day
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  # Hypothesis no longer supports Python 2 and
+  # there is a bug that appears with our slow tests
+  # only on Python 2.
+  # Until we also drop Python 2 support,
+  # the workaround is just that we don't run the slow tests
+  # on Python 2.
+  py2-tests:
+    runs-on: ${{ matrix.platform.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        platform:
+          - os: ubuntu-latest
+            architecture: x64
+          - os: windows-latest
+            architecture: x64
+          # x86 builds are only meaningful for Windows
+          - os: windows-latest
+            architecture: x86
+          - os: macos-latest
+            architecture: x64
+        category:
+          - local-fast
+    # These require credentials.
+    # Enable them once we sort how to provide them.
+    #          - integ-fast
+    #          - examples
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 2.7
+          architecture: ${{ matrix.platform.architecture }}
+      - run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade -r ci-requirements.txt
+      - name: run test
+        env:
+          TOXENV: ${{ matrix.category }}
+        run: tox -- -vv
+  tests:
+    runs-on: ${{ matrix.platform.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        platform:
+          - os: ubuntu-latest
+            architecture: x64
+          - os: windows-latest
+            architecture: x64
+          # x86 builds are only meaningful for Windows
+          - os: windows-latest
+            architecture: x86
+          - os: macos-latest
+            architecture: x64
+        python:
+          - 3.5
+          - 3.6
+          - 3.7
+          - 3.8
+          - 3.x
+        category:
+          - local-slow
+# These require credentials.
+# Enable them once we sort how to provide them.
+#          - integ-slow
+#          - examples
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python }}
+          architecture: ${{ matrix.platform.architecture }}
+      - run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade -r ci-requirements.txt
+      - name: run test
+        env:
+          TOXENV: ${{ matrix.category }}
+        run: tox -- -vv
+  upstream-py3:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        category:
+          - nocmk
+          - sourcebuildcheck
+          - test-upstream-requirements-py37
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade -r ci-requirements.txt
+      - name: run test
+        env:
+          TOXENV: ${{ matrix.category }}
+        run: tox -- -vv
+  upstream-py2:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        category:
+          - test-upstream-requirements-py27
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 2.7
+      - run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade -r ci-requirements.txt
+      - name: run test
+        env:
+          TOXENV: ${{ matrix.category }}
+        run: tox -- -vv

.travis.yml

@@ -2,20 +2,17 @@ sudo: false
 language: python
 matrix:
   include:
+    # Hypothesis no longer supports Python 2 and
+    # there is a bug that appears with our slow tests
+    # only on Python 2.
+    # Until we also drop Python 2 support,
+    # the workaround is just that we don't run the slow tests
+    # on Python 2.
     # CPython 2.7
     - python: 2.7
-      env: TOXENV=py27-travis-local-slow
+      env: TOXENV=py27-travis-local-fast
     - python: 2.7
       env: TOXENV=py27-travis-integ-slow
-    - python: 2.7
-      env: TOXENV=py27-travis-isolation
-    # CPython 3.4
-    - python: 3.4
-      env: TOXENV=py34-travis-local-slow
-    - python: 3.4
-      env: TOXENV=py34-travis-integ-slow
-    - python: 3.4
-      env: TOXENV=py34-travis-isolation
     # CPython 3.5
     - python: 3.5
       env: TOXENV=py35-travis-local-slow

ci-requirements.txt

@@ -0,0 +1 @@
+tox

setup.cfg

@@ -32,7 +32,7 @@ log_level=DEBUG
 
 # Flake8 Configuration
 [flake8]
-max_complexity = 10
+max_complexity = 11
 max_line_length = 120
 import_order_style = google
 application_import_names = dynamodb_encryption_sdk
@@ -61,6 +61,5 @@ multi_line_output = 3
 include_trailing_comma = True
 force_grid_wrap = 0
 combine_as_imports = True
-not_skip = __init__.py
 known_first_party = dynamodb_encryption_sdk
 known_third_party =attr,aws_kms_encrypted_client,aws_kms_encrypted_item,aws_kms_encrypted_resource,aws_kms_encrypted_table,boto3,botocore,cryptography,dynamodb_encryption_sdk,functional_test_utils,functional_test_vector_generators,hypothesis,hypothesis_strategies,integration_test_utils,mock,most_recent_provider_encrypted_table,moto,mypy_extensions,pytest,pytest_mock,setuptools,six,wrapped_rsa_encrypted_table,wrapped_symmetric_encrypted_table

setup.py

@@ -23,7 +23,7 @@ def get_version():
 def get_requirements():
     """Reads the requirements file."""
     requirements = read("requirements.txt")
-    return [r for r in requirements.strip().splitlines()]
+    return requirements.strip().splitlines()
 
 
 setup(

src/dynamodb_encryption_sdk/encrypted/client.py

@@ -96,7 +96,8 @@ def __getattr__(self, name):
 
     def paginate(self, **kwargs):
         # type: (**Any) -> Iterator[Dict]
-        # TODO: narrow this down
+        # narrow this down
+        # https://github.com/aws/aws-dynamodb-encryption-python/issues/66
         """Create an iterator that will paginate through responses from the underlying paginator,
         transparently decrypting any returned items.
         """

src/dynamodb_encryption_sdk/encrypted/item.py

@@ -171,7 +171,7 @@ def decrypt_dynamodb_item(item, crypto_config):
     :rtype: dict
     """
     unique_actions = set([crypto_config.attribute_actions.default_action.name])
-    unique_actions.update(set([action.name for action in crypto_config.attribute_actions.attribute_actions.values()]))
+    unique_actions.update({action.name for action in crypto_config.attribute_actions.attribute_actions.values()})
 
     if crypto_config.attribute_actions.take_no_actions:
         # If we explicitly have been told not to do anything to this item, just copy it.

src/dynamodb_encryption_sdk/encrypted/resource.py

@@ -41,7 +41,7 @@
 
 @attr.s(init=False)
 class EncryptedTablesCollectionManager(object):
-    # pylint: disable=too-few-public-methods
+    # pylint: disable=too-few-public-methods,too-many-instance-attributes
     """Tables collection manager that provides :class:`EncryptedTable` objects.
 
     https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#DynamoDB.ServiceResource.tables
@@ -119,7 +119,7 @@ def _transform_table(self, method, **kwargs):
 
 @attr.s(init=False)
 class EncryptedResource(object):
-    # pylint: disable=too-few-public-methods
+    # pylint: disable=too-few-public-methods,too-many-instance-attributes
     """High-level helper class to provide a familiar interface to encrypted tables.
 
     >>> import boto3

src/dynamodb_encryption_sdk/encrypted/table.py

@@ -42,7 +42,7 @@
 
 @attr.s(init=False)
 class EncryptedTable(object):
-    # pylint: disable=too-few-public-methods
+    # pylint: disable=too-few-public-methods,too-many-instance-attributes
     """High-level helper class to provide a familiar interface to encrypted tables.
 
     >>> import boto3

src/dynamodb_encryption_sdk/identifiers.py

@@ -35,12 +35,12 @@ def __gt__(self, other):
     def __lt__(self, other):
         # type: (CryptoAction) -> bool
         """Define CryptoAction equality."""
-        return self.value < other.value
+        return self.value < other.value  # pylint: disable=comparison-with-callable
 
     def __eq__(self, other):
         # type: (CryptoAction) -> bool
         """Define CryptoAction equality."""
-        return self.value == other.value
+        return self.value == other.value  # pylint: disable=comparison-with-callable
 
 
 class EncryptionKeyType(Enum):

src/dynamodb_encryption_sdk/internal/crypto/authentication.py

@@ -28,6 +28,7 @@
 
 try:  # Python 3.5.0 and 3.5.1 have incompatible typing modules
     from typing import Text  # noqa pylint: disable=unused-import
+
     from dynamodb_encryption_sdk.internal import dynamodb_types  # noqa pylint: disable=unused-import
 except ImportError:  # pragma: no cover
     # We only actually need these imports when running the mypy checks
@@ -55,7 +56,8 @@ def sign_item(encrypted_item, signing_key, crypto_config):
             attribute_actions=crypto_config.attribute_actions,
         ),
     )
-    return {Tag.BINARY.dynamodb_tag: signature}
+    # for some reason pylint can't follow the Enum member attributes
+    return {Tag.BINARY.dynamodb_tag: signature}  # pylint: disable=no-member
 
 
 def verify_item_signature(signature_attribute, encrypted_item, verification_key, crypto_config):
@@ -67,7 +69,8 @@ def verify_item_signature(signature_attribute, encrypted_item, verification_key,
     :param DelegatedKey verification_key: DelegatedKey to use to calculate the signature
     :param CryptoConfig crypto_config: Cryptographic configuration
     """
-    signature = signature_attribute[Tag.BINARY.dynamodb_tag]
+    # for some reason pylint can't follow the Enum member attributes
+    signature = signature_attribute[Tag.BINARY.dynamodb_tag]  # pylint: disable=no-member
     verification_key.verify(
         algorithm=verification_key.algorithm,
         signature=signature,
@@ -97,10 +100,11 @@ def _string_to_sign(item, table_name, attribute_actions):
 
         data_to_sign.extend(_hash_data(hasher=hasher, data=key.encode(TEXT_ENCODING)))
 
+        # for some reason pylint can't follow the Enum member attributes
         if action is CryptoAction.SIGN_ONLY:
-            data_to_sign.extend(SignatureValues.PLAINTEXT.sha256)
+            data_to_sign.extend(SignatureValues.PLAINTEXT.sha256)  # pylint: disable=no-member
         else:
-            data_to_sign.extend(SignatureValues.ENCRYPTED.sha256)
+            data_to_sign.extend(SignatureValues.ENCRYPTED.sha256)  # pylint: disable=no-member
 
         data_to_sign.extend(_hash_data(hasher=hasher, data=serialize_attribute(item[key])))
     return bytes(data_to_sign)

src/dynamodb_encryption_sdk/internal/crypto/encryption.py

@@ -18,6 +18,7 @@
 """
 try:  # Python 3.5.0 and 3.5.1 have incompatible typing modules
     from typing import Text  # noqa pylint: disable=unused-import
+
     from dynamodb_encryption_sdk.internal import dynamodb_types  # noqa pylint: disable=unused-import
 except ImportError:  # pragma: no cover
     # We only actually need these imports when running the mypy checks
@@ -46,7 +47,8 @@ def encrypt_attribute(attribute_name, attribute, encryption_key, algorithm):
     encrypted_attribute = encryption_key.encrypt(
         algorithm=algorithm, name=attribute_name, plaintext=serialized_attribute
     )
-    return {Tag.BINARY.dynamodb_tag: encrypted_attribute}
+    # for some reason pylint can't follow the Enum member attributes
+    return {Tag.BINARY.dynamodb_tag: encrypted_attribute}  # pylint: disable=no-member
 
 
 def decrypt_attribute(attribute_name, attribute, decryption_key, algorithm):
@@ -60,7 +62,8 @@ def decrypt_attribute(attribute_name, attribute, decryption_key, algorithm):
     :returns: Plaintext DynamoDB attribute
     :rtype: dict
     """
-    encrypted_attribute = attribute[Tag.BINARY.dynamodb_tag]
+    # for some reason pylint can't follow the Enum member attributes
+    encrypted_attribute = attribute[Tag.BINARY.dynamodb_tag]  # pylint: disable=no-member
     decrypted_attribute = decryption_key.decrypt(
         algorithm=algorithm, name=attribute_name, ciphertext=encrypted_attribute
     )