Skip to content

chore: Fix broken CI #441

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Nov 14, 2022
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 0 additions & 61 deletions .github/workflows/ci_tests.yaml
Original file line number Diff line number Diff line change
@@ -9,46 +9,6 @@ on:
- cron: '0 0 * * *'

jobs:
# Hypothesis no longer supports Python 2 and
# there is a bug that appears with our slow tests
# only on Python 2.
# Until we also drop Python 2 support,
# the workaround is just that we don't run the slow tests
# on Python 2.
py2-tests:
runs-on: ${{ matrix.platform.os }}
strategy:
fail-fast: true
matrix:
platform:
- os: ubuntu-latest
architecture: x64
- os: windows-latest
architecture: x64
# x86 builds are only meaningful for Windows
- os: windows-latest
architecture: x86
- os: macos-latest
architecture: x64
category:
- local-fast
# These require credentials.
# Enable them once we sort how to provide them.
# - integ-fast
# - examples
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: 2.7
architecture: ${{ matrix.platform.architecture }}
- run: |
python -m pip install --upgrade pip
pip install --upgrade -r ci-requirements.txt
- name: run test
env:
TOXENV: ${{ matrix.category }}
run: tox -- -vv
tests:
runs-on: ${{ matrix.platform.os }}
strategy:
@@ -65,8 +25,6 @@ jobs:
- os: macos-latest
architecture: x64
python:
- 3.5
- 3.6
- 3.7
- 3.8
- 3.x
@@ -110,22 +68,3 @@ jobs:
env:
TOXENV: ${{ matrix.category }}
run: tox -- -vv
upstream-py2:
runs-on: ubuntu-latest
strategy:
fail-fast: true
matrix:
category:
- test-upstream-requirements-py27
steps:
- uses: actions/checkout@v2
- uses: actions/setup-python@v1
with:
python-version: 2.7
- run: |
python -m pip install --upgrade pip
pip install --upgrade -r ci-requirements.txt
- name: run test
env:
TOXENV: ${{ matrix.category }}
run: tox -- -vv
6 changes: 0 additions & 6 deletions buildspec.yml
Original file line number Diff line number Diff line change
@@ -3,12 +3,6 @@ version: 0.2
batch:
fast-fail: false
build-list:
- identifier: python2_7
buildspec: codebuild/python2.7.yml
- identifier: python3_5
buildspec: codebuild/python3.5.yml
- identifier: python3_6
buildspec: codebuild/python3.6.yml
- identifier: python3_7
buildspec: codebuild/python3.7.yml
- identifier: python3_8
18 changes: 0 additions & 18 deletions codebuild/python2.7.yml

This file was deleted.

32 changes: 0 additions & 32 deletions codebuild/python3.5.yml

This file was deleted.

18 changes: 0 additions & 18 deletions codebuild/python3.6.yml

This file was deleted.

4 changes: 3 additions & 1 deletion codebuild/python3.8.yml
Original file line number Diff line number Diff line change
@@ -14,5 +14,7 @@ phases:
python: latest
build:
commands:
- pip install tox
- pyenv install 3.8.6
- pyenv local 3.8.6
- pip install tox tox-pyenv
- tox
4 changes: 2 additions & 2 deletions doc/conf.py
Original file line number Diff line number Diff line change
@@ -29,7 +29,7 @@ def get_version():
return _release


project = u"dynamodb-encryption-sdk-python"
project = "dynamodb-encryption-sdk-python"
version = get_version()
release = get_release()

@@ -53,7 +53,7 @@ def get_version():
source_suffix = ".rst" # The suffix of source filenames.
master_doc = "index" # The master toctree document.

copyright = u"%s, Amazon" % datetime.now().year # pylint: disable=redefined-builtin
copyright = "%s, Amazon" % datetime.now().year # pylint: disable=redefined-builtin

# List of directories, relative to source directory, that shouldn't be searched
# for source files.
4 changes: 2 additions & 2 deletions examples/src/aws_kms_encrypted_client.py
Original file line number Diff line number Diff line change
@@ -124,15 +124,15 @@ def encrypt_batch_items(table_name, aws_cmk_id):
def _select_index_from_item(item):
"""Find the index keys that match this item."""
for index in index_keys:
if all([item[key] == value for key, value in index.items()]):
if all(item[key] == value for key, value in index.items()):
return index

raise Exception("Index key not found in item.")

def _select_item_from_index(index, all_items):
"""Find the item that matches these index keys."""
for item in all_items:
if all([item[key] == value for key, value in index.items()]):
if all(item[key] == value for key, value in index.items()):
return item

raise Exception("Index key not found in item.")
4 changes: 2 additions & 2 deletions examples/src/aws_kms_encrypted_resource.py
Original file line number Diff line number Diff line change
@@ -76,15 +76,15 @@ def encrypt_batch_items(table_name, aws_cmk_id):
def _select_index_from_item(item):
"""Find the index keys that match this item."""
for index in index_keys:
if all([item[key] == value for key, value in index.items()]):
if all(item[key] == value for key, value in index.items()):
return index

raise Exception("Index key not found in item.")

def _select_item_from_index(index, all_items):
"""Find the item that matches these index keys."""
for item in all_items:
if all([item[key] == value for key, value in index.items()]):
if all(item[key] == value for key, value in index.items()):
return item

raise Exception("Index key not found in item.")
1 change: 1 addition & 0 deletions examples/src/pylintrc
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@
disable =
duplicate-code, # these examples often feature similar code
too-many-locals, # for these examples, we prioritize keeping everything together for simple readability
consider-using-f-string, # Not supported in Python 3.5

[BASIC]
# Allow function names up to 50 characters
1 change: 1 addition & 0 deletions examples/test/pylintrc
Original file line number Diff line number Diff line change
@@ -10,6 +10,7 @@ disable =
# pylint does not recognize this
duplicate-code, # tests for similar things tend to be similar
redefined-outer-name, # raises false positives with fixtures
consider-using-f-string, # Not supported in Python 3.5

[DESIGN]
max-args = 10
3 changes: 2 additions & 1 deletion src/dynamodb_encryption_sdk/delegated_keys/__init__.py
Original file line number Diff line number Diff line change
@@ -45,7 +45,8 @@ class DelegatedKey(object):
a :class:`NotImplementedError` detailing this.
"""

@abc.abstractproperty
@property
@abc.abstractmethod
def algorithm(self):
# type: () -> Text
"""Text description of algorithm used by this delegated key."""
4 changes: 2 additions & 2 deletions src/dynamodb_encryption_sdk/material_providers/most_recent.py
Original file line number Diff line number Diff line change
@@ -272,7 +272,7 @@ def _get_provider_with_grace_period(self, version, ttl_action):
:raises AttributeError: if provider could not locate version
"""
blocking_wait = bool(ttl_action is TtlActions.EXPIRED)
acquired = self._lock.acquire(blocking_wait)
acquired = self._lock.acquire(blocking_wait) # pylint: disable=consider-using-with
if not acquired:
# We failed to acquire the lock.
# If blocking, we will never reach this point.
@@ -310,7 +310,7 @@ def _get_most_recent_version(self, ttl_action):
:rtype: CryptographicMaterialsProvider
"""
blocking_wait = bool(ttl_action is TtlActions.EXPIRED)
acquired = self._lock.acquire(blocking_wait)
acquired = self._lock.acquire(blocking_wait) # pylint: disable=consider-using-with

if not acquired:
# We failed to acquire the lock.
15 changes: 10 additions & 5 deletions src/dynamodb_encryption_sdk/materials/__init__.py
Original file line number Diff line number Diff line change
@@ -33,7 +33,8 @@
class CryptographicMaterials(object):
"""Base class for all cryptographic materials."""

@abc.abstractproperty
@property
@abc.abstractmethod
def material_description(self):
# type: () -> Dict[Text, Text]
"""Material description to use with these cryptographic materials.
@@ -42,7 +43,8 @@ def material_description(self):
:rtype: dict
"""

@abc.abstractproperty
@property
@abc.abstractmethod
def encryption_key(self):
# type: () -> DelegatedKey
"""Delegated key used for encrypting attributes.
@@ -51,7 +53,8 @@ def encryption_key(self):
:rtype: DelegatedKey
"""

@abc.abstractproperty
@property
@abc.abstractmethod
def decryption_key(self):
# type: () -> DelegatedKey
"""Delegated key used for decrypting attributes.
@@ -60,7 +63,8 @@ def decryption_key(self):
:rtype: DelegatedKey
"""

@abc.abstractproperty
@property
@abc.abstractmethod
def signing_key(self):
# type: () -> DelegatedKey
"""Delegated key used for calculating digital signatures.
@@ -69,7 +73,8 @@ def signing_key(self):
:rtype: DelegatedKey
"""

@abc.abstractproperty
@property
@abc.abstractmethod
def verification_key(self):
# type: () -> DelegatedKey
"""Delegated key used for verifying digital signatures.
2 changes: 1 addition & 1 deletion src/pylintrc
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
[MESSAGES CONTROL]
# Disabling messages that we either don't care about for tests or are necessary to break for tests.
disable =
bad-continuation, # we let black handle this
ungrouped-imports, # we let isort handle this
duplicate-code, # causes lots of problems with implementations of common interfaces
# All below are disabled because we need to support Python 2
useless-object-inheritance,
raise-missing-from,
super-with-arguments,
consider-using-f-string,

[BASIC]
# Allow function names up to 50 characters
6 changes: 3 additions & 3 deletions test/acceptance/acceptance_test_generators.py
Original file line number Diff line number Diff line change
@@ -43,7 +43,7 @@ def load_scenarios(online):
into a shared method.
"""
# pylint: disable=too-many-locals
with open(_SCENARIO_FILE) as f:
with open(_SCENARIO_FILE, encoding="utf-8") as f:
scenarios = json.load(f)
keys_file = _filename_from_uri(scenarios["keys"])
keys = _load_keys(keys_file)
@@ -128,7 +128,7 @@ def _generate(materials_provider, table_data, ciphertext_file, metastore_info):
if table:
table.delete()

with open(ciphertext_file, "w") as outfile:
with open(ciphertext_file, "w", encoding="utf-8") as outfile:
json.dump(data_table_output, outfile, indent=4)

if metatable:
@@ -137,7 +137,7 @@ def _generate(materials_provider, table_data, ciphertext_file, metastore_info):
metastore_output[metastore_info["table_name"]].append(ddb_to_json(wrapping_key))

metastore_ciphertext_file = _filename_from_uri(metastore_info["ciphertext"])
with open(metastore_ciphertext_file, "w") as outfile:
with open(metastore_ciphertext_file, "w", encoding="utf-8") as outfile:
json.dump(metastore_output, outfile, indent=4)

metatable.delete()
10 changes: 5 additions & 5 deletions test/acceptance/acceptance_test_utils.py
Original file line number Diff line number Diff line change
@@ -61,7 +61,7 @@ def _decode_item(item):

def _build_plaintext_items(plaintext_file, version):
# pylint: disable=too-many-locals
with open(plaintext_file) as f:
with open(plaintext_file, encoding="utf-8") as f:
plaintext_data = json.load(f)

actions = {}
@@ -92,7 +92,7 @@ def _build_plaintext_items(plaintext_file, version):


def _load_ciphertext_items(ciphertext_file):
with open(ciphertext_file) as f:
with open(ciphertext_file, encoding="utf-8") as f:
ciphertexts = json.load(f)

for _table, items in ciphertexts.items():
@@ -103,7 +103,7 @@ def _load_ciphertext_items(ciphertext_file):


def _load_keys(keys_file):
with open(keys_file) as f:
with open(keys_file, encoding="utf-8") as f:
return json.load(f)


@@ -165,7 +165,7 @@ def _meta_table_prep(table_name, items_filename):
table = boto3.resource("dynamodb", region_name="us-west-2").Table(table_name)
table.wait_until_exists()
try:
with open(_filename_from_uri(items_filename)) as f:
with open(_filename_from_uri(items_filename), encoding="utf-8") as f:
table_data = json.load(f)
request_items = {}

@@ -255,7 +255,7 @@ def _expand_items(ciphertext_items, plaintext_items):

def load_scenarios(online):
# pylint: disable=too-many-locals
with open(_SCENARIO_FILE) as f:
with open(_SCENARIO_FILE, encoding="utf-8") as f:
scenarios = json.load(f)
keys_file = _filename_from_uri(scenarios["keys"])
keys = _load_keys(keys_file)
6 changes: 3 additions & 3 deletions test/functional/functional_test_vector_generators.py
Original file line number Diff line number Diff line change
@@ -104,14 +104,14 @@ def _decode_complex_value(_value):

def attribute_test_vectors(mode):
filepath = _ATTRIBUTE_TEST_VECTOR_FILE_TEMPLATE.format(mode=mode)
with open(filepath) as f:
with open(filepath, encoding="utf-8") as f:
vectors = json.load(f)
for vector in vectors:
yield (decode_value(vector["attribute"]), base64.b64decode(codecs.encode(vector["serialized"], "utf-8")))


def material_description_test_vectors():
with open(_MATERIAL_DESCRIPTION_TEST_VECTORS_FILE) as f:
with open(_MATERIAL_DESCRIPTION_TEST_VECTORS_FILE, encoding="utf-8") as f:
vectors = json.load(f)
for vector in vectors:
yield (vector["material_description"], decode_value({"B": codecs.encode(vector["serialized"], "utf-8")}))
@@ -125,7 +125,7 @@ def material_description_test_vectors():


def string_to_sign_test_vectors():
with open(_STRING_TO_SIGN_TEST_VECTORS_FILE) as f:
with open(_STRING_TO_SIGN_TEST_VECTORS_FILE, encoding="utf-8") as f:
vectors = json.load(f)
for vector in vectors:
item = {key: decode_value(value["value"]) for key, value in vector["item"].items()}
1 change: 1 addition & 0 deletions test/functional/hypothesis_strategies.py
Original file line number Diff line number Diff line change
@@ -23,6 +23,7 @@
hypothesis.HealthCheck.too_slow,
hypothesis.HealthCheck.data_too_large,
hypothesis.HealthCheck.large_base_example,
hypothesis.HealthCheck.function_scoped_fixture,
),
deadline=None,
)
8 changes: 4 additions & 4 deletions test/functional/internal/test_str_ops.py
Original file line number Diff line number Diff line change
@@ -26,8 +26,8 @@
(
("asdf", "asdf"),
(b"asdf", "asdf"),
(codecs.encode(u"Предисловие", "utf-8"), u"Предисловие"),
(u"Предисловие", u"Предисловие"),
(codecs.encode("Предисловие", "utf-8"), "Предисловие"),
("Предисловие", "Предисловие"),
),
)
def test_to_str(data, expected_output):
@@ -41,8 +41,8 @@ def test_to_str(data, expected_output):
("asdf", b"asdf"),
(b"asdf", b"asdf"),
(b"\x3a\x00\x99", b"\x3a\x00\x99"),
(u"Предисловие", codecs.encode(u"Предисловие", "utf-8")),
(codecs.encode(u"Предисловие", "utf-8"), codecs.encode(u"Предисловие", "utf-8")),
("Предисловие", codecs.encode("Предисловие", "utf-8")),
(codecs.encode("Предисловие", "utf-8"), codecs.encode("Предисловие", "utf-8")),
),
)
def test_to_bytes(data, expected_output):
2 changes: 2 additions & 0 deletions test/pylintrc
Original file line number Diff line number Diff line change
@@ -10,10 +10,12 @@ disable =
protected-access, # raised when calling _ methods
redefined-outer-name, # raised when using pytest-mock
unused-argument, # raised when patches and fixtures are needed but not called
no-self-use, # raised on Classes in tests used for logically grouping tests
# All below are disabled because we need to support Python 2
useless-object-inheritance,
raise-missing-from,
super-with-arguments,
consider-using-f-string,

[DESIGN]
max-args = 10
5 changes: 2 additions & 3 deletions test/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
hypothesis>=5.0.0,<6.0.0;python_version>='3'
hypothesis==4.57.1;python_version=='2.7'
hypothesis==6.31.6
mock
moto>=1.3.8
moto==3.0.2
pytest>=3.4.0
pytest-cov
pytest-mock
2 changes: 1 addition & 1 deletion test/unit/material_providers/test_aws_kms.py
Original file line number Diff line number Diff line change
@@ -225,7 +225,7 @@ def test_loaded_key_infos():
[
pytest.param(val, id=str(val))
for val in all_possible_combinations_kwargs(
dict(),
{},
dict(botocore_session=botocore.session.Session()),
dict(grant_tokens=("sdvoaweih", "auwshefiouawh")),
dict(material_description={"asoiufeoia": "soajfijewi"}),
107 changes: 42 additions & 65 deletions test/upstream-requirements-py37.txt
Original file line number Diff line number Diff line change
@@ -1,65 +1,42 @@
apipkg==1.5
asn1crypto==1.0.1
atomicwrites==1.3.0
attrs==19.2.0
aws-sam-translator==1.15.0
aws-xray-sdk==2.4.2
boto==2.49.0
boto3==1.9.246
botocore==1.12.246
certifi==2019.9.11
cffi==1.12.3
cfn-lint==0.24.4
chardet==3.0.4
coverage==4.5.4
cryptography==3.2
DateTime==4.3
docker==4.1.0
docutils==0.15.2
ecdsa==0.13.3
execnet==1.7.1
future==0.18.0
hypothesis==4.40.0
idna==2.8
importlib-metadata==0.23
Jinja2==2.10.3
jmespath==0.9.4
jsondiff==1.1.2
jsonpatch==1.24
jsonpickle==1.2
jsonpointer==2.0
jsonschema==3.1.1
MarkupSafe==1.1.1
mock==3.0.5
more-itertools==7.2.0
moto==1.3.13
packaging==19.2
pluggy==0.13.0
py==1.8.0
pyasn1==0.4.7
pycparser==2.19
pyparsing==2.4.2
pyrsistent==0.15.4
pytest==5.2.1
pytest-cov==2.8.1
pytest-forked==1.0.2
pytest-mock==1.11.1
pytest-xdist==1.30.0
python-dateutil==2.8.0
python-jose==3.0.1
pytz==2019.3
PyYAML==5.1.2
requests==2.22.0
responses==0.10.6
rsa==4.0
s3transfer==0.2.1
six==1.12.0
sshpubkeys==3.1.0
urllib3==1.25.6
wcwidth==0.1.7
websocket-client==0.56.0
Werkzeug==0.16.0
wrapt==1.11.2
xmltodict==0.12.0
zipp==0.6.0
zope.interface==4.6.0
attrs==22.1.0
boto3==1.26.7
botocore==1.29.7
certifi==2022.9.24
cffi==1.15.1
charset-normalizer==2.1.1
coverage==6.5.0
cryptography==38.0.3
exceptiongroup==1.0.1
execnet==1.9.0
hypothesis==6.56.4
idna==3.4
importlib-metadata==4.13.0
iniconfig==1.1.1
Jinja2==3.1.2
jmespath==1.0.1
MarkupSafe==2.1.1
mock==4.0.3
moto==3.0.2
packaging==21.3
pluggy==1.0.0
pycparser==2.21
pyparsing==3.0.9
pytest==7.2.0
pytest-cov==4.0.0
pytest-mock==3.10.0
pytest-xdist==3.0.2
python-dateutil==2.8.2
pytz==2022.6
requests==2.28.1
responses==0.22.0
s3transfer==0.6.0
six==1.16.0
sortedcontainers==2.4.0
toml==0.10.2
tomli==2.0.1
types-toml==0.10.8
typing_extensions==4.4.0
urllib3==1.26.12
Werkzeug==2.2.2
xmltodict==0.13.0
zipp==3.10.0
40 changes: 6 additions & 34 deletions tox.ini
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
[tox]
envlist =
py{27,35,36,37,38}-{local,integ,ddb,examples}-fast,
py{37,38}-{local,integ,ddb,examples}-fast,
nocmk, sourcebuildcheck,
docs, bandit, doc8, readme,
flake8{,-tests,-examples}, pylint{,-tests,-examples},
vulture,
test-upstream-requirements-py{2,3}7
test-upstream-requirements-py37

# Additional environments:
#
@@ -106,15 +106,6 @@ recreate = True
deps =
commands = {toxinidir}/test/freeze-upstream-requirements.sh

# Freeze for Python 2.7
[testenv:freeze-upstream-requirements-py27]
basepython = python2.7
sitepackages = {[testenv:freeze-upstream-requirements-base]sitepackages}
skip_install = {[testenv:freeze-upstream-requirements-base]skip_install}
recreate = {[testenv:freeze-upstream-requirements-base]recreate}
deps = {[testenv:freeze-upstream-requirements-base]deps}
commands = {[testenv:freeze-upstream-requirements-base]commands} test/upstream-requirements-py27.txt

# Freeze for Python 3.7
[testenv:freeze-upstream-requirements-py37]
basepython = python3.7
@@ -131,15 +122,6 @@ recreate = True
passenv =
commands = {[testenv:base-command]commands} -m "local and not slow and not veryslow and not nope"

# Test frozen upstream requirements for Python 2.7
[testenv:test-upstream-requirements-py27]
basepython = python2.7
passenv =
deps = -rtest/upstream-requirements-py27.txt
sitepackages = {[testenv:test-upstream-requirements-base]sitepackages}
recreate = {[testenv:test-upstream-requirements-base]recreate}
commands = {[testenv:test-upstream-requirements-base]commands}

# Test frozen upstream requirements for Python 3.7
[testenv:test-upstream-requirements-py37]
basepython = python3.7
@@ -190,17 +172,6 @@ commands =
{posargs}
{[testenv:mypy-coverage]commands}

[testenv:mypy-py2]
basepython = python2.7
deps = {[testenv:mypy-common]deps}
commands =
python -m mypy \
--py2 \
--linecoverage-report build \
src/dynamodb_encryption_sdk/ \
{posargs}
{[testenv:mypy-coverage]commands}

# Linters
[testenv:flake8]
basepython = python3
@@ -250,8 +221,8 @@ commands =
basepython = python3
deps =
{[testenv]deps}
pyflakes
pylint
pyflakes==2.4.0
pylint==2.12.2
commands =
pylint \
--rcfile=src/pylintrc \
@@ -262,6 +233,7 @@ commands =
[testenv:pylint-tests]
basepython = {[testenv:pylint]basepython}
deps = {[testenv:pylint]deps}
moto==3.0.2
commands =
pylint \
--rcfile=test/pylintrc \
@@ -362,7 +334,7 @@ commands =

[testenv:readme]
basepython = python3
deps = readme_renderer
deps = readme_renderer==34.0
commands = python setup.py check -r -s

[testenv:bandit]