diff --git a/MANIFEST.in b/MANIFEST.in index 3cf0e93c89..c2110afafb 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,7 @@ recursive-include src/sagemaker *.py include src/sagemaker/image_uri_config/*.json +recursive-include requirements * include VERSION include LICENSE.txt diff --git a/requirements/extras/local_requirements.txt b/requirements/extras/local_requirements.txt new file mode 100644 index 0000000000..17512c3388 --- /dev/null +++ b/requirements/extras/local_requirements.txt @@ -0,0 +1,4 @@ +urllib3==1.26.8 +docker-compose==1.29.2 +docker~=5.0.0 +PyYAML==5.4.1 diff --git a/requirements/extras/scipy_requirements.txt b/requirements/extras/scipy_requirements.txt new file mode 100644 index 0000000000..9136ba9f03 --- /dev/null +++ b/requirements/extras/scipy_requirements.txt @@ -0,0 +1 @@ +scipy==1.5.4 diff --git a/requirements/extras/test_requirements.txt b/requirements/extras/test_requirements.txt new file mode 100644 index 0000000000..b002686404 --- /dev/null +++ b/requirements/extras/test_requirements.txt @@ -0,0 +1,20 @@ +tox==3.24.5 +flake8==4.0.1 +pytest==6.0.2 +pytest-cov==3.0.0 +pytest-rerunfailures==10.2 +pytest-timeout==2.1.0 +pytest-xdist==2.4.0 +coverage>=5.2, <6.2 +mock==4.0.3 +contextlib2==21.6.0 +awslogs==0.14.0 +black==22.3.0 +stopit==1.1.2 +apache-airflow==2.2.4 +apache-airflow-providers-amazon==3.0.0 +attrs==20.3.0 +fabric==2.6.0 +requests==2.27.1 +sagemaker-experiments==0.1.35 +Jinja2==3.0.3 diff --git a/requirements/tox/doc8_requirements.txt b/requirements/tox/doc8_requirements.txt new file mode 100644 index 0000000000..385e5257aa --- /dev/null +++ b/requirements/tox/doc8_requirements.txt @@ -0,0 +1,2 @@ +doc8==0.10.1 +Pygments==2.11.2 diff --git a/requirements/tox/docstyle_requirements.txt b/requirements/tox/docstyle_requirements.txt new file mode 100644 index 0000000000..eae14f47f7 --- /dev/null +++ b/requirements/tox/docstyle_requirements.txt @@ -0,0 +1 @@ +pydocstyle==6.1.1 diff --git a/requirements/tox/flake8_requirements.txt b/requirements/tox/flake8_requirements.txt new file mode 100644 index 0000000000..b3ccfca84f --- /dev/null +++ b/requirements/tox/flake8_requirements.txt @@ -0,0 +1,2 @@ +flake8==4.0.1 +flake8-future-import==0.4.6 diff --git a/requirements/tox/mypy_requirements.txt b/requirements/tox/mypy_requirements.txt new file mode 100644 index 0000000000..2f3b421b72 --- /dev/null +++ b/requirements/tox/mypy_requirements.txt @@ -0,0 +1 @@ +mypy==0.942 diff --git a/requirements/tox/pydocstyle_requirements.txt b/requirements/tox/pydocstyle_requirements.txt new file mode 100644 index 0000000000..eae14f47f7 --- /dev/null +++ b/requirements/tox/pydocstyle_requirements.txt @@ -0,0 +1 @@ +pydocstyle==6.1.1 diff --git a/requirements/tox/pylint_requirements.txt b/requirements/tox/pylint_requirements.txt new file mode 100644 index 0000000000..b307f21762 --- /dev/null +++ b/requirements/tox/pylint_requirements.txt @@ -0,0 +1,2 @@ +pylint==2.6.2 +astroid==2.4.2 diff --git a/requirements/tox/spelling_requirements.txt b/requirements/tox/spelling_requirements.txt new file mode 100644 index 0000000000..769415eb2c --- /dev/null +++ b/requirements/tox/spelling_requirements.txt @@ -0,0 +1,2 @@ +pyenchant==3.2.2 +pylint==2.6.2 diff --git a/requirements/tox/twine_requirements.txt b/requirements/tox/twine_requirements.txt new file mode 100644 index 0000000000..f9d2c8fdce --- /dev/null +++ b/requirements/tox/twine_requirements.txt @@ -0,0 +1 @@ +twine==3.8.0 diff --git a/setup.py b/setup.py index 4680f18125..140f671763 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,20 @@ def read_version(): return read("VERSION").strip() +def read_requirements(filename): + """Reads requirements file which lists package dependencies. + + Args: + filename: type(str) Relative file path of requirements.txt file + + Returns: + list of dependencies extracted from file + """ + with open(os.path.abspath(filename)) as fp: + deps = [line.strip() for line in fp.readlines()] + return deps + + # Declare minimal set for installation required_packages = [ "attrs==20.3.0", @@ -47,43 +61,15 @@ def read_version(): ] # Specific use case dependencies +# Keep format of *_requirements.txt to be tracked by dependabot extras = { - "local": [ - "urllib3==1.26.8", - "docker-compose==1.29.2", - "docker~=5.0.0", - "PyYAML==5.4.1", # PyYAML version has to match docker-compose requirements - ], - "scipy": ["scipy==1.5.4"], + "local": read_requirements("requirements/extras/local_requirements.txt"), + "scipy": read_requirements("requirements/extras/scipy_requirements.txt"), } # Meta dependency groups extras["all"] = [item for group in extras.values() for item in group] # Tests specific dependencies (do not need to be included in 'all') -extras["test"] = ( - [ - extras["all"], - "tox==3.24.5", - "flake8==4.0.1", - "pytest==6.0.2", - "pytest-cov==3.0.0", - "pytest-rerunfailures==10.2", - "pytest-timeout==2.1.0", - "pytest-xdist==2.4.0", - "coverage>=5.2, <6.2", - "mock==4.0.3", - "contextlib2==21.6.0", - "awslogs==0.14.0", - "black==22.1.0", - "stopit==1.1.2", - "apache-airflow==2.2.3", - "apache-airflow-providers-amazon==3.0.0", - "attrs==20.3.0", - "fabric==2.6.0", - "requests==2.27.1", - "sagemaker-experiments==0.1.35", - "Jinja2==3.0.3", - ], -) +extras["test"] = (extras["all"] + read_requirements("requirements/extras/test_requirements.txt"),) setup( name="sagemaker", diff --git a/tox.ini b/tox.ini index 4d34870f39..8822f897a2 100644 --- a/tox.ini +++ b/tox.ini @@ -79,62 +79,51 @@ depends = [testenv:flake8] skipdist = true skip_install = true -deps = - flake8==4.0.1 - flake8-future-import==0.4.6 -commands = flake8 +commands = + pip install --exists-action=w -r requirements/tox/flake8_requirements.txt + flake8 [testenv:pylint] skipdist = true skip_install = true -deps = - pylint==2.6.2 - astroid==2.4.2 commands = + pip install --exists-action=w -r requirements/tox/pylint_requirements.txt python -m pylint --rcfile=.pylintrc -j 0 src/sagemaker [testenv:spelling] skipdist = true skip_install = true -deps = - pyenchant - pylint commands = + pip install --exists-action=w -r requirements/tox/spelling_requirements.txt python -m pylint --rcfile=.pylintrc --disable all --enable spelling --spelling-dict en_US src/sagemaker/{posargs} [testenv:twine] -# twine check was added starting in 1.12.0 -# https://github.com/pypa/twine/blob/master/docs/changelog.rst -deps = - twine==3.8.0 # https://packaging.python.org/guides/making-a-pypi-friendly-readme/#validating-restructuredtext-markup commands = + pip install --exists-action=w -r requirements/tox/twine_requirements.txt python setup.py sdist twine check dist/*.tar.gz [testenv:sphinx] +pip_version = pip==21.3 changedir = doc # pip install requirements.txt is separate as RTD does it in separate steps # having the requirements.txt installed in deps above results in Double Requirement exception # https://github.com/pypa/pip/issues/988 -deps = - pip==21.3 commands = pip install --exists-action=w -r requirements.txt sphinx-build -T -W -b html -d _build/doctrees-readthedocs -D language=en . _build/html [testenv:doc8] -deps = - doc8==0.10.1 - Pygments==2.11.2 -commands = doc8 +commands = + pip install --exists-action=w -r requirements/tox/doc8_requirements.txt + doc8 [testenv:black-format] # Used during development (before committing) to format .py files. setenv = LC_ALL=C.UTF-8 LANG=C.UTF-8 -deps = black==22.3.0 commands = black -l 100 ./ @@ -143,23 +132,22 @@ commands = setenv = LC_ALL=C.UTF-8 LANG=C.UTF-8 -deps = black==22.3.0 commands = black -l 100 --check ./ [testenv:clean] -deps = coverage==6.2 skip_install = true -commands = coverage erase +commands = + coverage erase [testenv:typing] -deps = mypy commands = + pip install --exists-action=w -r requirements/tox/mypy_requirements.txt mypy src/sagemaker [testenv:docstyle] -deps = pydocstyle==6.1.1 commands = + pip install --exists-action=w -r requirements/tox/pydocstyle_requirements.txt pydocstyle src/sagemaker [testenv:collect-tests]