feature: dependabot integ - move all deps to requirements.txt (#2981)

Author: mufaddal-rohawala

MANIFEST.in

@@ -1,6 +1,7 @@
 recursive-include src/sagemaker *.py
 
 include src/sagemaker/image_uri_config/*.json
+recursive-include requirements *
 
 include VERSION
 include LICENSE.txt

requirements/extras/local_requirements.txt

@@ -0,0 +1,4 @@
+urllib3==1.26.8
+docker-compose==1.29.2
+docker~=5.0.0
+PyYAML==5.4.1

requirements/extras/scipy_requirements.txt

@@ -0,0 +1 @@
+scipy==1.5.4

requirements/extras/test_requirements.txt

@@ -0,0 +1,20 @@
+tox==3.24.5
+flake8==4.0.1
+pytest==6.0.2
+pytest-cov==3.0.0
+pytest-rerunfailures==10.2
+pytest-timeout==2.1.0
+pytest-xdist==2.4.0
+coverage>=5.2, <6.2
+mock==4.0.3
+contextlib2==21.6.0
+awslogs==0.14.0
+black==22.3.0
+stopit==1.1.2
+apache-airflow==2.2.4
+apache-airflow-providers-amazon==3.0.0
+attrs==20.3.0
+fabric==2.6.0
+requests==2.27.1
+sagemaker-experiments==0.1.35
+Jinja2==3.0.3

requirements/tox/doc8_requirements.txt

@@ -0,0 +1,2 @@
+doc8==0.10.1
+Pygments==2.11.2

requirements/tox/docstyle_requirements.txt

@@ -0,0 +1 @@
+pydocstyle==6.1.1

requirements/tox/flake8_requirements.txt

@@ -0,0 +1,2 @@
+flake8==4.0.1
+flake8-future-import==0.4.6

requirements/tox/mypy_requirements.txt

@@ -0,0 +1 @@
+mypy==0.942

requirements/tox/pydocstyle_requirements.txt

@@ -0,0 +1 @@
+pydocstyle==6.1.1

requirements/tox/pylint_requirements.txt

@@ -0,0 +1,2 @@
+pylint==2.6.2
+astroid==2.4.2

requirements/tox/spelling_requirements.txt

@@ -0,0 +1,2 @@
+pyenchant==3.2.2
+pylint==2.6.2

requirements/tox/twine_requirements.txt

@@ -0,0 +1 @@
+twine==3.8.0

setup.py

@@ -31,6 +31,20 @@ def read_version():
     return read("VERSION").strip()
 
 
+def read_requirements(filename):
+    """Reads requirements file which lists package dependencies.
+
+    Args:
+        filename: type(str) Relative file path of requirements.txt file
+
+    Returns:
+        list of dependencies extracted from file
+    """
+    with open(os.path.abspath(filename)) as fp:
+        deps = [line.strip() for line in fp.readlines()]
+    return deps
+
+
 # Declare minimal set for installation
 required_packages = [
     "attrs==20.3.0",
@@ -47,43 +61,15 @@ def read_version():
 ]
 
 # Specific use case dependencies
+# Keep format of *_requirements.txt to be tracked by dependabot
 extras = {
-    "local": [
-        "urllib3==1.26.8",
-        "docker-compose==1.29.2",
-        "docker~=5.0.0",
-        "PyYAML==5.4.1",  # PyYAML version has to match docker-compose requirements
-    ],
-    "scipy": ["scipy==1.5.4"],
+    "local": read_requirements("requirements/extras/local_requirements.txt"),
+    "scipy": read_requirements("requirements/extras/scipy_requirements.txt"),
 }
 # Meta dependency groups
 extras["all"] = [item for group in extras.values() for item in group]
 # Tests specific dependencies (do not need to be included in 'all')
-extras["test"] = (
-    [
-        extras["all"],
-        "tox==3.24.5",
-        "flake8==4.0.1",
-        "pytest==6.0.2",
-        "pytest-cov==3.0.0",
-        "pytest-rerunfailures==10.2",
-        "pytest-timeout==2.1.0",
-        "pytest-xdist==2.4.0",
-        "coverage>=5.2, <6.2",
-        "mock==4.0.3",
-        "contextlib2==21.6.0",
-        "awslogs==0.14.0",
-        "black==22.1.0",
-        "stopit==1.1.2",
-        "apache-airflow==2.2.3",
-        "apache-airflow-providers-amazon==3.0.0",
-        "attrs==20.3.0",
-        "fabric==2.6.0",
-        "requests==2.27.1",
-        "sagemaker-experiments==0.1.35",
-        "Jinja2==3.0.3",
-    ],
-)
+extras["test"] = (extras["all"] + read_requirements("requirements/extras/test_requirements.txt"),)
 
 setup(
     name="sagemaker",

tox.ini

@@ -79,62 +79,51 @@ depends =
 [testenv:flake8]
 skipdist = true
 skip_install = true
-deps =
-    flake8==4.0.1
-    flake8-future-import==0.4.6
-commands = flake8
+commands =
+    pip install --exists-action=w -r requirements/tox/flake8_requirements.txt
+    flake8
 
 [testenv:pylint]
 skipdist = true
 skip_install = true
-deps =
-    pylint==2.6.2
-    astroid==2.4.2
 commands =
+    pip install --exists-action=w -r requirements/tox/pylint_requirements.txt
     python -m pylint --rcfile=.pylintrc -j 0 src/sagemaker
 
 [testenv:spelling]
 skipdist = true
 skip_install = true
-deps =
-    pyenchant
-    pylint
 commands =
+    pip install --exists-action=w -r requirements/tox/spelling_requirements.txt
     python -m pylint --rcfile=.pylintrc --disable all --enable spelling --spelling-dict en_US src/sagemaker/{posargs}
 
 [testenv:twine]
-# twine check was added starting in 1.12.0
-# https://github.com/pypa/twine/blob/master/docs/changelog.rst
-deps =
-    twine==3.8.0
 # https://packaging.python.org/guides/making-a-pypi-friendly-readme/#validating-restructuredtext-markup
 commands =
+    pip install --exists-action=w -r requirements/tox/twine_requirements.txt
     python setup.py sdist
     twine check dist/*.tar.gz
 
 [testenv:sphinx]
+pip_version = pip==21.3
 changedir = doc
 # pip install requirements.txt is separate as RTD does it in separate steps
 # having the requirements.txt installed in deps above results in Double Requirement exception
 # https://github.com/pypa/pip/issues/988
-deps =
-    pip==21.3
 commands =
     pip install --exists-action=w -r requirements.txt
     sphinx-build -T -W -b html -d _build/doctrees-readthedocs -D language=en . _build/html
 
 [testenv:doc8]
-deps =
-    doc8==0.10.1
-    Pygments==2.11.2
-commands = doc8
+commands =
+    pip install --exists-action=w -r requirements/tox/doc8_requirements.txt
+    doc8
 
 [testenv:black-format]
 # Used during development (before committing) to format .py files.
 setenv =
     LC_ALL=C.UTF-8
     LANG=C.UTF-8
-deps = black==22.3.0
 commands =
     black -l 100 ./
 
@@ -143,23 +132,22 @@ commands =
 setenv =
     LC_ALL=C.UTF-8
     LANG=C.UTF-8
-deps = black==22.3.0
 commands =
     black -l 100 --check ./
 
 [testenv:clean]
-deps = coverage==6.2
 skip_install = true
-commands = coverage erase
+commands =
+    coverage erase
 
 [testenv:typing]
-deps = mypy
 commands =
+    pip install --exists-action=w -r requirements/tox/mypy_requirements.txt
     mypy src/sagemaker
 
 [testenv:docstyle]
-deps = pydocstyle==6.1.1
 commands =
+    pip install --exists-action=w -r requirements/tox/pydocstyle_requirements.txt
     pydocstyle src/sagemaker
 
 [testenv:collect-tests]