refactor requirements into own file with parent inheritance (#336)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: Borda

.actions/assistant.py

@@ -18,7 +18,6 @@
 
 _PATH_HERE = os.path.dirname(__file__)
 _PATH_ROOT = os.path.dirname(_PATH_HERE)
-PATH_REQ_DEFAULT = os.path.join(_PATH_ROOT, "_requirements", "default.txt")
 PATH_SCRIPT_RENDER = os.path.join(_PATH_HERE, "_ipynb-render.sh")
 PATH_SCRIPT_TEST = os.path.join(_PATH_HERE, "_ipynb-test.sh")
 # https://askubuntu.com/questions/909918/how-to-show-unzip-progress
@@ -93,8 +92,18 @@
 """
 
 
-def load_requirements(path_req: str = PATH_REQ_DEFAULT) -> list:
-    """Load the requirements from a file."""
+def load_requirements(folder: str, fname: str = "requirements.txt") -> List[str]:
+    """Load the requirements from a file.
+
+    Args:
+        folder: path to the folder with requirements
+        fname: filename
+
+    """
+    path_req = os.path.join(folder, fname)
+    if not os.path.isfile(path_req):
+        warnings.warn(f"Missing expected requirement file '{path_req}'")
+        return []
     with open(path_req) as fopen:
         req = fopen.readlines()
     req = [r[: r.index("#")] if "#" in r else r for r in req]
@@ -184,7 +193,7 @@ class AssistantCLI:
     _EXT_ARCHIVE_TAR = (".tar", ".gz")
     _EXT_ARCHIVE = _EXT_ARCHIVE_ZIP + _EXT_ARCHIVE_TAR
     _AZURE_POOL = "lit-rtx-3090"
-    _AZURE_DOCKER = "pytorchlightning/tutorials:latest"
+    _AZURE_DOCKER = "pytorchlightning/tutorials:cuda"
 
     @staticmethod
     def _find_meta(folder: str) -> str:
@@ -276,22 +285,22 @@ def _parse_requirements(folder: str) -> Tuple[str, str]:
 
         """
         meta = AssistantCLI._load_meta(folder)
-        reqs = meta.get("requirements", [])
+        requires = set(load_requirements(folder) + load_requirements(os.path.dirname(folder)))
 
         meta_pip_args = {
             k.replace(AssistantCLI._META_PIP_KEY, ""): v
             for k, v in meta.items()
             if k.startswith(AssistantCLI._META_PIP_KEY)
         }
-        pip_args = ['--find-links="https://download.pytorch.org/whl/"' + _RUNTIME_VERSIONS.get("DEVICE")]
+        pip_args = ['--extra-index-url="https://download.pytorch.org/whl/"' + _RUNTIME_VERSIONS.get("DEVICE")]
         for pip_key in meta_pip_args:
             if not isinstance(meta_pip_args[pip_key], (list, tuple, set)):
                 meta_pip_args[pip_key] = [meta_pip_args[pip_key]]
             for arg in meta_pip_args[pip_key]:
                 arg = arg % _RUNTIME_VERSIONS
                 pip_args.append(f"--{pip_key} {arg}")
 
-        return " ".join([f'"{req}"' for req in reqs]), " ".join(pip_args)
+        return " ".join([f'"{req}"' for req in requires]), " ".join(pip_args)
 
     @staticmethod
     def _bash_download_data(folder: str) -> List[str]:
@@ -465,7 +474,8 @@ def convert_ipynb(folder: str) -> None:
         meta["description"] = meta["description"].replace(os.linesep, f"{os.linesep}# ")
 
         header = TEMPLATE_HEADER % meta
-        requires = set(load_requirements() + meta["requirements"])
+        # load local and parent requirements
+        requires = set(load_requirements(folder) + load_requirements(os.path.dirname(folder)))
         setup = TEMPLATE_SETUP % dict(requirements=" ".join([f'"{req}"' for req in requires]))
         py_script = [header + setup] + py_script + [TEMPLATE_FOOTER]
 
@@ -769,23 +779,20 @@ def update_env_details(folder: str, base_path: str = DIR_NOTEBOOKS) -> str:
 
         """
         meta = AssistantCLI._load_meta(folder)
-        # default is COU runtime
-        with open(PATH_REQ_DEFAULT) as fopen:
-            req = fopen.readlines()
-        req += meta.get("requirements", [])
-        req = [r.strip() for r in req]
+        # load local and parent requirements
+        requires = set(load_requirements(folder) + load_requirements(os.path.dirname(folder)))
 
-        require = {_parse_package_name(r) for r in req if r}
+        requires = {_parse_package_name(r) for r in requires if r}
         # duplicate package name/key for cases with -/_ separator
         env = {
             **{_parse_package_name(p).replace("-", "_"): p for p in freeze.freeze()},
             **{_parse_package_name(p).replace("_", "-"): p for p in freeze.freeze()},
         }
         # for debugging reasons print the env and requested packages
         try:
-            meta["environment"] = [env[r] for r in require]
+            meta["environment"] = [env[r] for r in requires]
         except KeyError:
-            raise KeyError(f"Missing matching requirements: {require}\n within environment: {env}")
+            raise KeyError(f"Missing matching requirements: {requires}\n within environment: {env}")
         meta["published"] = datetime.now().isoformat()
 
         fmeta = os.path.join(base_path, folder) + ".yaml"

.azure/ipynb-publish.yml

@@ -149,7 +149,7 @@ jobs:
         timeoutInMinutes: "15"
 
       - bash: |
-          set -e
+          set -ex
           pip list
           python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu > 0, f'GPU: {mgpu}'"
           python -m papermill --version

.azure/ipynb-validate.yml

@@ -86,6 +86,8 @@ jobs:
         displayName: "Install dependencies"
 
       - bash: |
+          set -ex
+          pip list
           python -c "import torch ; mgpu = torch.cuda.device_count() ; assert mgpu > 0, f'GPU: {mgpu}'"
         displayName: "Sanity check"
 

.github/CONTRIBUTING.md

@@ -33,7 +33,7 @@ ______________________________________________________________________
 
 We are always looking for help to implement new features or fixing bugs.
 
-A lot of good work has already been done in project mechanics (\_requirements/base.txt, setup.py, pep8, badges, ci, etc...) so we're in a good state there thanks to all sooner contributors!
+A lot of good work has already been done in project mechanics (setup.py, pep8, badges, ci, etc...) so we're in a good state there thanks to all sooner contributors!
 
 ### Bug Fixes:
 

.github/workflows/ci_internal.yml

@@ -36,7 +36,7 @@ jobs:
         run: |
           pip install -q -r .actions/requires.txt -r _requirements/test.txt
           # this is needed to be able to run package version parsing test
-          pip install -q -r _requirements/default.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
+          pip install -q -r _requirements/devel.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
           pip list
 
       - name: Prepare dummy inputs

_dockers/README.md

@@ -8,7 +8,7 @@ You can build it on your own, note it takes lots of time, be prepared.
 git clone https://github.com/Lightning-AI/torchmetrics.git
 
 # build with the default arguments
-docker image build -t tutorials:latest -f dockers/ubuntu-cuda/Dockerfile .
+docker image build -t tutorials:cuda -f dockers/ubuntu-cuda/Dockerfile .
 
 # build with specific arguments
 docker image build -t tutorials:ubuntu-cuda11.7.1-py3.9-torch1.13 \
@@ -23,14 +23,14 @@ To run your docker use
 
 ```bash
 docker image list
-docker run --rm -it tutorials:latest bash
+docker run --rm -it tutorials:cuda bash
 ```
 
 and if you do not need it anymore, just clean it:
 
 ```bash
 docker image list
-docker image rm tutorials:latest
+docker image rm tutorials:cuda
 ```
 
 ## Run docker image with GPUs

_requirements/default.txt

@@ -1,6 +1,13 @@
 virtualenv>=20.10
-jupytext>=1.10, <1.15  # converting
-pytest>=7.0, <9.0
+
+# pytest>=7.0, <9.0
 # testing with own fork with extended cell timeout
 # https://github.com/Borda/nbval/archive/refs/heads/timeout-limit.zip
-papermill>=2.3.4, <2.5.0  # render
+ipython[notebook]>=8.0.0, <8.17.0
+urllib3  # for ipython
+jupytext>=1.10, <1.15  # converting
+papermill>=2.3.4, <2.5.0  # rendering
+
+matplotlib
+# all is based on PyTorch
+torch

_requirements/devel.txt

@@ -1,16 +1,11 @@
 title: "Tutorial 1: Introduction to PyTorch"
 author: Phillip Lippe
 created: 2021-08-27
-updated: 2023-03-14
 license: CC BY-SA
-build: 1
 description: |
   This tutorial will give a short introduction to PyTorch basics, and get you setup for writing your own neural networks.
   This notebook is part of a lecture series on Deep Learning at the University of Amsterdam.
   The full list of tutorials can be found at https://uvadlc-notebooks.rtfd.io.
-requirements:
-  - matplotlib
-  - lightning>=2.0.0
 accelerator:
   - CPU
   - GPU

course_UvA-DL/01-introduction-to-pytorch/.meta.yml

@@ -0,0 +1,2 @@
+matplotlib
+lightning>=2.0.0

course_UvA-DL/01-introduction-to-pytorch/requirements.txt

@@ -1,7 +1,6 @@
 title: "Tutorial 2: Activation Functions"
 author: Phillip Lippe
 created: 2021-08-27
-updated: 2023-03-14
 license: CC BY-SA
 description: |
   In this tutorial, we will take a closer look at (popular) activation functions and investigate their effect on optimization properties in neural networks.
@@ -10,11 +9,6 @@ description: |
   The goal of this tutorial is to show the importance of choosing a good activation function (and how to do so), and what problems might occur if we don't.
   This notebook is part of a lecture series on Deep Learning at the University of Amsterdam.
   The full list of tutorials can be found at https://uvadlc-notebooks.rtfd.io.
-requirements:
-  - torchvision
-  - matplotlib
-  - seaborn
-  - lightning>=2.0.0
 accelerator:
   - CPU
   - GPU

course_UvA-DL/02-activation-functions/.meta.yml

@@ -0,0 +1,4 @@
+torchvision
+matplotlib
+seaborn
+lightning>=2.0.0

course_UvA-DL/02-activation-functions/Activation_Functions.py renamed to course_UvA-DL/02-activation-functions/notebook.py

@@ -1,7 +1,6 @@
 title: "Tutorial 3: Initialization and Optimization"
 author: Phillip Lippe
 created: 2021-08-27
-updated: 2023-03-14
 license: CC BY-SA
 tags:
   - Image
@@ -14,11 +13,6 @@ description: |
   This is why we will take a closer look at the following concepts: initialization and optimization.
   This notebook is part of a lecture series on Deep Learning at the University of Amsterdam.
   The full list of tutorials can be found at https://uvadlc-notebooks.rtfd.io.
-requirements:
-  - torchvision
-  - matplotlib
-  - seaborn
-  - lightning>=2.0.0
 accelerator:
   - CPU
   - GPU

course_UvA-DL/02-activation-functions/requirements.txt

@@ -0,0 +1,4 @@
+torchvision
+matplotlib
+seaborn
+lightning>=2.0.0

course_UvA-DL/03-initialization-and-optimization/.meta.yml

@@ -1,7 +1,6 @@
 title: "Tutorial 4: Inception, ResNet and DenseNet"
 author: Phillip Lippe
 created: 2021-08-27
-updated: 2023-03-14
 license: CC BY-SA
 tags:
   - Image
@@ -13,12 +12,5 @@ description: |
   Thus, it is important to understand these architectures in detail and learn how to implement them.
   This notebook is part of a lecture series on Deep Learning at the University of Amsterdam.
   The full list of tutorials can be found at https://uvadlc-notebooks.rtfd.io.
-requirements:
-  - torchvision
-  - matplotlib
-  - seaborn
-  - tabulate
-  - lightning>=2.0.0
-  - tensorboard
 accelerator:
   - GPU

course_UvA-DL/03-initialization-and-optimization/Initialization_and_Optimization.py renamed to course_UvA-DL/03-initialization-and-optimization/notebook.py

@@ -0,0 +1,6 @@
+torchvision
+matplotlib
+seaborn
+tabulate
+lightning>=2.0.0
+tensorboard

course_UvA-DL/03-initialization-and-optimization/requirements.txt

@@ -1,9 +1,7 @@
 title: "Tutorial 5: Transformers and Multi-Head Attention"
 author: Phillip Lippe
 created: 2021-06-30
-updated: 2023-03-14
 license: CC BY-SA
-build: 0
 tags:
   - Text
 description: |
@@ -15,10 +13,5 @@ description: |
   it is important to understand how it works, and have implemented it yourself, which we will do in this notebook.
   This notebook is part of a lecture series on Deep Learning at the University of Amsterdam.
   The full list of tutorials can be found at https://uvadlc-notebooks.rtfd.io.
-requirements:
-  - torchvision
-  - matplotlib
-  - seaborn
-  - lightning>=2.0.0
 accelerator:
   - GPU

course_UvA-DL/04-inception-resnet-densenet/.meta.yaml

@@ -0,0 +1,4 @@
+torchvision
+matplotlib
+seaborn
+lightning>=2.0.0

course_UvA-DL/04-inception-resnet-densenet/Inception_ResNet_DenseNet.py renamed to course_UvA-DL/04-inception-resnet-densenet/notebook.py

@@ -1,9 +1,7 @@
 title: "Tutorial 6: Basics of Graph Neural Networks"
 author: Phillip Lippe
 created: 2021-06-07
-updated: 2023-03-14
 license: CC BY-SA
-build: 0
 tags:
   - Graph
 description: |
@@ -17,13 +15,6 @@ description: |
   Finally, we will apply a GNN on semi-supervised node classification and molecule categorization.
   This notebook is part of a lecture series on Deep Learning at the University of Amsterdam.
   The full list of tutorials can be found at https://uvadlc-notebooks.rtfd.io.
-requirements:
-  - torch-scatter
-  - torch-sparse
-  - torch-cluster
-  - torch-spline-conv
-  - "torch-geometric>=2.0.0,<2.5.0"
-  - "lightning>=2.0.0"
 pip__find-link:
   # - https://pytorch-geometric.com/whl/torch-1.8.0+cu101.html
   - https://pytorch-geometric.com/whl/torch-%(TORCH_MAJOR_DOT_MINOR)s.0+%(DEVICE)s.html

course_UvA-DL/04-inception-resnet-densenet/requirements.txt

@@ -0,0 +1,6 @@
+torch-scatter
+torch-sparse
+torch-cluster
+torch-spline-conv
+torch-geometric>=2.0.0,<2.5.0
+lightning>=2.0.0

course_UvA-DL/05-transformers-and-MH-attention/.meta.yml

@@ -1,9 +1,7 @@
 title: "Tutorial 7: Deep Energy-Based Generative Models"
 author: Phillip Lippe
 created: 2021-07-12
-updated: 2023-03-14
 license: CC BY-SA
-build: 1
 tags:
   - Image
 description: |
@@ -18,11 +16,6 @@ description: |
   we will show the idea of energy-based models with a lot of examples.
   This notebook is part of a lecture series on Deep Learning at the University of Amsterdam.
   The full list of tutorials can be found at https://uvadlc-notebooks.rtfd.io.
-requirements:
-  - torchvision
-  - matplotlib
-  - tensorboard
-  - pytorch-lightning>=2.0.0
 accelerator:
   - CPU
   - GPU

course_UvA-DL/05-transformers-and-MH-attention/Transformers_MHAttention.py renamed to course_UvA-DL/05-transformers-and-MH-attention/MHAttention.py

@@ -0,0 +1,4 @@
+torchvision
+matplotlib
+tensorboard
+pytorch-lightning>=2.0.0

course_UvA-DL/05-transformers-and-MH-attention/requirements.txt

@@ -1,9 +1,7 @@
 title: "Tutorial 8: Deep Autoencoders"
 author: Phillip Lippe
 created: 2021-07-12
-updated: 2023-03-14
 license: CC BY-SA
-build: 0
 tags:
   - Image
 description: |
@@ -18,12 +16,6 @@ description: |
   and need to output an image of full size (e.g. in VAE, GANs, or super-resolution applications).
   This notebook is part of a lecture series on Deep Learning at the University of Amsterdam.
   The full list of tutorials can be found at https://uvadlc-notebooks.rtfd.io.
-requirements:
-  - torchvision
-  - matplotlib
-  - seaborn
-  - lightning>=2.0.0
-  - tensorboard
 accelerator:
   - CPU
   - GPU