Merge remote-tracking branch 'origin/main' into publication

Author: Azure Pipelines

.actions/README.md

@@ -648,6 +648,7 @@ def copy_notebooks(
         path_docs_images: str = "_static/images",
         patterns: Sequence[str] = (".", "**"),
         ignore: Optional[Sequence[str]] = None,
+        strict: bool = True,
     ) -> None:
         """Copy all notebooks from a folder to doc folder.
 
@@ -658,12 +659,16 @@ def copy_notebooks(
             path_docs_images: destination path to the images' location relative to ``docs_root``
             patterns: patterns to use when glob-ing notebooks
             ignore: ignore some specific notebooks even when the given string is in path
+            strict: raise exception if copy fails
 
         """
-        all_ipynb = []
-        for pattern in patterns:
-            all_ipynb += glob.glob(os.path.join(path_root, DIR_NOTEBOOKS, pattern, "*.ipynb"))
         os.makedirs(os.path.join(docs_root, path_docs_ipynb), exist_ok=True)
+        all_ipynb = [
+            os.path.realpath(ipynb)
+            for pattern in patterns
+            for ipynb in glob.glob(os.path.join(path_root, DIR_NOTEBOOKS, pattern, "*.ipynb"))
+        ]
+        print(f"Copy following notebooks to docs folder: {all_ipynb}")
         if ignore and not isinstance(ignore, (list, set, tuple)):
             ignore = [ignore]
         elif not ignore:
@@ -683,8 +688,11 @@ def copy_notebooks(
                     path_docs_images=path_docs_images,
                 )
             except Exception as ex:
-                warnings.warn(f"Failed to copy notebook: {path_ipynb}\n{ex}", ResourceWarning)
-                continue
+                msg = f"Failed to copy notebook: {path_ipynb}\n{ex}"
+                if not strict:
+                    warnings.warn(msg, ResourceWarning)
+                    continue
+                raise FileNotFoundError(msg)
             ipynb_content.append(os.path.join(path_docs_ipynb, path_ipynb_in_dir))
 
     @staticmethod

.actions/assistant.py

@@ -30,4 +30,3 @@ git merge --ff -s resolve origin/$1
 python _TEMP/.actions/assistant.py group-folders target-diff.txt --fpath_actual_dirs "['dirs-$b1.txt', 'dirs-$b2.txt']"
 printf "\n================\nChanged folders:\n----------------\n" && cat changed-folders.txt
 printf "\n================\nDropped folders:\n----------------\n" && cat dropped-folders.txt
-

.actions/git-diff-sync.sh

@@ -1,6 +1,6 @@
 Fire
 tqdm
-PyYAML
+PyYAML <5.4  # todo: racing issue with cython compile
 wcmatch
 requests
 pip

.actions/requires.txt

@@ -37,6 +37,7 @@ jobs:
         name: mtrx
         displayName: "Changed matrix"
       - bash: echo '$(mtrx.dirs)' | python -m json.tool
+        continueOnError: "true" # not crash if the matrix is empty
         displayName: "Show matrix"
 
   - job: ipython

.azure/ipynb-validate.yml

@@ -4,7 +4,7 @@ updates:
   # Enable version updates for python
   - package-ecosystem: "pip"
     # Look for a `requirements` in the `root` directory
-    directory: "/_requirements"
+    directory: "/"
     # Check for updates once a week
     schedule:
       interval: "monthly"

.github/dependabot.yml

@@ -1,10 +1,10 @@
-name: validate Docs
+name: Docs validation
 
 on: # Trigger the workflow on push or pull request
   #  push:
   #    branches: [main]
   pull_request: {}
-  #workflow_dispatch: {}
+  workflow_dispatch: {}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref }}
@@ -20,10 +20,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        check: ["html", "linkcheck"]
+        target: ["html", "linkcheck"]
     env:
       PUB_BRANCH: publication
       PATH_DATASETS: ${{ github.workspace }}/.datasets
+      TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html"
     timeout-minutes: 20
     steps:
       - name: Checkout 🛎️
@@ -49,8 +50,8 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip --version
-          pip install -q -r requirements.txt -r _requirements/docs.txt
+          set -ex
+          pip install -q -r requirements.txt -r _requirements/docs.txt -f ${TORCH_URL}
           pip list
 
       - name: Process folders
@@ -97,23 +98,17 @@ jobs:
           tree changed-notebooks
 
       - uses: actions/upload-artifact@v3
-        if: ${{ matrix.check == 'html' && env.NB_DIRS != 0 }}
+        if: ${{ matrix.target == 'html' && env.NB_DIRS != 0 }}
         with:
           name: notebooks-${{ github.sha }}
           path: changed-notebooks/
 
-      - name: Link check
+      - name: Make ${{ matrix.target }}
         working-directory: ./_docs
-        if: ${{ matrix.check == 'linkcheck' }}
-        run: make linkcheck --jobs $(nproc) --debug SPHINXOPTS="--keep-going"
-
-      - name: Make Documentation
-        working-directory: ./_docs
-        if: ${{ matrix.check == 'html' }}
-        run: make html --jobs $(nproc) --debug SPHINXOPTS="-W --keep-going"
+        run: make ${{ matrix.target }} --jobs $(nproc) --debug SPHINXOPTS="-W --keep-going"
 
       - name: Upload built docs
-        if: ${{ matrix.check == 'html' }}
+        if: ${{ matrix.target == 'html' }}
         uses: actions/upload-artifact@v3
         with:
           name: docs-html-${{ github.sha }}

.github/workflows/ci_docs.yml

@@ -30,26 +30,14 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-
-      # Note: This uses an internal pip API and may not always work
-      # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
-      - name: Get pip cache dir
-        id: pip-cache
-        run: echo "::set-output name=dir::$(pip cache dir)"
-
-      - name: pip cache
-        uses: actions/cache@v3
-        with:
-          path: ${{ steps.pip-cache.outputs.dir }}
-          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('.actions/requires.txt') }}-${{ hashFiles('requirements/default.txt') }}
-          restore-keys: ${{ runner.os }}-pip-py${{ matrix.python-version }}-
+          cache: pip
 
       - name: Install requirements
         run: |
-          pip --version
           pip install -q -r .actions/requires.txt -r _requirements/test.txt
           # this is needed to be able to run package version parsing test
           pip install -q -r _requirements/default.txt --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
+          pip list
 
       - name: Prepare dummy inputs
         run: |

.github/workflows/ci_internal.yml

@@ -34,7 +34,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Build (and Push) image
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           #build-args: |
           #  UBUNTU_VERSION=${{ matrix.ubuntu }}
@@ -44,5 +44,5 @@ jobs:
           file: _dockers/ubuntu-cuda/Dockerfile
           push: ${{ env.PUSH_DOCKERHUB }}
           # todo: publish also tag YYYY.MM
-          tags: "pytorchlightning/tutorials"
+          tags: "pytorchlightning/tutorials:cuda"
         timeout-minutes: 55

.github/workflows/docker-build.yml

@@ -2,6 +2,11 @@ name: Deploy Docs
 on:
   push:
     branches: [publication]
+  pull_request:
+    branches: [main]
+    paths:
+      - ".actions/assistant.py"
+      - ".github/workflows/docs-deploy.yml"
   workflow_dispatch: {}
   workflow_run:
     workflows: ["Publish notebook"]
@@ -14,39 +19,59 @@ jobs:
     runs-on: ubuntu-20.04
     env:
       PATH_DATASETS: ${{ github.workspace }}/.datasets
+      TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html"
     steps:
-      - name: Checkout 🛎️
+      - name: Checkout 🛎️ Publication
+        if: ${{ github.event_name != 'pull_request' }}
         uses: actions/checkout@v4
         with:
           ref: publication
+      - name: Checkout 🛎️ PR
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
       - uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: "3.9"
+          cache: pip
+      - run: pip install -q py-tree
 
-      - name: Cache pip
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}-${{ hashFiles('_requirements/docs.txt') }}
-          restore-keys: ${{ runner.os }}-pip-
+      - name: pull notebooks from Publication
+        if: ${{ github.event_name == 'pull_request' }}
+        run: |
+          git checkout publication
+          py-tree .notebooks/
+          mkdir -p _notebooks
+          cp -r .notebooks/* _notebooks/
+          git checkout ${{ github.head_ref }}
+          cp -r _notebooks/* .notebooks/
+
+      - name: List notebooks
+        run: py-tree .notebooks/
 
       - name: Install dependencies
         run: |
           mkdir -p ${PATH_DATASETS}
-          # install Texlive, see https://linuxconfig.org/how-to-install-latex-on-ubuntu-20-04-focal-fossa-linux
-          sudo apt-get update
+          sudo apt-get update --fix-missing
           sudo apt-get install -y cmake pandoc
+          # install Texlive, see https://linuxconfig.org/how-to-install-latex-on-ubuntu-20-04-focal-fossa-linux
           sudo apt-get install -y texlive-latex-extra dvipng texlive-pictures
-          pip --version
-          pip install --quiet --requirement _requirements/docs.txt
+          pip install -q -r _requirements/docs.txt -f ${TORCH_URL}
           pip list
         shell: bash
 
       - name: Make Documentation
         working-directory: ./_docs
         run: make html --jobs $(nproc)
 
+      - name: Copied notebooks [debugging]
+        if: failure()
+        working-directory: ./_docs/source/
+        run: py-tree notebooks/
+
       - name: Deploy 🚀
+        if: ${{ github.event_name != 'pull_request' }}
         uses: JamesIves/[email protected]
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
@@ -55,4 +80,3 @@ jobs:
           clean: true # Automatically remove deleted files from the deploy branch
           target-folder: docs # If you'd like to push the contents of the deployment folder into a specific directory
           single-commit: true # you'd prefer to have a single commit on the deployment branch instead of full history
-        if: success()

.github/workflows/docs-deploy.yml

@@ -43,10 +43,8 @@
 # -- Project documents -------------------------------------------------------
 
 AssistantCLI.copy_notebooks(
-    _PATH_ROOT,
-    _PATH_HERE,
-    # ToDo: fix coping this specific notebooks, some JSON encode issue
-    ignore=["course_UvA-DL/13-contrastive-learning"],
+    path_root=_PATH_ROOT,
+    docs_root=_PATH_HERE,
 )
 
 # with open(os.path.join(_PATH_HERE, 'ipynb_content.rst'), 'w') as fp:

_docs/source/conf.py

@@ -373,7 +373,7 @@
 # <center style="width: 100%"><img src="comparison_CPU_GPU.png" width="700px"></center>
 #
 # CPUs and GPUs have both different advantages and disadvantages, which is why many computers contain both components and use them for different tasks.
-# In case you are not familiar with GPUs, you can read up more details in this [NVIDIA blog post](https://blogs.nvidia.com/blog/2009/12/16/whats-the-difference-between-a-cpu-and-a-gpu/) or [here](https://www.intel.com/content/www/us/en/products/docs/processors/what-is-a-gpu.html).
+# In case you are not familiar with GPUs, you can read up more details in this [NVIDIA blog post](https://blogs.nvidia.com/blog/2009/12/16/whats-the-difference-between-a-cpu-and-a-gpu/) or [here](https://blogs.nvidia.com/blog/whats-the-difference-between-a-cpu-and-a-gpu/).
 #
 # GPUs can accelerate the training of your network up to a factor of $100$ which is essential for large neural networks.
 # PyTorch implements a lot of functionality for supporting GPUs (mostly those of NVIDIA due to the libraries [CUDA](https://developer.nvidia.com/cuda-zone) and [cuDNN](https://developer.nvidia.com/cudnn)).

course_UvA-DL/01-introduction-to-pytorch/notebook.py

@@ -227,7 +227,7 @@
 # if the hyperparameters are not well tuned.
 # We will rely on training tricks proposed in the paper
 # [Implicit Generation and Generalization in Energy-Based Models](https://arxiv.org/abs/1903.08689)
-# by Yilun Du and Igor Mordatch ([blog](https://openai.com/research/energy-based-models)).
+# by Yilun Du and Igor Mordatch ([blog](https://openai.com/index/energy-based-models/)).
 # The important part of this notebook is however to see how the theory above can actually be used in a model.
 #
 # ### Dataset