Merge remote-tracking branch 'origin/main' into publication

Author: Azure Pipelines

.actions/assistant.py

@@ -19,7 +19,7 @@
 _PATH_HERE = os.path.dirname(__file__)
 _PATH_ROOT = os.path.dirname(_PATH_HERE)
 PATH_SCRIPT_RENDER = os.path.join(_PATH_HERE, "_ipynb-render.sh")
-PATH_SCRIPT_TEST = os.path.join(_PATH_HERE, "_ipynb-test.sh")
+PATH_SCRIPT_TEST = os.path.join(_PATH_HERE, "_ipynb-validate.sh")
 # https://askubuntu.com/questions/909918/how-to-show-unzip-progress
 UNZIP_PROGRESS_BAR = ' | awk \'BEGIN {ORS=" "} {if(NR%10==0)print "."}\''
 REPO_NAME = "lightning-tutorials"
@@ -393,7 +393,7 @@ def bash_render(folder: str, output_file: str = PATH_SCRIPT_RENDER) -> Optional[
             fopen.write(os.linesep.join(cmd))
 
     @staticmethod
-    def bash_test(folder: str, output_file: str = PATH_SCRIPT_TEST, virtualenv: bool = False) -> Optional[str]:
+    def bash_validate(folder: str, output_file: str = PATH_SCRIPT_TEST, virtualenv: bool = False) -> Optional[str]:
         """Prepare bash script for running tests of a particular notebook.
 
         Args:
@@ -551,7 +551,7 @@ def group_folders(
             fpath_drop_folders: output file with deleted folders
             fpath_actual_dirs: files with listed all folder in particular stat
             strict: raise error if some folder outside skipped does not have valid meta file
-            root_path: path to the root tobe added for all local folder paths in files
+            root_path: path to the root to be added for all local folder paths in files
 
         Example:
             $ python assistant.py group-folders ../target-diff.txt \
@@ -571,12 +571,26 @@ def group_folders(
         # not empty paths
         dirs = [ln for ln in dirs if ln]
 
-        if root_path:
-            dirs = [os.path.join(root_path, d) for d in dirs]
-        # unique folders
-        dirs = set(dirs)
         # drop folder  that start with . or _ as they are meant to be internal use only
         dirs = [pdir for pdir in dirs if not any(ndir[0] in (".", "_") for ndir in pdir.split(os.path.sep))]
+        # append a path to root in case you call this from other path then root
+        if root_path:
+            dirs = [os.path.join(root_path, d) for d in dirs]
+        # append all subfolders in case of parent requirements has been changed all related notebooks shall be updated
+        dirs_expanded = []
+        for dir in dirs:
+            # in case that the diff item comes from removed folder
+            if not os.path.isdir(dir):
+                dirs_expanded += [dir]
+                continue
+            # list folder and skip all internal files, starting with . or _
+            sub_dirs = [os.path.join(dir, it) for it in os.listdir(dir) if it[0] not in (".", "_")]
+            # filter only folders
+            sub_dirs = [it for it in sub_dirs if os.path.isdir(it)]
+            # if the dir has sub-folder append then otherwise append the dir itself
+            dirs_expanded += sub_dirs if sub_dirs else [dir]
+        # unique folders only, drop duplicates
+        dirs = set(dirs_expanded)
         # valid folder has meta
         dirs_exist = [d for d in dirs if os.path.isdir(d)]
         dirs_invalid = [d for d in dirs_exist if not AssistantCLI._find_meta(d)]

.actions/test_cli.py

@@ -14,17 +14,25 @@ def _path_in_dir(fname: str, folder: str = _PATH_ROOT) -> str:
 
 
 @pytest.mark.parametrize(
-    "cmd,args",
+    "cmd,kwargs",
     [
-        ("list_dirs", []),
-        ("list_dirs", [".", ".ipynb"]),
-        ("bash_render", [_PATH_DIR_SIMPLE]),
-        ("bash_test", [_PATH_DIR_SIMPLE]),
-        ("group_folders", [_path_in_dir("master-diff.txt"), _path_in_dir("dirs-b1.txt"), _path_in_dir("dirs-b2.txt")]),
-        ("convert_ipynb", [_PATH_DIR_SIMPLE]),
-        ("copy_notebooks", [_PATH_ROOT]),
-        ("update_env_details", [_PATH_DIR_SIMPLE]),
+        ("list_dirs", {}),
+        ("list_dirs", dict(folder=".", include_file_ext=".ipynb")),
+        ("bash_render", dict(folder=_PATH_DIR_SIMPLE)),
+        ("bash_validate", dict(folder=_PATH_DIR_SIMPLE)),
+        (
+            "group_folders",
+            dict(
+                fpath_gitdiff=_path_in_dir("master-diff.txt"),
+                fpath_change_folders=_path_in_dir("dirs-b1.txt"),
+                fpath_drop_folders=_path_in_dir("dirs-b2.txt"),
+                root_path=_PATH_ROOT,
+            ),
+        ),
+        ("convert_ipynb", dict(folder=_PATH_DIR_SIMPLE)),
+        ("copy_notebooks", dict(path_root=_PATH_ROOT)),
+        ("update_env_details", dict(folder=_PATH_DIR_SIMPLE)),
     ],
 )
-def test_assistant_commands(cmd: str, args: list):
-    AssistantCLI().__getattribute__(cmd)(*args)
+def test_assistant_commands(cmd: str, kwargs: dict):
+    AssistantCLI().__getattribute__(cmd)(**kwargs)

.azure/ipynb-validate.yml

@@ -25,6 +25,7 @@ jobs:
           # printf "Head: $head\n"  # this shall be commit hash
           # git diff --name-only $head --output=target-diff.txt
           git diff --name-only origin/main HEAD --output=target-diff.txt
+          printf "Diff to target:\n"
           cat target-diff.txt
           python .actions/assistant.py group-folders --fpath_gitdiff=target-diff.txt
           printf "Changed folders:\n"
@@ -97,9 +98,9 @@ jobs:
       - bash: |
           set -e
           mkdir $(PATH_DATASETS)
-          python .actions/assistant.py bash-test $(notebook)
-          cat .actions/_ipynb-test.sh
-          bash .actions/_ipynb-test.sh
+          python .actions/assistant.py bash-validate $(notebook)
+          cat .actions/_ipynb-validate.sh
+          bash .actions/_ipynb-validate.sh
         env:
           KAGGLE_USERNAME: $(KAGGLE_USERNAME)
           KAGGLE_KEY: $(KAGGLE_KEY)

_dockers/ubuntu-cuda/Dockerfile

@@ -19,7 +19,7 @@ ARG CUDA_VERSION=11.8.0
 FROM nvidia/cuda:${CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION}
 
 ARG PYTHON_VERSION=3.10
-ARG PYTORCH_VERSION=2.0
+ARG PYTORCH_VERSION=2.3
 
 SHELL ["/bin/bash", "-c"]
 # https://techoverflow.net/2019/05/18/how-to-fix-configuring-tzdata-interactive-input-when-building-docker-images/

course_UvA-DL/requirements.txt

@@ -1,4 +1,4 @@
 numpy <2.0 # needed for older Torch
 torch>=1.8.1, <2.1.0
 pytorch-lightning>=2.0, <2.1.0
-torchmetrics>=0.7, <1.3
+torchmetrics>=1.0, <1.3

lightning_examples/cifar10-baseline/baseline.py

@@ -16,7 +16,7 @@
 from lightning.pytorch.callbacks import LearningRateMonitor
 from lightning.pytorch.loggers import CSVLogger
 from torch.optim.lr_scheduler import OneCycleLR
-from torch.optim.swa_utils import AveragedModel, update_bn
+from torch.optim.swa_utils import AveragedModel
 from torch.utils.data import DataLoader, random_split
 from torchmetrics.functional import accuracy
 from torchvision.datasets import CIFAR10
@@ -173,15 +173,15 @@ def configure_optimizers(self):
 model = LitResnet(lr=0.05)
 
 trainer = L.Trainer(
-    max_epochs=30,
+    max_epochs=5,
     accelerator="auto",
     devices=1,
     logger=CSVLogger(save_dir="logs/"),
     callbacks=[LearningRateMonitor(logging_interval="step")],
 )
 
-trainer.fit(model, train_dataloader, val_dataloaders=val_dataloader)
-trainer.test(model, test_dataloader)
+trainer.fit(model, train_dataloaders=train_dataloader, val_dataloaders=val_dataloader)
+trainer.test(model, dataloaders=test_dataloader)
 
 # %%
 
@@ -229,22 +229,22 @@ def configure_optimizers(self):
         optimizer = torch.optim.SGD(self.model.parameters(), lr=self.hparams.lr, momentum=0.9, weight_decay=5e-4)
         return optimizer
 
-    def on_train_end(self):
-        update_bn(self.trainer.datamodule.train_dataloader(), self.swa_model, device=self.device)
+    # def on_train_end(self):  # todo: failing as trainer has only dataloaders, not datamodules
+    #     update_bn(self.trainer.datamodule.train_dataloader(), self.swa_model, device=self.device)
 
 
 # %%
 swa_model = SWAResnet(model.model, lr=0.01)
 
 swa_trainer = L.Trainer(
-    max_epochs=20,
+    max_epochs=5,
     accelerator="auto",
     devices=1,
     logger=CSVLogger(save_dir="logs/"),
 )
 
-swa_trainer.fit(swa_model, train_dataloader, val_dataloaders=val_dataloader)
-swa_trainer.test(swa_model, test_dataloader)
+swa_trainer.fit(swa_model, train_dataloaders=train_dataloader, val_dataloaders=val_dataloader)
+swa_trainer.test(swa_model, dataloaders=test_dataloader)
 
 # %%