Add per_gpu_scaling = False for distributed training (#1029)

* leave bs same for default distributed training

* Update CHANGELOG.md

* update all mentions aboutu per_gpu_scaling

Author: andriilitvynchuk

CHANGELOG.md

@@ -63,6 +63,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 - prevented modifying config during the experiment and runner initialization ([#1004](https://github.com/catalyst-team/catalyst/pull/1004))
 - a few test for RecSys MAP computation ([#1018](https://github.com/catalyst-team/catalyst/pull/1014))
+- leave batch size the same for default distributed training ([#1023](https://github.com/catalyst-team/catalyst/issues/1023))
 
 
 

catalyst/utils/loaders.py

@@ -8,7 +8,7 @@
 from torch.utils.data.dataloader import default_collate as default_collate_fn
 
 from catalyst.registry import SAMPLER
-from catalyst.utils.distributed import get_rank
+from catalyst.utils.distributed import get_distributed_params, get_rank
 from catalyst.utils.misc import merge_dicts, set_global_seed
 
 
@@ -205,7 +205,7 @@ def get_loaders_from_params(
         drop_last: ``drop_last`` parameter
             from ``torch.utils.data.DataLoader``
         per_gpu_scaling: boolean flag,
-            if ``True``, uses ``batch_size=batch_size*num_available_gpus``
+            if ``True``, scales batch_size in proportion to the number of GPUs
         loaders_params (Dict[str, Any]): additional loaders parameters
         samplers_params (Dict[str, Any]): additional sampler parameters
         initial_seed: initial seed for ``torch.utils.data.DataLoader``
@@ -275,6 +275,15 @@ def get_loaders_from_params(
             num_gpus = max(1, torch.cuda.device_count())
             batch_size *= num_gpus
             num_workers *= num_gpus
+        elif not per_gpu_scaling and distributed:
+            world_size = get_distributed_params().pop("world_size", 1)
+            if batch_size % world_size == 0:
+                batch_size = int(batch_size / world_size)
+            else:
+                raise ValueError(
+                    "For this distributed mode with per_gpu_scaling = False "
+                    "you need to have batch_size divisible by number of GPUs"
+                )
 
         loader_params = {
             "batch_size": batch_size,

examples/configs/config-description-eng.yml

@@ -41,7 +41,7 @@ stages:  # REQUIRED KEYWORD, dictionary of all stages of Catalyst, for training
     batch_size: 1  # KEYWORD, batch size for all the stages
     num_workers: 1  # KEYWORD, Number of parallel processes for DataLoader
     drop_last: False  # KEYWORD, parameter for DataLoader (Default is False)
-    per_gpu_scaling: False  # KEYWORD, if True and the working mode are not distributed, it increases the batch size and the number of workers in proportion to the number of GPUs
+    per_gpu_scaling: False  # KEYWORD, if True it increases the batch size and the number of workers in proportion to the number of GPUs (for distributed increases only batch_size)
     loaders_params:  # KEYWORD, parameters for loaders, optional
       # Example
       train:

examples/configs/config-description-rus.yml

@@ -41,7 +41,7 @@ stages:  # REQUIRED KEYWORD, словарь всех стадий Catalyst, дл
     batch_size: 1  # KEYWORD, размер батча для всех стейджей
     num_workers: 1  # KEYWORD, количество параллельных процессов для DataLoader
     drop_last: False  # KEYWORD, параметр для DataLoader (по умолчанию False)
-    per_gpu_scaling: False  # KEYWORD, если True и режим работы не distributed, то увеличивает батчсайз и количество воркеров пропорционально количиству видеокарт
+    per_gpu_scaling: False  # KEYWORD, если True, то увеличивает батчсайз и количество воркеров пропорционально количиству видеокарт (для distributed увеличивает только батчсайз)
     loaders_params:  # KEYWORD, параметры для лоадеров, опционально
       # Например
       train: