Merge pull request #751 from ESMValGroup/version2_exp_concatenation_name

Multiple experiments in preprocessor output file name

Author: mattiarighi

esmvaltool/_data_finder.py

@@ -268,6 +268,11 @@ def get_output_file(variable, preproc_dir):
     """Return the full path to the output (preprocessed) file."""
     cfg = get_project_config(variable['project'])
 
+    # Join different experiment names
+    if isinstance(variable.get('exp'), (list, tuple)):
+        variable = dict(variable)
+        variable['exp'] = '-'.join(variable['exp'])
+
     outfile = os.path.join(
         preproc_dir,
         variable['diagnostic'],

esmvaltool/preprocessor/_io.py

@@ -216,6 +216,9 @@ def write_metadata(products, write_ncl=False):
         )
         metadata = OrderedDict()
         for product in sorted_products:
+            if isinstance(product.attributes.get('exp'), (list, tuple)):
+                product.attributes = dict(product.attributes)
+                product.attributes['exp'] = '-'.join(product.attributes['exp'])
             metadata[product.filename] = product.attributes
 
         output_filename = os.path.join(output_dir, 'metadata.yml')

esmvaltool/recipes/examples/recipe_concatenate_exps.yml

@@ -19,17 +19,26 @@ datasets:
 
 diagnostics:
 
-  diag_concatenate_exps:
-    description: Concatenate historical and RCP 8.5 data
+  diag_regular:
+    description: Only use historical data.
     variables:
-      tas:
+      tas: &variable_settings
         project: CMIP5
         mip: Amon
+        field: T2Ms
         ensemble: r1i1p1
-        exp: [historical, rcp85]
+        exp: historical
         start_year: 1950
+        end_year: 2000
+        additional_datasets:
+          - {dataset: ERA-Interim, project: OBS, tier: 3, type: reanaly, version: 1, start_year: 1980, end_year: 2000}
+    scripts: null
+
+  diag_concatenate_exps:
+    description: Concatenate historical and RCP 8.5 data
+    variables:
+      tas:
+        <<: *variable_settings
+        exp: [historical, rcp85]
         end_year: 2050
-        field: T2Ms
-    additional_datasets:
-      - {dataset: ERA-Interim, project: OBS, tier: 3, type: reanaly, version: 1, start_year: 1980, end_year: 2000}
     scripts: null

tests/integration/data_finder.yml

@@ -1,8 +1,8 @@
 ---
 
-get_input_filelist:
-  - drs: default
-    variable: &variable
+get_output_file:
+  - variable: &variable
+      variable_group: test
       short_name: ta
       dataset: HadGEM2-ES
       project: CMIP5
@@ -11,10 +11,27 @@ get_input_filelist:
       frequency: mon
       modeling_realm: [atmos]
       mip: Amon
+      field: T3M
       exp: historical
       ensemble: r1i1p1
       start_year: 1960
       end_year: 1980
+      diagnostic: test_diag
+      preprocessor: test_preproc
+    preproc_dir: this/is/a/path
+    output_file: this/is/a/path/test_diag/test/CMIP5_HadGEM2-ES_Amon_historical_r1i1p1_T3M_ta_1960-1980.nc
+
+  - variable:
+      <<: *variable
+      exp: [historical, rcp85]
+    preproc_dir: /test
+    output_file: /test/test_diag/test/CMIP5_HadGEM2-ES_Amon_historical-rcp85_r1i1p1_T3M_ta_1960-1980.nc
+
+
+get_input_filelist:
+  - drs: default
+    variable:
+      <<: *variable
     available_files:
       - ta_Amon_HadGEM2-ES_historical_r1i1p1_193412-195911.nc
       - ta_Amon_HadGEM2-ES_historical_r1i1p1_195912-198411.nc
@@ -141,6 +158,7 @@ get_input_filelist:
     found_files:
       - historical/Amon/ta/HadGEM2-ES/r1i1p1/ta_Amon_HadGEM2-ES_historical_r1i1p1_198412-200511.nc
 
+
 get_input_fx_filelist:
   - drs: default
     variable:

tests/integration/test_data_finder.py

@@ -7,7 +7,8 @@
 import yaml
 
 import esmvaltool._config
-from esmvaltool._data_finder import get_input_filelist, get_input_fx_filelist
+from esmvaltool._data_finder import (get_input_filelist, get_input_fx_filelist,
+                                     get_output_file)
 from esmvaltool.cmor.table import read_cmor_tables
 
 # Initialize with standard config developer file
@@ -60,6 +61,13 @@ def create_tree(path, filenames=None, symlinks=None):
         os.symlink(symlink['target'], link_name)
 
 
+@pytest.mark.parametrize('cfg', CONFIG['get_output_file'])
+def test_get_output_file(cfg):
+    """Test getting output name for preprocessed files."""
+    output_file = get_output_file(cfg['variable'], cfg['preproc_dir'])
+    assert output_file == cfg['output_file']
+
+
 @pytest.fixture
 def root():
     """Root function for tests."""