Merge pull request #649 from ESMValGroup/version2_provenance

Version 2: Provenance tracking

Author: mattiarighi

.circleci/config.yml

@@ -39,13 +39,6 @@ jobs:
           path: test-reports/
       - store_artifacts:
           path: test-reports/
-      - run:
-          # Upload Python 3 test coverage to codacy, even when the actual
-          # running of the tests fails.
-          when: always
-          command: |
-            pip install codacy-coverage
-            python-codacy-coverage -r test-reports/python3/coverage.xml
 
   python2_test:
     # Run Python 2 tests
@@ -119,6 +112,15 @@ jobs:
             ncl -V
       - store_artifacts:
           path: /logs
+      - store_artifacts:
+          path: test-reports/
+      - store_test_results:
+          path: test-reports/
+      - run:
+          when: always
+          command: |
+            pip install codacy-coverage
+            python-codacy-coverage -r test-reports/python3/coverage.xml
 
   python2_install:
     # Test Python 2 installation

environment.yml

@@ -7,6 +7,7 @@ channels:
 dependencies:
   # Python packages that cannot be installed from PyPI:
   - iris=1.13
+  - graphviz
   - matplotlib<3
   - python-stratify
   - esmpy

esmvaltool/_config.py

@@ -48,7 +48,7 @@ def read_config_user_file(config_file, recipe_name):
 
     for key in defaults:
         if key not in cfg:
-            logger.warning(
+            logger.info(
                 "No %s specification in config file, "
                 "defaulting to %s", key, defaults[key])
             cfg[key] = defaults[key]
@@ -94,10 +94,9 @@ def get_config_user_file():
 
 
 def _normalize_path(path):
-    """
-    Normalize paths.
+    """Normalize paths.
 
-    Expand ~ character and environment variables and convert path to absolute
+    Expand ~ character and environment variables and convert path to absolute.
 
     Parameters
     ----------
@@ -154,6 +153,7 @@ def configure_logging(cfg_file=None, output=None, console_log_level=None):
 
     logging.config.dictConfig(cfg)
     logging.Formatter.converter = time.gmtime
+    logging.captureWarnings(True)
 
     return log_files
 
@@ -182,3 +182,34 @@ def replace_mip_fx(fx_file):
     new_mip = CFG['CMIP5']['fx_mip_change'].get(fx_file, default_mip)
     logger.debug("Switching mip for fx file finding to %s", new_mip)
     return new_mip
+
+
+TAGS_CONFIG_FILE = os.path.join(
+    os.path.dirname(__file__), 'config-references.yml')
+
+
+def _load_tags(filename=TAGS_CONFIG_FILE):
+    """Load the refence tags used for provenance recording."""
+    logger.debug("Loading tags from %s", filename)
+    with open(filename) as file:
+        return yaml.safe_load(file)
+
+
+TAGS = _load_tags()
+
+
+def get_tag_value(section, tag):
+    """Retrieve the value of a tag."""
+    if section not in TAGS:
+        raise ValueError("Section '{}' does not exist in {}".format(
+            section, TAGS_CONFIG_FILE))
+    if tag not in TAGS[section]:
+        raise ValueError(
+            "Tag '{}' does not exist in section '{}' of {}".format(
+                tag, section, TAGS_CONFIG_FILE))
+    return TAGS[section][tag]
+
+
+def replace_tags(section, tags):
+    """Replace a list of tags with their values."""
+    return tuple(get_tag_value(section, tag) for tag in tags)

esmvaltool/_data_finder.py

@@ -11,7 +11,7 @@
 
 import six
 
-from ._config import get_institutes, get_project_config, replace_mip_fx
+from ._config import get_project_config, replace_mip_fx
 from .cmor.table import CMOR_TABLES
 
 logger = logging.getLogger(__name__)
@@ -242,7 +242,6 @@ def _find_input_files(variable, rootpath, drs, fx_var=None):
 
 def get_input_filelist(variable, rootpath, drs):
     """Return the full path to input files."""
-    variable['institute'] = get_institutes(variable)
     files = _find_input_files(variable, rootpath, drs)
     files = select_files(files, variable['start_year'], variable['end_year'])
     return files
@@ -258,7 +257,6 @@ def get_input_fx_filelist(variable, rootpath, drs):
         var['frequency'] = table.frequency
         realm = getattr(table.get(var['short_name']), 'modeling_realm', None)
         var['modeling_realm'] = realm if realm else table.realm
-        var['institute'] = get_institutes(variable)
 
         files = _find_input_files(var, rootpath, drs, fx_var)
         fx_files[fx_var] = files[0] if files else None
@@ -272,23 +270,23 @@ def get_output_file(variable, preproc_dir):
 
     outfile = os.path.join(
         preproc_dir,
-        '{diagnostic}_{preprocessor}_{short_name}'.format(**variable),
-        _replace_tags(cfg['output_file'], variable)[0] + '.nc')
+        variable['diagnostic'],
+        variable['variable_group'],
+        _replace_tags(cfg['output_file'], variable)[0] + '.nc',
+    )
 
     return outfile
 
 
-def get_statistic_output_file(variable, statistic, preproc_dir):
+def get_statistic_output_file(variable, preproc_dir):
     """Get multi model statistic filename depending on settings."""
-    values = dict(variable)
-    values['stat'] = statistic.title()
-
     template = os.path.join(
         preproc_dir,
-        '{diagnostic}_{preprocessor}_{short_name}',
-        'MultiModel{stat}_{field}_{short_name}_{start_year}-{end_year}.nc',
+        '{diagnostic}',
+        '{variable_group}',
+        '{dataset}_{field}_{short_name}_{start_year}-{end_year}.nc',
     )
 
-    outfile = template.format(**values)
+    outfile = template.format(**variable)
 
     return outfile

esmvaltool/_main.py

@@ -12,6 +12,7 @@
   Benjamin Mueller (LMU, Germany - [email protected])
   Valeriu Predoi (URead, UK - [email protected])
   Mattia Righi (DLR, Germany - [email protected])
+  Manuel Schlund (DLR, Germany - [email protected])
   Javier Vegas-Regidor (BSC, Spain - [email protected])
 
 For further help, please read the documentation at
@@ -39,7 +40,7 @@
 
 from . import __version__
 from ._config import configure_logging, read_config_user_file
-from ._recipe import read_recipe_file
+from ._recipe import read_recipe_file, TASKSEP
 from ._task import resource_usage_logger
 
 # set up logging
@@ -89,6 +90,14 @@ def get_args():
         '--max-years',
         type=int,
         help='Limit the number of years to MAX_YEARS.')
+    parser.add_argument(
+        '--skip-nonexistent',
+        action='store_true',
+        help="Skip datasets that cannot be found.")
+    parser.add_argument(
+        '--diagnostics',
+        nargs='*',
+        help="Only run the named diagnostics from the recipe.")
     args = parser.parse_args()
     return args
 
@@ -129,6 +138,11 @@ def main(args):
     logger.info("Using config file %s", config_file)
     logger.info("Writing program log files to:\n%s", "\n".join(log_files))
 
+    cfg['skip-nonexistent'] = args.skip_nonexistent
+    cfg['diagnostics'] = {
+        pattern if TASKSEP in pattern else pattern + TASKSEP + '*'
+        for pattern in args.diagnostics or ()
+    }
     cfg['synda_download'] = args.synda_download
     for limit in ('max_datasets', 'max_years'):
         value = getattr(args, limit)
@@ -221,6 +235,11 @@ def run():
             "Program terminated abnormally, see stack trace "
             "below for more information",
             exc_info=True)
+        logger.info(
+            "If you suspect this is a bug or need help, please open an issue "
+            "on https://github.com/ESMValGroup/ESMValTool/issues and attach "
+            "the run/recipe_*.yml and run/main_log_debug.txt files from the "
+            "output directory.")
         sys.exit(1)
     else:
         if conf["remove_preproc_dir"]: