Merge remote-tracking branch 'private/version2_development' into version2_more_complex_variable_definition

Author: schlunma

.circleci/config.yml

@@ -5,12 +5,12 @@ jobs:
     # Run Python 3 tests
     working_directory: /python3_test
     docker:
-      - image: continuumio/miniconda
+      - image: continuumio/miniconda3
     steps:
       - checkout
       - run:
           command: |
-            apt-get update -y && apt-get install -y build-essential
+            apt update && apt -y install build-essential gfortran
             # Create a file to checksum as cache key
             date --rfc-3339 date > cache_key.txt
             cat environment.yml >> cache_key.txt
@@ -20,12 +20,10 @@ jobs:
       - run:
           # Update/Create Conda Environment
           command: |
+            . /opt/conda/etc/profile.d/conda.sh
             conda update -y conda
-            if ! test -d "/opt/conda/envs/esmvaltool"; then
-              conda create -y --name esmvaltool python=3
-            fi
-            conda env update --name esmvaltool
-            source activate esmvaltool
+            conda env update
+            conda activate esmvaltool
             Rscript esmvaltool/install/R/setup.R
       - save_cache:
           key: deps3-{{ .Branch }}-{{ checksum "cache_key.txt" }}
@@ -34,7 +32,8 @@ jobs:
       - run:
           # Activate Conda environment and run tests
           command: |
-            source activate esmvaltool
+            . /opt/conda/etc/profile.d/conda.sh
+            conda activate esmvaltool
             python setup.py test
       - store_test_results:
           path: test-reports/
@@ -45,12 +44,12 @@ jobs:
     # Run Python 2 tests
     working_directory: /python2_test
     docker:
-      - image: continuumio/miniconda
+      - image: continuumio/miniconda3
     steps:
       - checkout
       - run:
           command: |
-            apt-get update -y && apt-get install -y build-essential
+            apt update && apt -y install build-essential gfortran
             # Create a file to checksum as cache key
             date --rfc-3339 date > cache_key.txt
             cat environment.yml >> cache_key.txt
@@ -60,12 +59,13 @@ jobs:
       - run:
           # Update/Create Conda Environment
           command: |
+            . /opt/conda/etc/profile.d/conda.sh
             conda update -y conda
             if ! test -d "/opt/conda/envs/esmvaltool"; then
               conda create -y --name esmvaltool python=2
             fi
-            conda env update --name esmvaltool
-            source activate esmvaltool
+            conda env update
+            conda activate esmvaltool
             Rscript esmvaltool/install/R/setup.R
       - save_cache:
           key: deps2-{{ .Branch }}-{{ checksum "cache_key.txt" }}
@@ -74,7 +74,8 @@ jobs:
       - run:
           # Activate Conda environment and run tests
           command: |
-            source activate esmvaltool
+            . /opt/conda/etc/profile.d/conda.sh
+            conda activate esmvaltool
             python setup.py test
       - store_test_results:
           path: test-reports/
@@ -85,22 +86,20 @@ jobs:
     # Test Python 3 installation
     working_directory: /python3_install
     docker:
-      - image: continuumio/miniconda
+      - image: continuumio/miniconda3
     steps:
       - checkout
       - run:
           command: |
+            . /opt/conda/etc/profile.d/conda.sh
             set -x
             mkdir /logs
             # Install
-            apt-get update > /logs/apt.txt 2>&1
-            apt-get install -y build-essential >> /logs/apt.txt 2>&1
+            apt update > /logs/apt.txt 2>&1
+            apt -y install build-essential gfortran >> /logs/apt.txt 2>&1
             conda update -y conda > /logs/conda.txt 2>&1
-            conda create -y --name esmvaltool python=3 > /logs/conda.txt 2>&1
-            conda env update --name esmvaltool >> /logs/conda.txt 2>&1
-            set +x
-            source activate esmvaltool
-            set -x
+            conda env update >> /logs/conda.txt 2>&1
+            set +x; conda activate esmvaltool; set -x
             pip install . > /logs/install.txt 2>&1
             Rscript ./esmvaltool/install/R/setup.R > /logs/R_install.txt 2>&1
             # Log versions
@@ -127,22 +126,21 @@ jobs:
     # Test Python 2 installation
     working_directory: /python2_install
     docker:
-      - image: continuumio/miniconda
+      - image: continuumio/miniconda3
     steps:
       - checkout
       - run:
           command: |
+            . /opt/conda/etc/profile.d/conda.sh
             set -x
             mkdir /logs
             # Install
-            apt-get update > /logs/apt.txt 2>&1
-            apt-get install -y build-essential >> /logs/apt.txt 2>&1
+            apt update > /logs/apt.txt 2>&1
+            apt -y install build-essential gfortran >> /logs/apt.txt 2>&1
             conda update -y conda > /logs/conda.txt 2>&1
             conda create -y --name esmvaltool python=2 > /logs/conda.txt 2>&1
-            conda env update --name esmvaltool >> /logs/conda.txt 2>&1
-            set +x
-            source activate esmvaltool
-            set -x
+            conda env update >> /logs/conda.txt 2>&1
+            set +x; conda activate esmvaltool; set -x
             pip install . > /logs/install.txt 2>&1
             Rscript ./esmvaltool/install/R/setup.R > /logs/R_install.txt 2>&1
             # Log versions
@@ -160,22 +158,20 @@ jobs:
     # Test development installation
     working_directory: /develop
     docker:
-      - image: continuumio/miniconda
+      - image: continuumio/miniconda3
     steps:
       - checkout
       - run:
           command: |
+            . /opt/conda/etc/profile.d/conda.sh
             set -x
             mkdir /logs
             # Install
-            apt-get update > /logs/apt.txt 2>&1
-            apt-get install -y build-essential >> /logs/apt.txt 2>&1
+            apt update > /logs/apt.txt 2>&1
+            apt -y install build-essential gfortran >> /logs/apt.txt 2>&1
             conda update -y conda > /logs/conda.txt 2>&1
-            conda create -y --name esmvaltool python=3 > /logs/conda.txt 2>&1
-            conda env update --name esmvaltool >> /logs/conda.txt 2>&1
-            set +x
-            source activate esmvaltool
-            set -x
+            conda env update >> /logs/conda.txt 2>&1
+            set +x; conda activate esmvaltool; set -x
             pip install -e .[develop] > /logs/install.txt 2>&1
             Rscript ./esmvaltool/install/R/setup.R > /logs/R_install.txt 2>&1
             # Log versions
@@ -193,23 +189,20 @@ jobs:
     # Test building documentation
     working_directory: /doc
     docker:
-      - image: continuumio/miniconda
+      - image: continuumio/miniconda3
     steps:
       - checkout
       - run:
           command: |
+            . /opt/conda/etc/profile.d/conda.sh
             set -x
             mkdir /logs
             # Install
-            apt-get update > /logs/apt.txt 2>&1
-            apt-get install -y build-essential >> /logs/apt.txt 2>&1
-            # TODO: change to Python 3 once issue #218 is fixed.
+            apt update > /logs/apt.txt 2>&1
+            apt -y install build-essential gfortran >> /logs/apt.txt 2>&1
             conda update -y conda > /logs/conda.txt 2>&1
-            conda create -y --name esmvaltool python=2 > /logs/conda.txt 2>&1
-            conda env update --name esmvaltool >> /logs/conda.txt 2>&1
-            set +x
-            source activate esmvaltool
-            set -x
+            conda env update >> /logs/conda.txt 2>&1
+            set +x; conda activate esmvaltool; set -x
             pip install -e .[develop] > /logs/install.txt 2>&1
             # Log versions
             dpkg -l > /logs/versions.txt
@@ -224,57 +217,59 @@ jobs:
     # Test conda build
     working_directory: /esmvaltool
     docker:
-      - image: continuumio/miniconda
+      - image: continuumio/miniconda3
     steps:
       - checkout
       - run:
           command: |
+            . /opt/conda/etc/profile.d/conda.sh
             set -x
             # Install prerequisites
             mkdir /logs
-            apt-get update > /logs/apt.txt 2>&1
-            apt-get install -y build-essential >> /logs/apt.txt 2>&1
+            apt update > /logs/apt.txt 2>&1
+            apt -y install build-essential gfortran >> /logs/apt.txt 2>&1
             conda update -y conda > /logs/conda_base.txt 2>&1
-            conda install -y conda-build >> /logs/conda_base.txt 2>&1
+            conda install -y conda-build conda-verify >> /logs/conda_base.txt 2>&1
             # Log versions
             dpkg -l > /logs/versions.txt
             conda env export -n base > /logs/build_environment.yml
             # Build conda package
             conda build . -c conda-forge -c birdhouse > /logs/build_log.txt
             # Install Python 3 conda package
-            conda create -y --name esmvaltool3 python=3 > /logs/conda_esmvaltool3.txt 2>&1
-            set +x; source activate esmvaltool3; set -x
+            conda create -y --name esmvaltool3 > /logs/conda_esmvaltool3.txt 2>&1
+            set +x; conda activate esmvaltool3; set -x
             conda install -y esmvaltool --use-local -c conda-forge -c birdhouse
             conda env export > /logs/test_environment3.yml
             esmvaltool -h
-            set +x; source deactivate; set -x
+            set +x; conda deactivate; set -x
             # Install Python 2 conda package
             conda create -y --name esmvaltool2 python=2 > /logs/conda_esmvaltool2.txt 2>&1
-            set +x; source activate esmvaltool2; set -x
+            set +x; conda activate esmvaltool2; set -x
             conda install -y esmvaltool --use-local -c conda-forge -c birdhouse
             conda env export > /logs/environment2.yml
             esmvaltool -h
-            set +x; source deactivate; set -x
+            set +x; conda deactivate; set -x
       - store_artifacts:
           path: /logs
 
   conda_install:
     # Test conda package installation
     working_directory: /esmvaltool
     docker:
-      - image: continuumio/miniconda
+      - image: continuumio/miniconda3
     steps:
       - run:
           command: |
+            . /opt/conda/etc/profile.d/conda.sh
             set -x
             # Install prerequisites
             mkdir /logs
-            apt-get update > /logs/apt.txt 2>&1
-            apt-get install -y build-essential >> /logs/apt.txt 2>&1
+            apt update > /logs/apt.txt 2>&1
+            apt -y install build-essential gfortran >> /logs/apt.txt 2>&1
             conda update -y conda > /logs/conda.txt 2>&1
             # Create and activate conda environment
-            conda create -y --name esmvaltool python=3
-            set +x; source activate esmvaltool; set -x
+            conda create -y --name esmvaltool
+            set +x; conda activate esmvaltool; set -x
             # Install
             conda install -y esmvaltool -c esmvalgroup -c conda-forge -c birdhouse
             # Log versions
@@ -287,21 +282,20 @@ jobs:
     # Test ncl conda package
     working_directory: /ncl
     docker:
-      - image: continuumio/miniconda
+      - image: continuumio/miniconda3
     steps:
       - checkout
       - run:
           command: |
+            . /opt/conda/etc/profile.d/conda.sh
             set -x
             mkdir /logs
             # Install
-            apt-get update > /logs/apt.txt 2>&1
-            apt-get install -y build-essential >> /logs/apt.txt 2>&1
+            apt update > /logs/apt.txt 2>&1
+            apt -y install build-essential gfortran >> /logs/apt.txt 2>&1
             conda update -y conda > /logs/conda.txt 2>&1
             conda create -y --name ncl > /logs/conda.txt 2>&1
-            set +x
-            source activate ncl
-            set -x
+            set +x; conda activate ncl; set -x
             conda install -y --channel conda-forge ncl >> /logs/conda.txt 2>&1
             # Log versions
             dpkg -l > /logs/versions.txt

README.md

@@ -15,6 +15,8 @@ This is the development branch for version 2 of ESMValTool. To get started devel
 
 ### Getting started
 To install in development mode, follow these instructions.
+- Install gcc, g++ and gfortran if these are not available on your system. On Debian based systems, this can be done by
+running `apt install build-essential gfortran`.
 - [Download and install conda](https://conda.io/docs/user-guide/install/linux.html) (this should be done even if the system in use already has a preinstalled version of conda, as problems have been reported with NCL when using such a version)
 - If using (t)csh shell, do not prepend the installation path (`<prefix>`) to the environment variable PATH (as recommended by the installation procedure), but add `source <prefix>/etc/profile.d/conda.csh` to the `.cshrc`/`.tcshrc` file instead
 - Update conda: `conda update -y conda`
@@ -31,6 +33,10 @@ To install in development mode, follow these instructions.
 
 ### Anaconda Package
 The Anaconda packages can be found on [ESMValGroup Anaconda Channel.](https://anaconda.org/ESMValGroup)
+
+First install gcc, g++ and gfortran if these are not available on your system. On Debian based systems, this can be done by
+running `apt install build-essential gfortran`.
+
 If you already installed Anaconda, you can install ESMValTool by running:
 ```
 conda install -c esmvalgroup esmvaltool -c conda-forge -c birdhouse

doc/sphinx/source/conf.py

@@ -157,7 +157,7 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = []
 
 # Add any extra paths that contain custom files (such as robots.txt or
 # .htaccess) here, relative to this directory. These files are copied

doc/sphinx/source/developer_guide2/git_repository.inc

@@ -1,5 +1,6 @@
 .. _git_repository:
 
+**************
 Git repository
 **************
 
@@ -161,7 +162,7 @@ Do-s
 * Comment your code as much as possible and in English.
 * Use short but self-explanatory variable names (e.g., model_input and reference_input instead of xm and xr).
 * Consider a modular/functional programming style. This often makes code easier to read and deletes intermediate variables immediately. If possible, separate diagnostic calculations from plotting routines.
-* Consider reusing or extending existing code. General-purpose code can be found in diag_scripts/lib/ and in plot_scripts/.
+* Consider reusing or extending existing code. General-purpose code can be found in esmvaltool/diag_scripts/shared/.
 * Comment all switches and parameters including a list of all possible settings/options in the header section of your code (see also Section :ref:`std_diag`).
 * Use templates for recipes (Section :ref:`std_recipe`) and diagnostics (Section :ref:`std_diag`) to help with proper documentation.
 * Keep your *FEATURE BRANCH* regularly synchronized with the *DEVELOPMENT BRANCH* (git merge).
@@ -174,5 +175,5 @@ Don't-s
 * Do not develop without proper version control (see do-s above).
 * Avoid large (memory, disk space) intermediate results. Delete intermediate files/variables or see modular/functional programming style.
 * Do not use hard-coded pathnames or filenames.
-* Do not mix developments / modifications of the ESMValTool framework and developments / modifications of diagnotics in the same *FEATURE BRANCH*.
+* Do not mix developments / modifications of the ESMValTool framework and developments / modifications of diagnostics in the same *FEATURE BRANCH*.
 

doc/sphinx/source/developer_guide2/index.rst

@@ -2,6 +2,7 @@
 Developer's Guide
 #################
 
+.. include:: new_diagnostic.inc
 .. include:: porting.inc
-.. include:: core_team.inc
 .. include:: git_repository.inc
+.. include:: core_team.inc