Merge pull request #558 from ESMValGroup/version2_development

Version 2.0a1 release

.circleci/config.yml

@@ -245,6 +245,31 @@ jobs:
       - store_artifacts:
           path: /logs
 
+  conda_install:
+    # Test conda package installation
+    working_directory: /esmvaltool
+    docker:
+      - image: continuumio/miniconda
+    steps:
+      - run:
+          command: |
+            set -x
+            # Install prerequisites
+            mkdir /logs
+            apt-get update > /logs/apt.txt 2>&1
+            apt-get install -y build-essential >> /logs/apt.txt 2>&1
+            conda update -y conda > /logs/conda.txt 2>&1
+            # Create and activate conda environment
+            conda create -y --name esmvaltool python=3
+            set +x; source activate esmvaltool; set -x
+            # Install
+            conda install -y esmvaltool -c esmvalgroup -c conda-forge -c birdhouse
+            # Log versions
+            conda env export > /logs/environment.yml
+            # Test installation
+            esmvaltool -h
+            ncl -V
+
   ncl:
     # Test ncl conda package
     working_directory: /ncl
@@ -296,4 +321,5 @@ workflows:
       - develop
       - doc
       - conda_build
+      - conda_install
       - ncl

.gitignore

@@ -43,8 +43,12 @@ __pycache__/
 .pydevproject
 .settings
 
+#Create by VSCode
+.vscode
+
 #pytest
 .cache
+.pytest_cache
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -82,3 +86,13 @@ test-reports/
 doc/sphinx/source/diag_scripts/**
 doc/sphinx/source/plot_scripts/**
 doc/sphinx/build
+
+# Data files
+*.nc
+*.nc4
+*.grb
+*.grib
+*.RData
+*.Rdata
+*.rdata
+*.pkl

.pylintrc

@@ -65,7 +65,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating
+disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,import-error
 
 
 [REPORTS]

NOTICE

@@ -30,37 +30,37 @@ und Raumfahrt e.V. (DLR), ESMValTool Project PI “Entire ESMValTool”
 Copyright 2011- Swedish Meteorological and Hydrological Institute (SMHI) “Entire
 ESMValTool”
 
-British Crown Copyright 2015 “namelist_SAMonsoon.xml, namelist_SAMonsoon_AMIP.xml,
-namelist_SAMonsoon_daily.xml, SPHINX”
+British Crown Copyright 2015 “recipe_SAMonsoon.yml, recipe_SAMonsoon_AMIP.yml,
+recipe_SAMonsoon_daily.yml, SPHINX”
 
-Copyright 2014-2015 University of Exeter “Carbon Cycle “namelist_anav13jclim.xml”
+Copyright 2014-2015 University of Exeter “Carbon Cycle “recipe_anav13jclim.yml”
 
 Copyright 2014-2015 Tyndall Centre for Climate Change Research, School of Environmental
-Sciences, University of East Anglia, Norwich “namelist_GlobalOcean.xml”
+Sciences, University of East Anglia, Norwich “recipe_GlobalOcean.yml”
 
-Copyright 2015 Agenzia nazionale per le nuove tecnologie, l’energia e lo sviluppo economico sostenibile (ENEA) “Ozone (namelist_eyring13jgr.xml; namelist_eyring06jgr.xml)
+Copyright 2015 Agenzia nazionale per le nuove tecnologie, l’energia e lo sviluppo economico sostenibile (ENEA) “Ozone (recipe_eyring13jgr.yml; recipe_eyring06jgr.yml)
 
-Copyright 2014-2015 ETH Zurich, Switzerland “namelist_Evapotransport.xml, namelist_SPI.xml”
+Copyright 2014-2015 ETH Zurich, Switzerland “recipe_Evapotransport.yml, recipe_SPI.yml”
 
-Copyright 2015 University Corporation for Atmospheric Research (UCAR) “Climate Variability Diagnostic Package (namelist_CVDP.xml)”
+Copyright 2015 University Corporation for Atmospheric Research (UCAR) “Climate Variability Diagnostic Package (recipe_CVDP.yml)”
 
 Copyright 2015 Deutsches Klimarechenzentrum (DKRZ) “ESGF coupling“
 
-Copyright 2015 Max-Planck-Institute for Meteorology (MPI) “namelist_runoff_et.xml”
+Copyright 2015 Max-Planck-Institute for Meteorology (MPI) “recipe_runoff_et.yml”
 
 Copyright 2015 National Centre for Atmospheric Science, British Atmospheric Data Centre, STFC Rutherford Appleton Laboratory “ESGF coupling”
 
-Copyright 2015-2016 Ludwig Maximilians University Munich (LMU) “Automated Testing”, contributions to “namelist_lauer17rse.xml”
+Copyright 2015-2016 Ludwig Maximilians University Munich (LMU) “Automated Testing”, contributions to “recipe_lauer17rse.yml”
 
-Copyright 2015 Finnish Meteorological Institute (FMI) “namelist_TropicalVariability.xml, namelist_SouthernHemisphere.xml”
+Copyright 2015 Finnish Meteorological Institute (FMI) “recipe_TropicalVariability.yml, recipe_SouthernHemisphere.yml”
 
-Copyright 2015 Institut Pierre Simon Laplace, “namelist_diurnalcycle.xml”
+Copyright 2015 Institut Pierre Simon Laplace, “recipe_diurnalcycle.yml”
 
 Copyright 2015 University of Reading “ESGF coupling”
 
-Copyright 2015 CNRM-GAME, Météo France and CNRS, “namelist_WAMonsoon.xml”
+Copyright 2015 CNRM-GAME, Météo France and CNRS, “recipe_WAMonsoon.yml”
 
-Copyright 2014-2015 Royal Netherlands Meteorological Institute (KNMI) “namelist_SouthernOcean.xml”
+Copyright 2014-2015 Royal Netherlands Meteorological Institute (KNMI) “recipe_SouthernOcean.yml”
 
 Copyright 2008- 2012 UCAR “Chemistry-Climate Model Validation (CCMVal) Diagnostic Tool”
 

README.md

@@ -13,18 +13,19 @@ This is the development branch for version 2 of ESMValTool. To get started devel
 
 ### Getting started
 To install in development mode, follow these instructions.
-- [Download and install conda](https://conda.io/docs/user-guide/install/linux.html)
+- [Download and install conda](https://conda.io/docs/user-guide/install/linux.html) (this should be done even if the system in use already has a preinstalled version of conda, as problems have been reported with NCL when using such a version)
+- If using (t)csh shell, do not prepend the installation path (`<prefix>`) to the environment variable PATH (as recommended by the installation procedure), but add `source <prefix>/etc/profile.d/conda.csh` to the `.cshrc`/`.tcshrc` file instead
 - Update conda: `conda update -y conda`
 - Create a conda environment: `conda create -y -n esmvaltool python=3`
-- Activate the esmvaltool environment: `source activate esmvaltool`
+- Activate the esmvaltool environment: `source activate esmvaltool` (or `conda activate esmvaltool` if using (t)csh shell)
 - Clone the ESMValTool github repository: `git clone [email protected]/ESMValGroup/ESMValTool`
 - Go to the esmvaltool directory: `cd ESMValTool`
 - Check out the version 2 development branch: `git checkout version2_development`
 - Update the esmvaltool conda environment `conda env update`
-- Install in development mode: `pip install -e .[develop]`
+- Install in development mode: `pip install -e .[develop]` (or `pip install -e '.[develop]'` if using (t)csh shell)
 - Test that your installation was succesful by running `esmvaltool -h`.
 - Review `config-user.yml`. To customize for your system, create a copy, edit and use the command line option `-c` to instruct `esmvaltool` to use your custom configuration.
-- Available namelists are located in the directory `esmvaltool/namelists`.
+- Available recipes are located in the directory `esmvaltool/recipes`.
 
 ### Running tests
 Go to the directory where the repository is cloned and run `./setup.py test`. Tests will also be run automatically by CircleCI.

RELEASE_NOTES

@@ -18,10 +18,10 @@ GENERAL
 
 DIAGNOSTIC AND PLOT ROUTINES
 - added sea-ice, xco2 and soil moisture to perfmetrics
-- added optional uncertainty estimates to Taylor diagrams (namelist_lauer17rse.xml)
+- added optional uncertainty estimates to Taylor diagrams (recipe_lauer17rse.yml)
 - new ESA CCI diagnostics for SST, soil moisture, land cover, xco2
-  (namelist_lauer17rse.xml)
-- added IPCC ch.9 figs. 9.28 and 9.30 (namelist_flato13ipcc.xml)
+  (recipe_lauer17rse.yml)
+- added IPCC ch.9 figs. 9.28 and 9.30 (recipe_flato13ipcc.yml)
 - new variables including error estimates for extisting variables (<varname>Stderr)
 - added summary plot (various campaigns) for aerosol vertical profiles
 
@@ -37,28 +37,28 @@ MINOR CHANGES AND BUG FIXES
 - remove inappropriate conversion (C-->K) from recognized_units.dat 
   (only multiplicative factors are allowed)
 - added depth info for mrsos in reformat
-- added diagnostic for calculating multi-model means (namelist_mmm.xml)
+- added diagnostic for calculating multi-model means (recipe_mmm.yml)
 
 ===============================================================================
 Version 1.0.1 - June 2016
 ===============================================================================
 GENERAL
 - paths to workdir, climodir, plotdir, model and observational data can now be
-  set in a single configuration file (config_private.xml) and included in the
-  main namelists using the @{} syntax (e.g., @{MODELPATH}/CMIP5/MPI-ESM-LR/...)
-- reformat scripts for the observations can now be defined in a main namelist
-  (namelist_reformat_obs.xml) and passed to main.py with the -r option:
-    python main.py -r nml/namelist_reformat_obs.xml
+  set in a single configuration file (config_private.yml) and included in the
+  main recipes using the @{} syntax (e.g., @{MODELPATH}/CMIP5/MPI-ESM-LR/...)
+- reformat scripts for the observations can now be defined in a main recipe
+  (recipe_reformat_obs.yml) and passed to main.py with the -r option:
+    python main.py -r nml/recipe_reformat_obs.yml
 
 DIAGNOSTIC AND PLOT ROUTINES
 - new diagnostics for precipitation over soil moisture anomalies in
-  namelist_sm_pr.xml
+  recipe_sm_pr.yml
 - enabled detrending in nml/cfg_GO/spco2compmapDiag.ncl
-- added od870aer, abs550aer, od550lt1aer, toz  to namelist_perfmetrics_CMIP5
+- added od870aer, abs550aer, od550lt1aer, toz  to recipe_perfmetrics_CMIP5
 - added display name wrapper to all(?) model entries for the Tropical
   Variability diagnostics
 - added option to alternative panelling of Tropical variability scatter plots
-- added more options for plotting in namelist_aerosol_EMAC.xml
+- added more options for plotting in recipe_aerosol_EMAC.yml
 
 OBSERVATIONS
 - added reformat_obs_ACCESS-2 and reformat_obs_HadCRUT4
@@ -67,16 +67,16 @@ OBSERVATIONS
   od550lt1aer
 
 MINOR CHANGES AND BUG FIXES
-- cleaned up config files for namelist_perfmetrics_CMIP5
+- cleaned up config files for recipe_perfmetrics_CMIP5
 - added fix file for CMIP5_CNRM-CM5
-- added dpi option to namelist for Python based diagnostics
+- added dpi option to recipe for Python based diagnostics
 - added routines to use external file for model display names in Python
 - added functionality to switch display names in Python plots
 - small generalization and bugfix in logging routine in esmval_lib
 - added functionality for mapping model name onto display model names for 
   figure headings
-- added explanatory comment how to run the pybot easytest namelist test cases
-- added namelist switch to control PNG figure resolution from namelist
+- added explanatory comment how to run the pybot easytest recipe test cases
+- added recipe switch to control PNG figure resolution from recipe
 - generalized handling of styles with Python diagnostics