initial version of Julia support

.circleci/config.yml

@@ -14,27 +14,22 @@ jobs:
             # Create a file to checksum as cache key
             date --rfc-3339 date > cache_key.txt
             cat environment.yml >> cache_key.txt
-            cat esmvaltool/install/R/r_requirements.txt >> cache_key.txt
       - restore_cache:
           key: deps3-{{ .Branch }}-{{ checksum "cache_key.txt" }}
       - run:
-          # Update/Create Conda Environment
+          # Update/Create Conda environment and run tests
           command: |
             . /opt/conda/etc/profile.d/conda.sh
             conda update -y conda
             conda env update
             conda activate esmvaltool
-            Rscript esmvaltool/install/R/setup.R
+            conda install -yS r-lintr
+            python setup.py test
       - save_cache:
           key: deps3-{{ .Branch }}-{{ checksum "cache_key.txt" }}
           paths:
             - "/opt/conda/envs/esmvaltool"
-      - run:
-          # Activate Conda environment and run tests
-          command: |
-            . /opt/conda/etc/profile.d/conda.sh
-            conda activate esmvaltool
-            python setup.py test
+            - ".eggs"
       - store_test_results:
           path: test-reports/
       - store_artifacts:
@@ -47,6 +42,8 @@ jobs:
       - image: continuumio/miniconda3
     steps:
       - checkout
+      - restore_cache:
+          key: python3-install-{{ .Branch }}
       - run:
           command: |
             . /opt/conda/etc/profile.d/conda.sh
@@ -55,11 +52,15 @@ jobs:
             # Install
             apt update > /logs/apt.txt 2>&1
             apt -y install build-essential gfortran >> /logs/apt.txt 2>&1
+            wget https://julialang-s3.julialang.org/bin/linux/x64/1.0/julia-1.0.3-linux-x86_64.tar.gz
+            tar xfz julia-*-linux-x86_64.tar.gz
+            ln -s $(pwd)/julia-*/bin/julia /usr/bin/julia
             conda update -y conda > /logs/conda.txt 2>&1
             conda env update >> /logs/conda.txt 2>&1
             set +x; conda activate esmvaltool; set -x
             pip install . > /logs/install.txt 2>&1
-            Rscript ./esmvaltool/install/R/setup.R > /logs/R_install.txt 2>&1
+            Rscript esmvaltool/install/R/setup.R > /logs/R_install.txt 2>&1
+            julia esmvaltool/install/Julia/setup.jl > /logs/julia_install.txt 2>&1
             # Log versions
             dpkg -l > /logs/versions.txt
             conda env export > /logs/environment.yml
@@ -73,6 +74,11 @@ jobs:
             echo 0 | cdo -f nc input,r1x1 tmp.nc
             ncdump tmp.nc | ncgen -k hdf5 -o tmp.nc
             cdo -f nc copy tmp.nc tmp2.nc
+      - save_cache:
+          key: python3-install-{{ .Branch }}
+          paths:
+            - "/opt/conda/pkgs"
+            - ".eggs"
       - store_artifacts:
           path: /logs
       - store_artifacts:
@@ -100,11 +106,15 @@ jobs:
             # Install
             apt update > /logs/apt.txt 2>&1
             apt -y install build-essential gfortran >> /logs/apt.txt 2>&1
+            wget https://julialang-s3.julialang.org/bin/linux/x64/1.0/julia-1.0.3-linux-x86_64.tar.gz
+            tar xfz julia-*-linux-x86_64.tar.gz
+            ln -s $(pwd)/julia-*/bin/julia /usr/bin/julia
             conda update -y conda > /logs/conda.txt 2>&1
             conda env update >> /logs/conda.txt 2>&1
             set +x; conda activate esmvaltool; set -x
             pip install -e .[develop] > /logs/install.txt 2>&1
-            Rscript ./esmvaltool/install/R/setup.R > /logs/R_install.txt 2>&1
+            Rscript esmvaltool/install/R/setup.R > /logs/R_install.txt 2>&1
+            julia esmvaltool/install/Julia/setup.jl > /logs/julia_install.txt 2>&1
             # Log versions
             dpkg -l > /logs/versions.txt
             conda env export > /logs/environment.yml
@@ -160,6 +170,9 @@ jobs:
             mkdir /logs
             apt update > /logs/apt.txt 2>&1
             apt -y install build-essential gfortran >> /logs/apt.txt 2>&1
+            wget https://julialang-s3.julialang.org/bin/linux/x64/1.0/julia-1.0.3-linux-x86_64.tar.gz
+            tar xfz julia-*-linux-x86_64.tar.gz
+            ln -s $(pwd)/julia-*/bin/julia /usr/bin/julia
             conda update -y conda > /logs/conda_base.txt 2>&1
             conda install -y conda-build conda-verify >> /logs/conda_base.txt 2>&1
             # Log versions
@@ -190,6 +203,9 @@ jobs:
             mkdir /logs
             apt update > /logs/apt.txt 2>&1
             apt -y install build-essential gfortran >> /logs/apt.txt 2>&1
+            wget https://julialang-s3.julialang.org/bin/linux/x64/1.0/julia-1.0.3-linux-x86_64.tar.gz
+            tar xfz julia-*-linux-x86_64.tar.gz
+            ln -s $(pwd)/julia-*/bin/julia /usr/bin/julia
             conda update -y conda > /logs/conda.txt 2>&1
             # Create and activate conda environment
             conda create -y --name esmvaltool

CONTRIBUTING.md

@@ -19,6 +19,7 @@ To install in development mode, follow these instructions.
   - Update the esmvaltool conda environment `conda env update`
   - Install in development mode: `pip install -e '.[develop]'`. If you are installing behind a proxy that does not trust the usual pip-urls you can declare them with the option `--trusted-host`, e.g. `pip install --trusted-host=pypi.python.org --trusted-host=pypi.org --trusted-host=files.pythonhosted.org -e .[develop]`
   - If you want to use R diagnostics, run `Rscript esmvaltool/install/R/setup.R` to install the R dependences.
+  - If you want to use Julia diagnostics, run `julia esmvaltool/install/Julia/setup.jl` to install the Julia dependences.
   - Test that your installation was succesful by running `esmvaltool -h`.
 
 ## Running tests

README.md

@@ -21,7 +21,8 @@ Please see [CONTRIBUTING.md](https://github.com/ESMValGroup/ESMValTool/blob/vers
 The Anaconda package can be found on [ESMValGroup Anaconda Channel.](https://anaconda.org/ESMValGroup)
 
 First install gcc, g++ and gfortran if these are not available on your system. On Debian based systems, this can be done by
-running `apt install build-essential gfortran`.
+running `apt install build-essential gfortran`. In order to run
+Julia diagnostics, you will also need to [install Julia](https://julialang.org/downloads/) version 1 or greater.
 
 If you already installed Anaconda, you can install ESMValTool by running:
 ```

doc/sphinx/source/user_guide2/install.inc

@@ -148,6 +148,13 @@ the following command:
     cd ESMValTool
     python setup.py install
 
+If you would like to run Julia diagnostic scripts, you will also need to
+`install Julia <https://julialang.org/downloads/>`_ and the Julia dependencies:
+
+.. code-block:: bash
+
+    julia ESMValTool/esmvaltool/install/Julia/setup.jl
+
 If you would like to run R diagnostic scripts, you will also need to install the R
 dependencies:
 

environment.yml

@@ -21,9 +21,9 @@ channels:
 dependencies:
   # Python packages that cannot be installed from PyPI:
   - iris
+  - esmpy
   - matplotlib<3  # Can be installed from PyPI, but is a dependency of iris and should be pinned.
   - python-stratify
-  - esmpy
   - xarray  # Can be installed from PyPI, but here to get a consistent set of depencies with iris.
   # Non-Python dependencies
   - graphviz
@@ -34,4 +34,4 @@ dependencies:
   - jasper!=1.900.31  # pinned NCL dependency
   - r-base
   - libunwind  # Needed for Python3.7+
-  # TODO: add julia
+  # - julia>=1.0.0  # The Julia package on conda is apparently broken

esmvaltool/_task.py

@@ -282,6 +282,7 @@ def _initialize_cmd(self, script):
                     'py': [which('python')],
                     'ncl': [which('ncl'), '-n', '-p'],
                     'r': [which('Rscript')],
+                    'jl': [which('julia')],
                 }
             else:
                 profile_file = os.path.join(self.settings['run_dir'],
@@ -293,6 +294,7 @@ def _initialize_cmd(self, script):
                     ],
                     'ncl': [which('ncl'), '-n', '-p'],
                     'r': [which('Rscript')],
+                    'jl': [which('julia')],
                 }
 
             if extension not in executables:

esmvaltool/install/Julia/julia_requirements.txt

@@ -0,0 +1,2 @@
+DataFrames
+YAML

esmvaltool/install/Julia/setup.jl

@@ -0,0 +1,29 @@
+#!/usr/bin/env julia
+
+scriptDir=@__DIR__
+
+if VERSION >= v"0.7.0-DEV.2005"
+    using Pkg
+end
+
+@info "Installing the packages from" scriptDir * "/julia_requirements.txt"
+pkgName=in
+open(scriptDir * "/julia_requirements.txt") do f
+    for i in enumerate(eachline(f))
+
+      pkgId=i[1]
+      pkgName=i[2]
+      @info "Installing" pkgName
+      Pkg.add(pkgName)
+
+      @info "Testing: ", pkgName
+      # load the package this needs to be called at top-level
+      Expr(:toplevel, :(module ($pkgName) end))
+
+    end
+end
+
+# Show the package list
+@info "Installed Julia packages:"
+Pkg.installed()
+Pkg.status()

language_support.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-tools=(R Rscript)
+tools=(R Rscript julia)
 
 for tool in "${tools[@]}"; do
     if ! command -v "$tool" > /dev/null 2>&1; then
@@ -10,3 +10,4 @@ for tool in "${tools[@]}"; do
 done
 
 Rscript $PREFIX/lib/python*/site-packages/esmvaltool/install/R/setup.R >> $PREFIX/.messages.txt
+julia $PREFIX/lib/python*/site-packages/esmvaltool/install/Julia/setup.jl >> $PREFIX/.messages.txt

meta.yaml

@@ -72,7 +72,6 @@ requirements:
     - jasper!=1.900.31  # pinned NCL dependency
     - r-base
     - cdo
-    # TODO: add julia
 
 test:
   # TODO: add unit tests? This seems to require installing the tests

tests/integration/test_diagnostic_run.py

@@ -92,6 +92,17 @@ def main(cfg):
         print(paste0("INFO    Writing settings to ", settings$setting_name))
         yaml::write_yaml(settings, settings$setting_name)
         """),
+    'diagnostic.jl':
+    dedent("""
+        import YAML
+        @info "Starting diagnostic script with" ARGS
+        config_file = ARGS[1]
+        cfg = YAML.load_file(config_file)
+        out_file = cfg["setting_name"]
+        @info "Copying file to" out_file
+        Base.Filesystem.cp(config_file, out_file)
+        @info "Done"
+    """),
 }