Merge pull request #586 from UW-Hydro/develop

Release PR for VIC.5.0.0.rc2

Author: Joe Hamman

.gitignore

@@ -24,16 +24,23 @@ vic_headers.py
 *.mod
 
 vicNl
-vic_classic
-vic_image
+vic_classic.exe
+vic_image.exe
 liblnd.a
 _vic.py
+vic_timing_*txt
+massif.out.*
+vic_call_graph_*
+
+vic_call_graph*
+gmon.out
 
 .DS_Store
 TAGS
 
 *egg-info*
 
 # Python stuff
+__pycache__
 .ipynb_checkpoints
 .eggs

.travis.yml

@@ -13,6 +13,8 @@ matrix:
     addons:
       apt_packages:
         - libnetcdf-dev
+        - netcdf-bin
+        - netcdf-doc
         - gfortran
         - valgrind
         # open-mpi is built from source in image.travis
@@ -23,6 +25,8 @@ matrix:
     addons:
       apt_packages:
         - libnetcdf-dev
+        - netcdf-bin
+        - netcdf-doc
         - gfortran
         # open-mpi is built from source in cesm.travis
   # Classic Driver

ci/cesm.travis

@@ -1,7 +1,5 @@
 #!/usr/bin/env bash
 
-set -e
-
 DRIVER_PATH="./vic/drivers/cesm/"
 DRIVER_LIB="${DRIVER_PATH}lndlib.a"
 

ci/classic.travis

@@ -1,7 +1,5 @@
 #!/usr/bin/env bash
 
-set -e
-
 DRIVER_PATH="${TRAVIS_BUILD_DIR}/vic/drivers/classic/"
 DRIVER_EXE="${DRIVER_PATH}vic_classic.exe"
 SAMPLES_PATH="${TRAVIS_BUILD_DIR}/samples/"
@@ -59,10 +57,8 @@ function vic_script {
 
     # Run test package
     ./tests/run_tests.py unit examples system \
-        --vic_exe=${DRIVER_EXE} \
-        --driver=${TESTID} \
+        --classic=${DRIVER_EXE} \
         --data_dir=${SAMPLES_PATH}/data
-
 }
 
 function vic_after_success {

ci/image.travis

@@ -1,7 +1,5 @@
 #!/usr/bin/env bash
 
-set -e
-
 DRIVER_PATH="${TRAVIS_BUILD_DIR}/vic/drivers/image/"
 DRIVER_EXE="${DRIVER_PATH}vic_image.exe"
 SAMPLES_PATH="${TRAVIS_BUILD_DIR}/samples/"
@@ -34,23 +32,29 @@ function vic_before_install {
             brew install homebrew/versions/gcc49
             brew reinstall open-mpi --build-from-source
         fi
-		export TRAVIS_NETCDFPATH=/usr/local
-		export TRAVIS_MPIPATH=/usr/local
+        export TRAVIS_MPIPATH=/usr/local
+        export NC_LIBS=$(/usr/local/bin/nc-config --libs)
+        export NC_CFLAGS=$(/usr/local/bin/nc-config --cflags)
     else
-		export TRAVIS_NETCDFPATH=/usr
 		export TRAVIS_MPIPATH="${HOME}/mpich"
+        export NC_LIBS=$(/usr/bin/nc-config --libs)
+        export NC_CFLAGS=$(/usr/bin/nc-config --cflags)
 
         # Install MPICH
         if [ ! -d ${TRAVIS_MPIPATH} ]; then
             install_mpich
         else
             echo "MPICH installed..."
         fi
-        find ${TRAVIS_MPIPATH} -name mpiexec
-        find ${TRAVIS_MPIPATH} -name mpicc
 	fi
-    export PATH=$TRAVIS_MPIPATH/bin:$PATH
+    export PATH=${TRAVIS_MPIPATH}/bin:${PATH}
+    find ${TRAVIS_MPIPATH} -name mpiexec
     mpiexec --version
+    find ${TRAVIS_MPIPATH} -name mpicc
+    export MPICC="${TRAVIS_MPIPATH}/bin/mpicc"
+    $MPICC --version
+    echo "NETCDF LIB FLAGS ------> ${NC_LIBS}"
+    echo "NETCDF C FLAGS --------> ${NC_CFLAGS}"
 }
 
 function vic_install {
@@ -79,10 +83,8 @@ function vic_script {
 
     # Run test package
     ./tests/run_tests.py unit examples system \
-        --vic_exe=${DRIVER_EXE} \
-        --driver=${TESTID} \
+        --image=${DRIVER_EXE} \
         --data_dir=${SAMPLES_PATH}/data
-
 }
 
 function vic_after_success {

ci/requirements.yml

@@ -10,8 +10,12 @@ dependencies:
   - configobj
   - xarray
   - bottleneck
+  - matplotlib
+  - seaborn
+  - psutil
   - pip:
     - git+https://github.com/UW-Hydro/tonic.git
     - engarde
     - pytest-faulthandler
     - pytest-xdist
+    - gprof2dot

ci/vic_install_utils

@@ -1,16 +1,16 @@
 #!/usr/bin/env bash
 
-set -e
-
 if [ -z "$WORKDIR" ]; then
     export WORKDIR=$HOME/workdir
     mkdir -p $WORKDIR
 fi
 
+export VIC_VALGRIND_SUPPRESSIONS=${TRAVIS_BUILD_DIR}/tests/vic_valgrind_suppressions.supp
+
 function install_mpich {
-    wget --no-check-certificate -q http://www.mpich.org/static/downloads/3.2/mpich-3.2.tar.gz
-    tar -xzf mpich-3.2.tar.gz
-    cd mpich-3.2
+    wget --no-check-certificate -q http://www.mpich.org/static/downloads/3.1b1/mpich-3.1b1.tar.gz
+    tar -xzf mpich-3.1b1.tar.gz
+    cd mpich-3.1b1
     mkdir build && cd build
     ../configure CC=$CC CXX=$CXX --prefix=${TRAVIS_MPIPATH}
     make -j2

docs/Development/ReleaseNotes.md

@@ -15,9 +15,9 @@ For VIC 5 and later, type `vic _{classic,image}.exe -v`
 
 ------------------------------
 
-## VIC 5.0.0 (Release Candidate 1)
+## VIC 5.0.0 (Release Candidate 2)
 
-**Release date: June 28, 2016**
+**Release date: August 18, 2016**
 
 This is a major update from VIC 4. The VIC 5.0.0 release aims to have nearly identical physics as VIC 4.2 while providing a clean, refactored code base supporting multiple drivers. There are a number of new features, bug fixes, and backward incompatible changes. See the VIC Github page for more details on the changes included in this release.
 
@@ -60,19 +60,22 @@ This is a major update from VIC 4. The VIC 5.0.0 release aims to have nearly ide
 
 9. Tests Datasets ([GH#79](https://github.com/UW-Hydro/VIC/issues/79))
 
-	**TODO:** See https://github.com/UW-Hydro/VIC/issues/79 for more information.
+	See https://github.com/UW-Hydro/VIC/issues/79 for more information. A temporary location of the test data is here: ftp://ftp.hydro.washington.edu/pub/gergel/VIC5_test_data/
 
 10. Testing and Continuous Integration ([GH#190](https://github.com/UW-Hydro/VIC/pull/190))
 
 	A comprehensive testing platform has been implemented and is available for public use along with the VIC model. A small subset of the test platform is run on [Travis-CI](https://travis-ci.org/UW-Hydro/VIC), which facilitates continuous integration of the VIC test platform. More information on the test platform is [here](Testing.md)
 
+11. Run-time profiling and timing ([GH#442](https://github.com/UW-Hydro/VIC/pull/442))
+
+	A timing module has been added to VIC in order to assess the computational cost and throughput of the VIC model. New output variables (`OUT_TIME_VICRUN_WALL` and `OUT_TIME_VICRUN_CPU`) document the time spent in `vic_run` for each variable. Additionally, a timing table is printed to `LOG_DEST` at the end of each simulation.
+
 #### Backwards Incompatible Changes:
 
 1.  Classic Driver I/O Formatting ([GH#18](https://github.com/UW-Hydro/VIC/issues/18), [GH#204](https://github.com/UW-Hydro/VIC/issues/204), [GH#227](https://github.com/UW-Hydro/VIC/pull/227))
 
 	The format of ASCII forcing and output files has changed in VIC 5. These changes were motivated by the desire to improve simulation metadata tracking and reproducibility of VIC simulations.
 
-	- Forcing files now require date stamps for each timestep and a header specifies the names of the forcing variables.   
 	- Output files now include a header with simulation metadata and variable names. The `PRT_HEADER` option has been deprecated.
 
 2.  Classic Driver Global Parameter Options
@@ -155,18 +158,26 @@ This is a major update from VIC 4. The VIC 5.0.0 release aims to have nearly ide
 
 3. Fix related to exact restart ([GH#481](https://github.com/UW-Hydro/VIC/pull/481), [GH#507](https://github.com/UW-Hydro/VIC/pull/507), [GH#509](https://github.com/UW-Hydro/VIC/pull/509))
 
-	Previously, VIC did not produce the same results (fluxes and states) if a simulation is separated into multiple shorter-period runs by saving the state variables and restarting. This was due to: 1) the MTCLIM algorithm resulted in slightly different sub-daily meteorological variable values for different length of forcing (MTCLIM is deprecated in the current version); 2) a few bugs resulting in inexact restart. The following bugs have been fixed:
+	Previously, VIC did not produce the same results (fluxes and states) if a simulation was separated into multiple shorter-period runs by saving the state variables and restarting. This was due to:
+		1. The MTCLIM algorithm resulted in slightly different sub-daily meteorological variable values for different lengths of forcings (MTCLIM is deprecated in the current version)
+		2. A few bugs resulting in inexact restart.
+
+	The following bugs have been fixed:
 
 	- The prognostic state variable `energy.Tfoliage` (foliage temperature) is now saved to the state file
 	- Two flux variables `energy.LongUnderOut` and `energy.snow_flux` are now saved to the state file.
 
 		!!!Note
-				This is a temporary solution to ensure exact restart. A better way of handling the two flux variables needs to be done in the future (see [GH#479](https://github.com/UW-Hydro/VIC/issues/479))
+				This is a temporary solution to ensure exact restart. A better way of handling these two flux variables needs to be done in the future (see [GH#479](https://github.com/UW-Hydro/VIC/issues/479))
 
 4. Fix for binary state file I/O ([GH#487](https://github.com/UW-Hydro/VIC/pull/487))
 
 	Fixed a bug so that the binary format state file I/O works correctly.
 
+5. Fix for a physical constant (water heat capacity) ([GH#574](https://github.com/UW-Hydro/VIC/pull/574))
+
+	Fixed a bug where volumetric heat capacity of water should be used in `func_canopy_energy_bal` (previously specific heat capacity was used).
+
 ------------------------------
 
 ## VIC 4.2.c [![DOI](https://zenodo.org/badge/doi/10.5281/zenodo.35302.svg)](http://dx.doi.org/10.5281/zenodo.35302)

docs/Documentation/Drivers/Image/RunVIC.md

@@ -1,7 +1,7 @@
 # Running the VIC Image Driver
 
 ## Dependencies:
-The Image Driver's has three dependencies:
+The Image Driver has three dependencies:
 
 1. A C compiler.  We routinely test VIC using the following compilers:
 
@@ -22,9 +22,16 @@ The Image Driver's has three dependencies:
 ## Compiling
 In most cases, you will need to edit the `NETCDF_PATH` and `MPI_PATH` variables in the `Makefile`.
 
-If you want to use a compiler other than `gcc`, either edit the Makefile or set the `CC` environment variable, e.g.
+If you want to use a compiler other than `mpicc`, either edit the Makefile or set the `MPICC` environment variable, e.g.
 
-        export CC=icc
+        MPICC=/path/to/mpi_c_compiler
+
+The flags and libraries required to compile VIC with netCDF are automatically determined in the `Makefile`.  They can be overwritten by setting the following two environment variables.  These variables can be determined by running `nc-config --all`.
+
+        NC_LIBS="-L/path/to/libs ..."
+        NC_CFLAGS="-I/path/to/includes -your_c_flags ..."
+
+In some versions of the MPI library (e.g. OPEN-MPI with Intel), you may also need to set the environment variable `MX_RCACHE=2` prior to compiling.
 
 - Change directory, `cd`, to the "Image Driver" source code directory and type `make`
 
@@ -37,10 +44,16 @@ If you want to use a compiler other than `gcc`, either edit the Makefile or set
 
 At the command prompt, type:
 
-`vic_image.exe -g global_parameter_filename`
+`vic_image.exe -g global_parameter_filename.txt`
 
 where `global_parameter_filename` = name of the global parameter file corresponding to your project.
 
+To run VIC image driver using multiple processor, type the following instead:
+
+`mpiexec -np n_proc vic_image.exe -g global_parameter_filename.txt`
+
+where `n_proc` = number of processors to be used
+
 ## Other Command Line Options
 
 VIC has a few other command line options:

samples/global.param.sample.image.txt

@@ -139,22 +139,19 @@ WIND_H        10.0                # height of wind speed measurement (m)
 #######################################################################
 # Land Surface Files and Parameters
 #######################################################################
-SOIL            (put the soil parameter path/file here)	# Soil parameter path/file
+PARAMETERS             (put the parameters path/file here)	# Parameters path/file
 BASEFLOW	ARNO	# ARNO = columns 5-8 are the standard VIC baseflow parameters; NIJSSEN2001 = columns 5-8 of soil file are baseflow parameters from Nijssen et al (2001)
 JULY_TAVG_SUPPLIED	FALSE	# TRUE = final column of the soil parameter file will contain average July air temperature, for computing treeline; this will be ignored if COMPUTE_TREELINE is FALSE; FALSE = compute the treeline based on the average July air temperature of the forcings over the simulation period
 ORGANIC_FRACT	FALSE	# TRUE = simulate organic soils; soil param file contains 3*Nlayer extra columns, listing for each layer the organic fraction, and the bulk density and soil particle density of the organic matter in the soil layer; FALSE = soil param file does not contain any information about organic soil, and organic fraction should be assumed to be 0
-VEGPARAM        (put the veg parameter path/file here)	# Veg parameter path/file
-ROOT_ZONES      3	# Number of root zones (must match format of veg param file)
 #LAI_SRC 	FROM_VEGPARAM    # FROM_VEGPARAM = read LAI from veg param file; FROM_VEGHIST = read LAI from veg_hist forcing file
 #ALB_SRC 	FROM_VEGPARAM    # FROM_VEGPARAM = read ALBEDO from veg param file; FROM_VEGHIST = read ALBEDO from veg_hist forcing file
 #FCAN_SRC 	FROM_DEFAULT     # FROM_DEFAULT = set all fcanopy values to 1.0; FROM_VEGPARAM = read FCANOPY from veg param file; FROM_VEGHIST = read FCANOPY from veg_hist forcing file
-SNOW_BAND	1	# Number of snow bands; if number of snow bands > 1, you must insert the snow band path/file after the number of bands (e.g. SNOW_BAND 5 my_path/my_snow_band_file)
+SNOW_BAND	TRUE	# TRUE if VIC should use snowbands in PARAMETERS file, else FALSE.
 
 #######################################################################
 # Lake Simulation Parameters
-# These need to be un-commented and set to correct values only when running lake model (LAKES is not FALSE)
 #######################################################################
-#LAKES		(put lake parameter path/file here)	# Lake parameter path/file
+#LAKES	TRUE	# TRUE if VIC should use the lake module, else FALSE. Requires lake parameters in the PARAMETERS file.
 #LAKE_PROFILE	FALSE	# TRUE = User-specified depth-area parameters in lake parameter file; FALSE = VIC computes a parabolic depth-area profile
 
 #######################################################################

tests/examples/domain.stehekin.20151028.nc

@@ -15,7 +15,7 @@ FROZEN_SOIL	FALSE
 
 AERO_RESIST_CANSNOW AR_406
 
-DOMAIN $test_data_dir/image/Stehekin/parameters/domain.stehekin.20151028.nc
+DOMAIN         $test_data_dir/image/Stehekin/parameters/domain.stehekin.20151028.nc
 DOMAIN_TYPE    LAT     lat
 DOMAIN_TYPE    LON     lon
 DOMAIN_TYPE    MASK    mask
@@ -24,22 +24,19 @@ DOMAIN_TYPE    FRAC    frac
 DOMAIN_TYPE    YDIM    lat
 DOMAIN_TYPE    XDIM    lon
 
-FORCING1	  $test_data_dir/image/Stehekin/forcings/Stehekin_image_test.forcings_10days.
+FORCING1	    $test_data_dir/image/Stehekin/forcings/Stehekin_image_test.forcings_10days.
 FORCE_TYPE    AIR_TEMP 	   tas
 FORCE_TYPE    PREC 	       prcp
 FORCE_TYPE    PRESSURE 	   pres
 FORCE_TYPE    SWDOWN       dswrf
-FORCE_TYPE    LWDOWN 	   dlwrf
+FORCE_TYPE    LWDOWN 	     dlwrf
 FORCE_TYPE    VP           shum
 FORCE_TYPE    WIND         wind
 WIND_H        10.0
 
-SOIL            $test_data_dir/image/Stehekin/parameters/Stehekin_test_params_20160327.nc
+PARAMETERS      $test_data_dir/image/Stehekin/parameters/Stehekin_test_params_20160327.nc
 NODES           3
-VEGPARAM        $test_data_dir/image/Stehekin/parameters/Stehekin_test_params_20160327.nc
-ROOT_ZONES      3
-VEGLIB	        $test_data_dir/image/Stehekin/parameters/Stehekin_test_params_20160327.nc
-SNOW_BAND	    1 $test_data_dir/image/Stehekin/parameters/Stehekin_test_params_20160327.nc
+SNOW_BAND       TRUE
 
 RESULT_DIR  $result_dir
 

tests/examples/global_param.image.STEHE.feb.txt

@@ -0,0 +1,4 @@
+VIC Profiling Tests
+=======
+
+These tests quantify the performance of VIC in terms of CPU/wall time and memory usage.