feat: build option --no-aux to disable downloading of all auxiliary data (#1374)

Author: c-dilks

.github/workflows/ci.yml

@@ -172,7 +172,7 @@ jobs:
       - name: untar build
         run: tar xzvf coatjava.tar.gz
       - name: spotbugs
-        run: ./build-coatjava.sh --spotbugs --nomaps --no-progress
+        run: ./build-coatjava.sh --spotbugs --no-maps --no-progress
 
   test_decoder:
     needs: [ build, download_test_data ]

build-coatjava.sh

@@ -11,13 +11,13 @@ set -o pipefail
 
 cleanBuild=false
 runSpotBugs=false
-downloadMaps=true
-downloadNets=true
-downloadSqlites=true
 runUnitTests=false
-dataRetrieval=lfs
 installClara=false
 downloadData=false
+auxRetrieval=lfs
+auxDownloadMaps=true
+auxDownloadNets=true
+auxDownloadSqlites=true
 
 ################################################################################
 # usage
@@ -32,17 +32,18 @@ GENERAL OPTIONS
     --no-progress     no download progress printouts
     --help            show this message
 
-DATA RETRIEVAL OPTIONS
+AUXILIARY DATA RETRIEVAL OPTIONS
   How to retrieve magnetic field maps, neural network models, etc.
-  Choose only one; default is `--'$dataRetrieval'`
+  Choose only one; default is `--'$auxRetrieval'`
     --lfs             use Git Large File Storage (requires `git-lfs`)
     --cvmfs           use CernVM-FS (requires `/cvfms`)
     --https           use clasweb HTTPS (field maps only)
-  Additional options
-    --nomaps          do not download/overwrite field maps
-    --nonets          do not download/overwrite neural networks
-    --nosqlites       do not download/overwrite SQLite files for CCDB/RCDB
-    --wipe            remove retrieved data
+    --wipe            remove ALL retrieved auxiliary data and exit
+  Options for disabling downloads:
+    --no-aux          ignore ALL auxiliary data
+    --no-maps         ignore field maps
+    --no-nets         ignore neural networks
+    --no-sqlites      ignore CCDB/RCDB SQLite files
 
 TESTING OPTIONS
     --spotbugs        also run spotbugs plugin
@@ -64,17 +65,8 @@ wgetArgs=()
 for xx in $@
 do
   case $xx in
-    --spotbugs)  runSpotBugs=true   ;;
-    -n)          runSpotBugs=false  ;;
-    --nomaps)    downloadMaps=false ;;
-    --nonets)    downloadNets=false ;;
-    --nosqlites) downloadSqlites=false ;;
-    --unittests) runUnitTests=true  ;;
-    --clean)     cleanBuild=true    ;;
-    --depana)
-      echo "ERROR: option \`$xx\` has been removed; dependency tree printout and analysis now happen automatically in the Maven build lifecycle" >&2
-      exit 1
-      ;;
+    --clara) installClara=true ;;
+    --clean) cleanBuild=true   ;;
     --quiet)
       mvnArgs+=(--quiet --batch-mode)
       wgetArgs+=(--quiet)
@@ -83,12 +75,26 @@ do
       mvnArgs+=(--no-transfer-progress)
       wgetArgs+=(--no-verbose)
       ;;
-    --cvmfs) dataRetrieval=cvmfs ;;
-    --lfs)   dataRetrieval=lfs   ;;
-    --https) dataRetrieval=https ;;
-    --wipe)  dataRetrieval=wipe  ;;
-    --clara) installClara=true   ;;
-    --data)  downloadData=true   ;;
+    --lfs)   auxRetrieval=lfs   ;;
+    --cvmfs) auxRetrieval=cvmfs ;;
+    --https) auxRetrieval=https ;;
+    --wipe)  auxRetrieval=wipe  ;;
+    --no-aux|--noaux)
+      auxDownloadMaps=false
+      auxDownloadNets=false
+      auxDownloadSqlites=false
+      ;;
+    --no-maps|--nomaps)       auxDownloadMaps=false    ;;
+    --no-nets|--nonets)       auxDownloadNets=false    ;;
+    --no-sqlites|--nosqlites) auxDownloadSqlites=false ;;
+    --spotbugs)  runSpotBugs=true  ;;
+    -n)          runSpotBugs=false ;;
+    --unittests) runUnitTests=true ;;
+    --data)      downloadData=true ;;
+    --depana)
+      echo "ERROR: option \`$xx\` has been removed; dependency tree printout and analysis now happen automatically in the Maven build lifecycle" >&2
+      exit 1
+      ;;
     --xrootd)
       echo "ERROR: option \`$xx\` has been removed; use \`--help\` for guidance" >&2
       exit 1
@@ -101,6 +107,25 @@ do
   esac
 done
 
+echo "---------------------- ARGUMENTS ----------------------"
+print_arg() {
+  printf "  %30s = " "$1"
+  shift
+  echo "$@"
+}
+print_arg "cleanBuild" "$cleanBuild"
+print_arg "runSpotBugs" "$runSpotBugs"
+print_arg "runUnitTests" "$runUnitTests"
+print_arg "installClara" "$installClara"
+print_arg "downloadData" "$downloadData"
+print_arg "auxRetrieval" "$auxRetrieval"
+print_arg "auxDownloadMaps" "$auxDownloadMaps"
+print_arg "auxDownloadNets" "$auxDownloadNets"
+print_arg "auxDownloadSqlites" "$auxDownloadSqlites"
+print_arg "mvnArgs" "${mvnArgs[@]:-}"
+print_arg "wgetArgs" "${wgetArgs[@]:-}"
+echo "-------------------------------------------------------"
+
 
 ################################################################################
 # setup
@@ -142,20 +167,20 @@ if $cleanBuild; then
 fi
 
 # wipe retrieved data (field maps, NN models, etc.)
-if [ "$dataRetrieval" = "wipe" ]; then
+if [ "$auxRetrieval" = "wipe" ]; then
   git submodule deinit --all --force
 fi
 
 # print cleanup note and exit
-if $cleanBuild || [ "$dataRetrieval" = "wipe" ]; then
-  [ "$dataRetrieval" = "wipe" ] && echo "[+] REMOVED RETRIEVED DATA" || echo "[+] NOTE: retrieved data not removed; use \`--wipe\` if you need to remove them"
+if $cleanBuild || [ "$auxRetrieval" = "wipe" ]; then
+  [ "$auxRetrieval" = "wipe" ] && echo "[+] REMOVED RETRIEVED DATA" || echo "[+] NOTE: retrieved data not removed; use \`--wipe\` if you need to remove them"
   $cleanBuild && echo "[+] DONE CLEANING; rerun without \`--clean\` to build"
   exit
 fi
 
 
 ################################################################################
-# download field maps, NN models, etc.
+# download auxiliary files
 ################################################################################
 
 # check if a command exists
@@ -164,7 +189,7 @@ command_exists () {
 }
 
 # check data-retrieval options, and prepare accordingly
-case $dataRetrieval in
+case $auxRetrieval in
   lfs)
     if ! command_exists git-lfs ; then
       echo 'ERROR: `git-lfs` not found; please install it, or use a different data-retrieval option other than `--lfs`' >&2
@@ -182,7 +207,7 @@ case $dataRetrieval in
   https)
     ;;
   *)
-    echo "ERROR: data retrieval option '$dataRetrieval' is not supported" >&2
+    echo "ERROR: data retrieval option \`--$auxRetrieval\` is not supported" >&2
     exit 1
     ;;
 esac
@@ -202,7 +227,7 @@ download_lfs() {
 # download a magnetic field map
 download_map () {
   ret=0
-  case $dataRetrieval in
+  case $auxRetrieval in
     cvmfs)
       notify_retrieval 'field map' 'cvmfs'
       cp $1 ./
@@ -226,23 +251,23 @@ download_map () {
       ;;
     *)
       ret=1
-      echo "ERROR:::::::::::  called 'download_map' with bad 'dataRetrieval'." >&2
+      echo "ERROR:::::::::::  called 'download_map' with bad 'auxRetrieval' option \`--$auxRetrieval\`." >&2
       ;;
   esac
   return $ret
 }
 
 # download the default field maps, as defined in libexec/env.sh:
 # (and duplicated in etc/services/reconstruction.yaml):
-if $downloadMaps; then
-  case $dataRetrieval in
+if $auxDownloadMaps; then
+  case $auxRetrieval in
     lfs)
       notify_retrieval 'field maps' 'lfs'
       download_lfs etc/data/magfield
       ;;
     cvmfs|https)
       webDir=https://clasweb.jlab.org/clas12offline/magfield
-      if [ "$dataRetrieval" = "cvmfs" ]; then
+      if [ "$auxRetrieval" = "cvmfs" ]; then
         webDir=/cvmfs/oasis.opensciencegrid.org/jlab/hallb/clas12/sw/noarch/data/magfield
       fi
       mkdir -p $magfield_dir
@@ -253,22 +278,22 @@ if $downloadMaps; then
         if [ $? -ne 0 ]; then
             echo "ERROR:::::::::::  Could not download field map:" >&2
             echo "$webDir/$map" >&2
-            echo "One option is to download manually into etc/data/magfield and then run this build script with --nomaps" >&2
+            echo "One option is to download manually into \`etc/data/magfield\` and then run this build script with \`--no-maps\`" >&2
             exit 1
         fi
       done
       cd -
       ;;
     *)
-      echo "ERROR: data retrieval option '$dataRetrieval' not supported for field maps" >&2
+      echo "ERROR: data retrieval option \`--$auxRetrieval\` not supported for field maps" >&2
       exit 1
       ;;
   esac
 fi
 
 # download neural networks
-if $downloadNets; then
-  case $dataRetrieval in
+if $auxDownloadNets; then
+  case $auxRetrieval in
     lfs)
       notify_retrieval 'neural networks' 'lfs'
       download_lfs etc/data/nnet
@@ -285,8 +310,8 @@ if $downloadNets; then
 fi
 
 # download neural networks
-if $downloadSqlites; then
-  case $dataRetrieval in
+if $auxDownloadSqlites; then
+  case $auxRetrieval in
     lfs)
       notify_retrieval 'ccdb/rcdb SQLite files' 'lfs'
       download_lfs etc/data/sqlite
@@ -299,7 +324,7 @@ fi
 
 # download validation data
 if $downloadData; then
-  case $dataRetrieval in
+  case $auxRetrieval in
     lfs)
       notify_retrieval 'validation data' 'lfs'
       download_lfs validation/advanced-tests/data