|
| 1 | +#!/usr/bin/env bash |
| 2 | +{ |
| 3 | +echo "==============================" |
| 4 | +echo "=== VLBI-cwl Runner ===" |
| 5 | +echo "=== Facet subtract ===" |
| 6 | +echo "=== Author: Frits Sweijen ===" |
| 7 | +echo "==============================" |
| 8 | +echo "If you think you've found a bug, report it at https://github.com/tikk3r/flocs/issues" |
| 9 | +echo |
| 10 | +HELP="$(basename $0) [-s <container path>] [-b <container bindpaths>] [-l <user-defined LINC>] [-f <user-defined FLoCS>] [-v <user-defined VLBI-cwl>] [-r <running directory>] [-e<options for create_ms_list.py>] -d <data path> -m <WSClean model images> -c <DD calibration h5parm>" |
| 11 | +if [[ $1 == "-h" || $1 == "--help" ]]; then |
| 12 | + echo "Usage:" |
| 13 | + echo $HELP |
| 14 | + exit 0 |
| 15 | +fi |
| 16 | + |
| 17 | +while getopts ":d:s:r:l:f:b:e:m:c:v:" opt; do |
| 18 | + case $opt in |
| 19 | + d) DATADIR="$OPTARG" |
| 20 | + ;; |
| 21 | + s) SIMG="$OPTARG" |
| 22 | + ;; |
| 23 | + b) BINDPATHS="$OPTARG" |
| 24 | + ;; |
| 25 | + r) RUNDIR="$OPTARG" |
| 26 | + ;; |
| 27 | + l) LINC_DATA_ROOT="$OPTARG" |
| 28 | + ;; |
| 29 | + f) FLOCS_ROOT="$OPTARG" |
| 30 | + ;; |
| 31 | + m) MODEL_IMAGES="$OPTARG" |
| 32 | + ;; |
| 33 | + c) DD_SOLS="$OPTARG" |
| 34 | + ;; |
| 35 | + v) VLBI_DATA_ROOT="$OPTARG" |
| 36 | + ;; |
| 37 | + e) EXTRAOPTS="$OPTARG" |
| 38 | + ;; |
| 39 | + \?) echo "Invalid option -$OPTARG" >&2 |
| 40 | + echo |
| 41 | + echo "Usage:" |
| 42 | + echo $HELP |
| 43 | + exit 1 |
| 44 | + ;; |
| 45 | + esac |
| 46 | +done |
| 47 | + |
| 48 | +# Check if user gave sensible paths. |
| 49 | +if [[ ! -d $DATADIR ]]; then |
| 50 | + echo "Data directory $DATADIR does not exist or is not accessible!" |
| 51 | + exit 2 |
| 52 | +else |
| 53 | + export DATADIR=$(readlink -f $DATADIR) |
| 54 | +fi |
| 55 | + |
| 56 | +if [[ ! -z "$SIMG" ]]; then |
| 57 | + if [[ ! -f $SIMG ]]; then |
| 58 | + echo "Container $SIMG does not exist or is not accessible!" |
| 59 | + exit 3 |
| 60 | + fi |
| 61 | +fi |
| 62 | + |
| 63 | +if [[ ! -d $MODEL_IMAGES ]]; then |
| 64 | + echo "Model image directory $MODEL_IMAGES does not exist or is not accessible!" |
| 65 | + exit 4 |
| 66 | +fi |
| 67 | + |
| 68 | +if [[ ! -f $DD_SOLS ]]; then |
| 69 | + echo "$DD_SOLS does not exist or is not accessible!" |
| 70 | + exit 5 |
| 71 | +fi |
| 72 | + |
| 73 | +if [[ -z $RUNDIR ]]; then |
| 74 | + echo "No running directory specified, running in $PWD" |
| 75 | + RUNDIR=$PWD |
| 76 | +else |
| 77 | + echo "Using user-specified running directory $RUNDIR" |
| 78 | +fi |
| 79 | +export RUNDIR |
| 80 | + |
| 81 | +# Automatically bind the data and runtime directories. |
| 82 | +APPTAINER_BINDPATH=$RUNDIR,$DATADIR,$APPTAINER_BINDPATH |
| 83 | +export APPTAINER_BINDPATH |
| 84 | +echo "Binding the following paths to the container:" |
| 85 | +sed 's/:/\n/g' <<< "$APPTAINER_BINDPATH" |
| 86 | + |
| 87 | +## WORKDIR is where all the other directories will be stored. |
| 88 | +export WORKDIR=$(mktemp -d -p "$RUNDIR") |
| 89 | +echo "Working directory is $WORKDIR" |
| 90 | + |
| 91 | +## Final results will be copied here. |
| 92 | +export RESULTSDIR=$WORKDIR/results_LINC_calibrator/ |
| 93 | +## Logs of the various steps will be put here. |
| 94 | +export LOGSDIR=$WORKDIR/logs_LINC_calibrator/ |
| 95 | +## Temporary files are stored here. |
| 96 | +## The trailing slash is important here. |
| 97 | +export TMPDIR=$WORKDIR/tmpdir_LINC_calibrator/ |
| 98 | + |
| 99 | +if [[ -z "$LINC_DATA_ROOT" ]]; then |
| 100 | + export LINC_DATA_ROOT=$WORKDIR/LINC |
| 101 | +fi |
| 102 | + |
| 103 | +if [[ -z "$VLBI_DATA_ROOT" ]]; then |
| 104 | + VLBI_DATA_ROOT=$WORKDIR/VLBI_cwl |
| 105 | +fi |
| 106 | + |
| 107 | +if [[ -z "$FLOCS_ROOT" ]]; then |
| 108 | + export FLOCS_ROOT=$WORKDIR/flocs |
| 109 | +fi |
| 110 | + |
| 111 | +LOFAR_HELPERS_ROOT=$WORKDIR/lofar_helpers |
| 112 | +FACETSELFCAL_ROOT=$WORKDIR/lofar_facet_selfcal |
| 113 | +git clone https://github.com/jurjen93/lofar_helpers.git $LOFAR_HELPERS_ROOT |
| 114 | +git clone https://github.com/rvweeren/lofar_facet_selfcal.git $FACETSELFCAL_ROOT |
| 115 | + |
| 116 | +# Check if LINC directory exists or is valid. |
| 117 | +if [ ! -d $LINC_DATA_ROOT ]; then |
| 118 | + echo $LINC_DATA_ROOT does not exist and will be created. Cloning LINC... |
| 119 | + mkdir -p $LINC_DATA_ROOT |
| 120 | + git clone https://git.astron.nl/RD/LINC.git $LINC_DATA_ROOT |
| 121 | +fi |
| 122 | +# |
| 123 | +# Check if VLBI directory exists or is valid. |
| 124 | +if [ ! -d $VLBI_DATA_ROOT ]; then |
| 125 | + echo $VLBI_DATA_ROOT does not exist and will be created. Cloning VLBI-cwl... |
| 126 | + mkdir -p $VLBI_DATA_ROOT |
| 127 | + git clone https://git.astron.nl/RD/VLBI-cwl.git $VLBI_DATA_ROOT |
| 128 | +elif [ -d $VLBI_DATA_ROOT ] && [ ! -d $VLBI_DATA_ROOT/steps ]; then |
| 129 | + echo $VLBI_DATA_ROOT exists, but is empty. Cloning VLBI-cwl... |
| 130 | + git clone https://git.astron.nl/RD/VLBI-cwl.git $VLBI_DATA_ROOT |
| 131 | +fi |
| 132 | + |
| 133 | +# If the directory is not empty, check if it contains VLBI-cwl |
| 134 | +if [ -d $VLBI_DATA_ROOT ] && [ ! -d $VLBI_DATA_ROOT/steps ]; then |
| 135 | + echo WARNING: $VLBI_DATA_ROOT is not empty, but required VLBI-cwl folders are not found. |
| 136 | + exit 1 |
| 137 | +elif [ -d $VLBI_DATA_ROOT ] && [ -d $VLBI_DATA_ROOT/steps ]; then |
| 138 | + echo $VLBI_DATA_ROOT exists and seems to contain VLBI-cwl. Continueing... |
| 139 | +fi |
| 140 | + |
| 141 | +# Check if FLoCs directory exists or is valid. |
| 142 | +if [ ! -d $FLOCS_ROOT ]; then |
| 143 | + echo $FLOCS_ROOT does not exist and will be created. Cloning LINC... |
| 144 | + mkdir -p $FLOCS_ROOT |
| 145 | + git clone https://github.com/tikk3r/flocs.git $FLOCS_ROOT |
| 146 | +fi |
| 147 | + |
| 148 | +# If the directory is not empty, check if it contains LINC |
| 149 | +if [ -d $LINC_DATA_ROOT ] && [ ! -d $LINC_DATA_ROOT/steps ]; then |
| 150 | + echo WARNING: $LINC_DATA_ROOT found, but required LINC folders are not found. |
| 151 | + exit 1 |
| 152 | +elif [ -d $LINC_DATA_ROOT ] && [ -d $LINC_DATA_ROOT/steps ]; then |
| 153 | + echo $LINC_DATA_ROOT exists and seems to contain LINC. Continuing... |
| 154 | +fi |
| 155 | +# Get the full path to avoid pathing issues later on. |
| 156 | +LINC_DATA_ROOT=$(realpath $LINC_DATA_ROOT) |
| 157 | +export LINC_DATA_ROOT |
| 158 | + |
| 159 | +# Obtain LINC commit used |
| 160 | +cd $LINC_DATA_ROOT |
| 161 | +export LINC_COMMIT=$(git rev-parse --short HEAD) |
| 162 | +cd - |
| 163 | +# |
| 164 | +# Obtain LOFAR-VLBI commit used |
| 165 | +cd $VLBI_DATA_ROOT |
| 166 | +export VLBI_COMMIT=$(git rev-parse --short HEAD) |
| 167 | +cd - |
| 168 | + |
| 169 | +if [ -d $FLOCS_ROOT ] && [ ! -d $FLOCS_ROOT/runners ]; then |
| 170 | + echo WARNING: $FLOCS_ROOT found, but required flocs folders are not found. |
| 171 | + exit 1 |
| 172 | +elif [ -d $FLOCS_ROOT ] && [ -d $FLOCS_ROOT/runners ]; then |
| 173 | + echo $FLOCS_ROOT exists and seems to be valid. Continuing... |
| 174 | +fi |
| 175 | +# Get the full path to avoid pathing issues later on. |
| 176 | +FLOCS_ROOT=$(realpath $FLOCS_ROOT) |
| 177 | +export FLOCS_ROOT |
| 178 | + |
| 179 | +cd $FLOCS_ROOT |
| 180 | +export FLOCS_COMMIT=$(git rev-parse --short HEAD) |
| 181 | +cd - |
| 182 | + |
| 183 | +mkdir -p $RESULTSDIR |
| 184 | +mkdir -p $LOGSDIR |
| 185 | +mkdir -p $TMPDIR |
| 186 | +cd $WORKDIR |
| 187 | + |
| 188 | +if [[ -z "$SIMG" ]]; then |
| 189 | + echo "No container specified, this workflow requires a container." |
| 190 | + exit 6 |
| 191 | +else |
| 192 | + echo "Using container $SIMG" |
| 193 | + # Pass along necessary variables to the container. |
| 194 | + CONTAINERSTR=$(singularity --version) |
| 195 | + if [[ "$CONTAINERSTR" == *"apptainer"* ]]; then |
| 196 | + export APPTAINER_CACHEDIR=/cosma/apps/do011/dc-swei1/containers/apptainer_cache |
| 197 | + export APPTAINER_PULLDIR=$APPTAINER_CACHEDIR/pull |
| 198 | + export APPTAINER_TMPDIR=$APPTAINER_CACHEDIR/tmp |
| 199 | + export APPTAINERENV_LINC_DATA_ROOT=$LINC_DATA_ROOT |
| 200 | + export APPTAINERENV_RESULTSDIR=$WORKDIR/results_facet_subtract/ |
| 201 | + export APPTAINERENV_LOGSDIR=$WORKDIR/logs_facet_subtract/ |
| 202 | + export APPTAINERENV_TMPDIR=$WORKDIR/tmpdir_facet_subtract/ |
| 203 | + export APPTAINERENV_PREPEND_PATH=$LINC_DATA_ROOT/scripts |
| 204 | + else |
| 205 | + export SINGULARITY_CACHEDIR=/cosma/apps/do011/dc-swei1/containers/apptainer_cache |
| 206 | + export SINGULARITY_PULLDIR=$SINGULARITY_CACHEDIR/pull |
| 207 | + export SINGULARITY_TMPDIR=$SINGULARITY_CACHEDIR/tmp |
| 208 | + export SINGULARITYENV_LINC_DATA_ROOT=$LINC_DATA_ROOT |
| 209 | + export SINGULARITYENV_RESULTSDIR=$WORKDIR/results_facet_subtract/ |
| 210 | + export SINGULARITYENV_LOGSDIR=$WORKDIR/logs_facet_subtract/ |
| 211 | + export SINGULARITYENV_TMPDIR=$WORKDIR/tmpdir_facet_subtract/ |
| 212 | + export SINGULARITYENV_PREPEND_PATH=$LINC_DATA_ROOT/scripts |
| 213 | + fi |
| 214 | + |
| 215 | + echo "Generating default pipeline configuration" |
| 216 | + singularity exec -B $PWD,$BINDPATHS $SIMG python $FLOCS_ROOT/runners/create_ms_list.py VLBI facet-subtract --h5parm $DD_SOLS --model_image_folder $MODEL_IMAGES --scratch True --lofar_helpers=$LOFAR_HELPERS_ROOT --selfcal=$FACETSELFCAL_ROOT $EXTRAOPTS $DATADIR |
| 217 | + |
| 218 | + export TOIL_CHECK_ENV=True |
| 219 | + mkdir -p $WORKDIR/coordination |
| 220 | + export JOBSTORE=$WORKDIR/jobstore |
| 221 | + export TOIL_SLURM_ARGS="--export=ALL -A do011 -p dine2 -t 24:00:00" |
| 222 | + mkdir $LOGSDIR/slurmlogs |
| 223 | + |
| 224 | + export CWL_SINGULARITY_CACHE=/cosma/apps/do011/dc-swei1/containers/apptainer_cache/ |
| 225 | + |
| 226 | + echo Facet subtract starting |
| 227 | + toil-cwl-runner \ |
| 228 | + --no-read-only \ |
| 229 | + --retryCount 2 \ |
| 230 | + --singularity \ |
| 231 | + --disableCaching \ |
| 232 | + --writeLogsFromAllJobs True \ |
| 233 | + --logFile full_log.log \ |
| 234 | + --writeLogs $LOGSDIR \ |
| 235 | + --outdir $RESULTSDIR \ |
| 236 | + --tmp-outdir-prefix $TMPDIR/ \ |
| 237 | + --jobStore $JOBSTORE \ |
| 238 | + --workDir $WORKDIR \ |
| 239 | + --tmpdir-prefix ${TMPDIR}_interm/ \ |
| 240 | + --disableAutoDeployment True \ |
| 241 | + --bypass-file-store \ |
| 242 | + --preserve-entire-environment \ |
| 243 | + --batchSystem slurm \ |
| 244 | + --clean onSuccess \ |
| 245 | + --no-compute-checksum \ |
| 246 | + $VLBI_DATA_ROOT/workflows/facet_subtract.cwl \ |
| 247 | + mslist_VLBI_facet_subtract.json |
| 248 | + echo Facet subtract ended |
| 249 | +fi |
| 250 | +echo Cleaning up... |
| 251 | +echo == Deleting tmpdir.. |
| 252 | +rm -rf $WORKDIR/tmpdir_facet_subtract |
| 253 | + |
| 254 | +echo == Moving results... |
| 255 | +FINALDIR=$(dirname $WORKDIR) |
| 256 | +pattern="${DATADIR}/*.MS" |
| 257 | +files=( $pattern ) |
| 258 | +ms="${files[0]}" # printf is safer! |
| 259 | +obsid=$(echo $(basename $ms) | awk -F'_' '{print $1}') |
| 260 | +mv "$WORKDIR" "$FINALDIR/${obsid}_facet_subtract" |
| 261 | + |
| 262 | +echo "===============================" |
| 263 | +echo "=== LINC Calibrator Summary ===" |
| 264 | +echo "===============================" |
| 265 | +echo FLoCs version: $FLOCS_COMMIT |
| 266 | +echo LINC version: $LINC_COMMIT |
| 267 | +echo Output: "$FINALDIR/${obsid}_facet_subtract" |
| 268 | +} |& tee job_output_full.txt |
0 commit comments