Add tunn

README.md

@@ -1,6 +1,31 @@
 # lpc-scripts
 scripts of use on the cmslpc cluster
 
+Table of Contents
+=================
+
+* [call_host.sh](#call_hostsh)
+   * [Note](#note)
+   * [Usage](#usage)
+      * [Manual](#manual)
+      * [Automatic](#automatic)
+   * [Details](#details)
+   * [Options](#options)
+   * [Caveats](#caveats)
+* [bind_condor.sh](#bind_condorsh)
+   * [Usage](#usage-1)
+   * [Setting up bindings](#setting-up-bindings)
+* [tunn](#tunn)
+   * [Detailed usage](#detailed-usage)
+   * [Web browser usage](#web-browser-usage)
+* [Unit and Integration testing](#unit-and-integration-testing)
+   * [Automated](#automated)
+   * [Manual](#manual-1)
+      * [Bats for Bash scripts](#bats-for-bash-scripts)
+      * [Pytest for Python modules](#pytest-for-python-modules)
+
+<!-- Created by https://github.com/ekalinin/github-markdown-toc -->
+
 ## `call_host.sh`
 
 Many commands are installed on interactive nodes but are not accessible inside containers.
@@ -189,6 +214,75 @@ In this particular case, it is necessary to upgrade `pip` because the Python ver
 **NOTE**: These recipes only install the bindings for Python3. (Python2 was still the default in `CMSSW_10_6_X`.)
 You will need to make sure any scripts using the bindings are compatible with Python3.
 
+## `tunn`
+
+A simple utility to create and manage SSH tunnels.
+
+The basic usage of `tunn` follows this pattern:
+1. `tunn make [email protected]`
+2. `tunn list`:
+    ```
+    index: socket port command
+    0: "/home/[user]/.tsock_xyz" 8XXX "[email protected]"
+    ```
+3. `tunn kill 0`:
+    ```
+    Exit request sent.
+    ```
+
+If you have host aliases defined in your `~/.ssh/config` file, you can use them with `tunn`.
+
+### Detailed usage
+
+The configuration and command-line options for `tunn` are described in its usage message:
+```
+tunn [operation] [options] [arguments]
+
+Default settings are obtained from the config file at /home/[user]/.tunnconfig.
+To override the config file location, put this in your .bashrc or other login file:
+    export TUNN_CONFIG=/my/preferred/file
+The available config variables are: TUNN_PREFIX, TUNN_PORT, TUNN_VERBOSE.
+Their values should be specified in the config file using bash syntax, e.g.:
+    TUNN_PORT=8XXX
+(If TUNN_PORT is not specified in the config file or via the command line option,
+the default value is taken from the last three digits of your UID.)
+
+Operations:
+
+make         make new tunnel
+    -n [name]        tunnel socket name prefix (default: /home/[user]/.tsock)
+    -p [port]        tunnel port (default: 8XXX)
+    [destination]    ssh destination for tunnel (required)
+
+list         list open tunnels
+
+kill         kill specified tunnel
+    [index]          index of tunnel (required)
+
+Common options:
+-u           (unclean) do not auto-remove closed tunnels from list
+-v           toggle verbosity (default: false)
+-h           print this message and exit
+```
+
+### Web browser usage
+
+There are addons available for web browsers to route traffic through ssh tunnels.
+[FoxyProxy](https://getfoxyproxy.org/downloads/) is recommended for most browsers (Chrome, Firefox, and derivatives; Safari has [equivalent internal settings](https://help.getfoxyproxy.org/index.php/knowledge-base/how-to-use-your-proxy-services-with-safari/).)
+
+You can add LPC as a proxy server in the "Proxies" tab, with settings as follows:
+* Title: LPC
+* Hostname: localhost
+* Type: SOCKS5
+* Port: [your TUNN_PORT value 8XXX]
+* Proxy by patterns: [blank], "Include", "Wildcard", Title: FNAL, Pattern: \*://\*.fnal.gov/
+
+<details>
+<summary>Firefox example screenshot:</summary>
+
+![Foxyproxy settings screenshot](./docs/foxyproxy_lpc_generic.png)
+</details>
+
 ## Unit and Integration testing
 
 ### Automated

tunn

@@ -0,0 +1,222 @@
+#!/bin/bash
+
+# helper functions
+
+tunn_check_op() {
+	if [ "$TUNN_OP" != "$1" ]; then
+		usage 1
+	fi
+}
+
+tunn_echo() {
+	if [ "$TUNN_VERBOSE" == "true" ]; then
+		echo "$@"
+	fi
+}
+
+tunn_read_impl() {
+	# get list of tunnels
+	readarray -t TUNN_LIST < "$TUNN_LISTFILE"
+	export TUNN_LIST
+}
+
+tunn_update() {
+	tunn_read_impl
+	# clear list of tunnels
+	: > "$TUNN_LISTFILE"
+	# write open tunnels back into list
+	for ((i=0; i < ${#TUNN_LIST[@]}; i++)); do
+		tunn_entry $i
+		# shellcheck disable=SC2086
+		if ssh $TUNN_DEST -S "$TUNN_SOCKETNAME" -O check >&/dev/null; then
+			echo "${TUNN_LIST[$i]}" >> "$TUNN_LISTFILE"
+		else
+			tunn_echo "Removing dead tunnel: ${TUNN_LIST[$i]}"
+			rm "$TUNN_SOCKETNAME" >&/dev/null
+		fi
+	done
+	unset TUNN_LIST
+}
+
+tunn_read() {
+	if [ -z "$TUNN_UNCLEAN" ]; then
+		tunn_read_impl
+		tunn_update
+	fi
+	tunn_read_impl
+}
+
+tunn_entry() {
+	ENTRY="${TUNN_LIST[$1]}"
+	# split, respecting quotes
+	declare -a "ENTRY_ARR=($ENTRY)"
+
+	# reconstruct command based on output format
+	export TUNN_SOCKETNAME="${ENTRY_ARR[0]}"
+	export TUNN_PORT="${ENTRY_ARR[1]}"
+	export TUNN_DEST="${ENTRY_ARR[2]}"
+}
+
+# operations
+
+tunn_make() {
+	TUNN_DEST="$*"
+	if [ -z "$TUNN_DEST" ]; then
+		usage 1
+	fi
+
+	# generate a random name for socket
+	TUNN_SOCKETNUM=$(uuidgen)
+	TUNN_SOCKETNAME="$TUNN_PREFIX"_"$TUNN_SOCKETNUM"
+
+	# execute tunnel command
+	# shellcheck disable=SC2086
+	ssh $TUNN_DEST -N -D "$TUNN_PORT" -f -M -S "$TUNN_SOCKETNAME" 2>/dev/null
+	TUNN_EXIT=$?
+
+	# add to list of tunnels
+	if [ $TUNN_EXIT -eq 0 ]; then
+		echo "\"$TUNN_SOCKETNAME\" $TUNN_PORT \"$TUNN_DEST\"" >> "$TUNN_LISTFILE"
+		tunn_echo "Tunnel ready, port: $TUNN_PORT"
+	else
+		echo "Could not create tunnel (exit code $TUNN_EXIT)"
+		exit $TUNN_EXIT
+	fi
+}
+
+tunn_list() {
+	if [ -n "$1" ]; then
+		usage 1
+	fi
+
+	tunn_read
+	if [ ${#TUNN_LIST[@]} -eq 0 ]; then
+		return
+	fi
+	echo "index: socket port destination"
+	for ((i=0; i < ${#TUNN_LIST[@]}; i++)); do
+		echo "$i: ${TUNN_LIST[$i]}"
+	done
+}
+
+tunn_kill() {
+	INDEX="$1"
+	if [ -z "$INDEX" ]; then
+		usage 1
+	fi
+
+	# get this tunnel
+	tunn_read
+	if [ "$INDEX" -lt ${#TUNN_LIST[@]} ]; then
+		tunn_entry "$INDEX"
+	else
+		echo "Tunnel index $INDEX not found"
+		exit 2
+	fi
+
+	# execute kill command
+	# shellcheck disable=SC2086
+	ssh $TUNN_DEST -S "$TUNN_SOCKETNAME" -O exit
+	tunn_echo "Killed tunnel: ${TUNN_LIST[$INDEX]}"
+
+	# update list
+	sed -i "$((INDEX+1))"d "$TUNN_LISTFILE"
+}
+
+# defaults
+TUNN_LISTFILE=~/.tunnlist
+: "${TUNN_CONFIG:=~/.tunnconfig}"
+TUNN_SOCKETNAME=""
+TUNN_DEST=""
+TUNN_UNCLEAN=""
+
+# get config defaults
+if [ -e "$TUNN_CONFIG" ]; then
+	# shellcheck source=/dev/null
+	source "$TUNN_CONFIG"
+fi
+: "${TUNN_PREFIX:=~/.tsock}"
+UTMP=$(id -u)
+UTMP=${UTMP:0-3}
+: "${TUNN_PORT:=8${UTMP}}"
+: "${TUNN_VERBOSE:=false}"
+
+declare -A TUNN_INVERT
+TUNN_INVERT[true]=false
+TUNN_INVERT[false]=true
+
+usage() {
+	ECHO="echo -e"
+	$ECHO "tunn [operation] [options] [arguments]"
+	$ECHO
+	$ECHO "Default settings are obtained from the config file at ${TUNN_CONFIG}."
+	$ECHO "To override the config file location, put this in your .bashrc or other login file:"
+	$ECHO "\texport TUNN_CONFIG=/my/preferred/file"
+	$ECHO "The available config variables are: TUNN_PREFIX, TUNN_PORT, TUNN_VERBOSE."
+	$ECHO "Their values should be specified in the config file using bash syntax, e.g.:"
+	$ECHO "\tTUNN_PORT=8XXX"
+	$ECHO "(If TUNN_PORT is not specified in the config file or via the command line option,"
+	$ECHO "the default value is taken from the last three digits of your UID.)"
+	$ECHO
+	$ECHO "Operations:"
+	$ECHO
+	$ECHO "make       \t make new tunnel"
+	$ECHO "\t-n [name]     \t tunnel socket name prefix (default: ${TUNN_PREFIX})"
+	$ECHO "\t-p [port]     \t tunnel port (default: $TUNN_PORT)"
+	$ECHO "\t[destination] \t ssh destination for tunnel (required)"
+	$ECHO
+	$ECHO "list       \t list open tunnels"
+	$ECHO
+	$ECHO "kill       \t kill specified tunnel"
+	$ECHO "\t[index]       \t index of tunnel (required)"
+	$ECHO
+	$ECHO "Common options:"
+	$ECHO "-u         \t (unclean) do not auto-remove closed tunnels from list"
+	$ECHO "-v         \t toggle verbosity (default: $TUNN_VERBOSE)"
+	$ECHO "-h         \t print this message and exit"
+	exit "$1"
+}
+
+# get operation
+TUNN_OP="$1"
+shift 1
+
+TUNN_OPFN=""
+case "$TUNN_OP" in
+	make) TUNN_OPFN=tunn_make
+	;;
+	list) TUNN_OPFN=tunn_list
+	;;
+	kill) TUNN_OPFN=tunn_kill
+	;;
+	*) usage 1
+	;;
+esac
+
+while getopts "n:p:uvh" opt; do
+	case "$opt" in
+		n) tunn_check_op make; TUNN_PREFIX="$OPTARG"
+		;;
+		p) tunn_check_op make; TUNN_PORT="$OPTARG"
+		;;
+		u) TUNN_UNCLEAN=true
+		;;
+		v) TUNN_VERBOSE="${TUNN_INVERT[$TUNN_VERBOSE]}"
+		;;
+		h) usage 0
+		;;
+		*) usage 1
+		;;
+	esac
+done
+
+# ensure list file exists
+if [ ! -e "$TUNN_LISTFILE" ]; then
+	touch "$TUNN_LISTFILE"
+fi
+
+# get args for operation (if any)
+shift $((OPTIND - 1))
+
+# execute operation
+$TUNN_OPFN "$@"