[Feature:System] Variable number of Vagrant Workers (Submitty#9537)

### Please check if the PR fulfills these requirements:

* [ ] Tests for the changes have been added/updated (if possible)
* [ ] Documentation has been updated/added if relevant

### What is the current behavior?
<!-- List issue if it fixes/closes/implements one using the "Fixes
#<number>" or "Closes #<number>" syntax -->
Developers can only create one vagrant worker VM at a time, by
specifying the `WORKER_PAIR` environment variable on initial vagrant up.

### What is the new behavior?
A variable number of worker VMs can be created by specifying the
forwarded SSH ports and private IP addresses of each in the
`.vagrant/workers.json` configuration file.

---------

Co-authored-by: Viane Matsibekker <[email protected]>
Co-authored-by: Mahi Pasarkar <[email protected]>

Author: 3 people

.gitignore

@@ -44,4 +44,4 @@ Pipfile
 Pipfile.lock
 .phpunit.result.cache
 
-.env
+.env

.setup/CONFIGURE_SUBMITTY.py

@@ -65,12 +65,6 @@ def __call__(self, parser, namespace, values, option_string=None):
                     help="Sets up Submitty for use with the sample courses. This is a Vagrant convenience "
                          "flag and should not be used in production!")
 parser.add_argument('--worker', action='store_true', default=False, help='Configure Submitty with autograding only')
-parser.add_argument(
-    '--worker-pair',
-    default=False,
-    action=StrToBoolAction,
-    help='Configure Submitty alongside a worker VM. This should only be used during development using Vagrant.'
-)
 parser.add_argument('--install-dir', default='/usr/local/submitty', help='Set the install directory for Submitty')
 parser.add_argument('--data-dir', default='/var/local/submitty', help='Set the data directory for Submitty')
 parser.add_argument('--websocket-port', default=8443, type=int, help='Port to use for websocket')
@@ -519,39 +513,33 @@ def write(x=''):
 
 if not args.worker:
     if not os.path.isfile(WORKERS_JSON):
+        capabilities = ["default"]
+        if args.setup_for_sample_courses:
+            capabilities.extend(["cpp", "python", "et-cetera", "notebook"])
+
         worker_dict = {
             "primary": {
-                "capabilities": ["default"],
+                "capabilities": capabilities,
                 "address": "localhost",
                 "username": "",
                 "num_autograding_workers": NUM_GRADING_SCHEDULER_WORKERS,
                 "enabled" : True
             }
         }
 
-        if args.worker_pair:
-            worker_dict["submitty-worker"] = {
-                "capabilities": ['default'],
-                "address": "192.168.56.21",
-                "username": "submitty",
-                "num_autograding_workers": NUM_GRADING_SCHEDULER_WORKERS,
-                "enabled": True
-            }
-            if args.setup_for_sample_courses:
-                worker_dict['submitty-worker']['capabilities'].extend([
-                    'cpp',
-                    'python',
-                    'et-cetera',
-                    'notebook',
-                ])
-
-        if args.setup_for_sample_courses:
-            worker_dict['primary']['capabilities'].extend([
-                'cpp',
-                'python',
-                'et-cetera',
-                'notebook',
-            ])
+        vagrant_workers_json = os.path.join(SUBMITTY_REPOSITORY, '.vagrant', 'workers.json')
+        if os.path.isfile(vagrant_workers_json):
+            with open(vagrant_workers_json) as f:
+                vagrant_workers = json.load(f, object_hook=OrderedDict)
+ 
+            for worker, data in vagrant_workers.items():
+                worker_dict[worker] = {
+                    "capabilities": capabilities,
+                    "address": data["ip_addr"],
+                    "username": "submitty",
+                    "num_autograding_workers": NUM_GRADING_SCHEDULER_WORKERS,
+                    "enabled": True
+                }
 
         with open(WORKERS_JSON, 'w') as workers_file:
             json.dump(worker_dict, workers_file, indent=4)

.setup/install_system.sh

@@ -64,31 +64,31 @@ export UTM=0
 export VAGRANT=0
 export NO_SUBMISSIONS=0
 export WORKER=0
-export WORKER_PAIR=0
 
 # Read through the flags passed to the script reading them in and setting
 # appropriate bash variables, breaking out of this once we hit something we
 # don't recognize as a flag
+echo ""
+echo "Running setup:"
 while :; do
     case $1 in
         --utm)
             export UTM=1
             export DEV_VM=1
+            echo "utm"
             ;;
         --vagrant)
             export VAGRANT=1
             export DEV_VM=1
+            echo "vagrant"
             ;;
         --worker)
             export WORKER=1
+            echo "worker"
             ;;
         --no_submissions)
             export NO_SUBMISSIONS=1
-            echo 'no_submissions'
-            ;;
-        --worker-pair)
-            export WORKER_PAIR=1
-            echo "worker_pair"
+            echo "no_submissions"
             ;;
         *) # No more options, so break out of the loop.
             break
@@ -319,12 +319,17 @@ fi
 
 if ! cut -d ':' -f 1 /etc/passwd | grep -q ${DAEMON_USER} ; then
     useradd -m -c "First Last,RoomNumber,WorkPhone,HomePhone" "${DAEMON_USER}"
-    if [ ${WORKER_PAIR} == 1 ]; then
+    if [ ${WORKER} == 0 ] && [ ${DEV_VM} == 1 ] && [ -f ${SUBMITTY_REPOSITORY}/.vagrant/workers.json ]; then
         echo -e "attempting to create ssh key for submitty_daemon..."
         su submitty_daemon -c "cd ~/"
         su submitty_daemon -c "ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N ''"
         su submitty_daemon -c "echo 'successfully created ssh key'"
-        su submitty_daemon -c "sshpass -p 'submitty' ssh-copy-id -i ~/.ssh/id_rsa.pub -o StrictHostKeyChecking=no [email protected]"
+
+        while read -r IP
+        do
+            su submitty_daemon -c "sshpass -p 'submitty' ssh-copy-id -i ~/.ssh/id_rsa.pub -o StrictHostKeyChecking=no submitty@${IP}"
+        done <<< "$(jq -r ".[].ip_addr" "${SUBMITTY_REPOSITORY}/.vagrant/workers.json")"
+        echo "DONE"
     fi
 fi
 
@@ -687,7 +692,7 @@ submitty@vagrant
 do-not-reply@vagrant
 localhost
 25
-" | python3 ${SUBMITTY_REPOSITORY}/.setup/CONFIGURE_SUBMITTY.py --debug --setup-for-sample-courses --websocket-port ${WEBSOCKET_PORT} --worker-pair ${WORKER_PAIR}
+" | python3 ${SUBMITTY_REPOSITORY}/.setup/CONFIGURE_SUBMITTY.py --debug --setup-for-sample-courses --websocket-port ${WEBSOCKET_PORT}
 
         # Set these manually as they're not asked about during CONFIGURE_SUBMITTY.py
         sed -i -e 's/"url": ""/"url": "ldap:\/\/localhost"/g' ${SUBMITTY_INSTALL_DIR}/config/authentication.json

.setup/install_worker.sh

@@ -4,7 +4,7 @@
 #   install_worker.sh [<extra> <extra> ...]
 
 # This script is used to set up the worker machine in a vagrant worker pair
-# made by running WORKER_PAIR=1 vagrant up
+# made by running WORKERS=n vagrant up
 
 GIT_PATH=/usr/local/submitty/GIT_CHECKOUT/Submitty
 SUPERVISOR_USER=submitty
@@ -15,11 +15,11 @@ echo "checking ${SUPERVISOR_USER} user"
 if ! cut -d ':' -f 1 /etc/passwd | grep -q "${SUPERVISOR_USER}" ; then
     echo "attempting to add ${SUPERVISOR_USER} user"
     #set up submitty user with password 'submitty'
-    useradd -m -p "$(openssl passwd -crypt submitty)" -c "First Last,RoomNumber,WorkPhone,HomePhone" "${SUPERVISOR_USER}"
+    useradd -m -p "$(openssl passwd -1 submitty)" -c "First Last,RoomNumber,WorkPhone,HomePhone" "${SUPERVISOR_USER}"
     [ -d "/home/${SUPERVISOR_USER}" ] && echo "Directory /home/${SUPERVISOR_USER} exists." || echo "Error: Directory /home/${SUPERVISOR_USER} does not exists."
 fi
 
-bash "${GIT_PATH}/.setup/install_system.sh" --worker --vagrant "${@}" 2>&1 | tee "${GIT_PATH}/.vagrant/install_worker_system.log"
+bash "${GIT_PATH}/.setup/install_system.sh" --worker --vagrant "${@}" 2>&1
 echo "--- FINISHED INSTALLING SYSTEM ---"
 echo "installing worker..."
 

.setup/vagrant/setup_vagrant.sh

@@ -27,6 +27,6 @@ if ! sudo bash ${SUBMITTY_REPOSITORY}/.setup/install_system.sh --vagrant ${@}; t
 For whatever reason, Vagrant has failed to build. If reporting
 an error to the developers, please be sure to also send the
 build log of Vagrant located at:
-.vagrant/install_${DISTRO}_${VERSION}.log.
+.vagrant/logs/${DISTRO}-${VERSION}.log.
 "
 fi

Vagrantfile

@@ -7,8 +7,6 @@
 #   NO_SUBMISSIONS=1 vagrant up
 #       or
 #   EXTRA=rpi vagrant up
-#       or
-#   WORKER_PAIR=1 vagrant up
 #
 #
 # If you want to override the default image used for the virtual machines, you can set the
@@ -32,33 +30,37 @@
 $stdout.sync = true
 $stderr.sync = true
 
-extra_command = ''
-autostart_worker = false
-if ENV.has_key?('NO_SUBMISSIONS')
-    extra_command << '--no_submissions '
-end
-if ENV.has_key?('EXTRA')
-    extra_command << ENV['EXTRA']
-end
-if ENV.has_key?('WORKER_PAIR')
-    autostart_worker = true
-    extra_command << '--worker-pair '
-end
+require 'json'
 
-$script = <<SCRIPT
-GIT_PATH=/usr/local/submitty/GIT_CHECKOUT/Submitty
-DISTRO=$(lsb_release -si | tr '[:upper:]' '[:lower:]')
-VERSION=$(lsb_release -sr | tr '[:upper:]' '[:lower:]')
-bash ${GIT_PATH}/.setup/vagrant/setup_vagrant.sh #{extra_command} 2>&1 | tee ${GIT_PATH}/.vagrant/install_${DISTRO}_${VERSION}.log
-SCRIPT
+def gen_script(machine_name, worker: false)
+  no_submissions = !ENV.fetch('NO_SUBMISSIONS', '').empty?
+  extra = ENV.fetch('EXTRA', '')
 
-$worker_script = <<SCRIPT
-GIT_PATH=/usr/local/submitty/GIT_CHECKOUT/Submitty
-DISTRO=$(lsb_release -si | tr '[:upper:]' '[:lower:]')
-VERSION=$(lsb_release -sr | tr '[:upper:]' '[:lower:]')
-bash ${GIT_PATH}/.setup/install_worker.sh #{extra_command} 2>&1 | tee ${GIT_PATH}/.vagrant/install_worker.log
+  setup_cmd = 'bash ${GIT_PATH}/.setup/'
+  if worker
+    setup_cmd += 'install_worker.sh'
+  else
+    setup_cmd += 'vagrant/setup_vagrant.sh'
+    if no_submissions
+      setup_cmd += ' --no_submissions'
+    end
+  end
+  unless extra.empty?
+    setup_cmd += " #{extra}"
+  end
+  setup_cmd += " 2>&1 | tee ${GIT_PATH}/.vagrant/logs/#{machine_name}.log"
+
+  script = <<SCRIPT
+    GIT_PATH=/usr/local/submitty/GIT_CHECKOUT/Submitty
+    DISTRO=$(lsb_release -si | tr '[:upper:]' '[:lower:]')
+    VERSION=$(lsb_release -sr | tr '[:upper:]' '[:lower:]')
+    mkdir -p ${GIT_PATH}/.vagrant/logs
+    #{setup_cmd}
 SCRIPT
 
+  return script
+end
+
 base_boxes = Hash[]
 
 # Should all be base Ubuntu boxes that use the same version
@@ -85,6 +87,15 @@ def mount_folders(config, mount_options)
   }
 end
 
+def get_workers()
+  worker_file = File.join(__dir__, '.vagrant', 'workers.json')
+  if File.file?(worker_file)
+    return JSON.parse(File.read(worker_file), symbolize_names: true)
+  else
+    return Hash[]
+  end
+end
+
 Vagrant.configure(2) do |config|
   if Vagrant.has_plugin?('vagrant-env')
     config.env.enable
@@ -108,21 +119,22 @@ Vagrant.configure(2) do |config|
   # that one) as well as making sure all non-primary ones have "autostart: false" set
   # so that when we do "vagrant up", it doesn't spin up those machines.
 
-  config.vm.define 'submitty-worker', autostart: autostart_worker do |ubuntu|
-    # If this IP address changes, it must be changed in install_system.sh and
-    # CONFIGURE_SUBMITTY.py to allow the ssh connection
-    ubuntu.vm.network "private_network", ip: "192.168.56.21"
-    ubuntu.vm.network 'forwarded_port', guest: 22, host: 2220, id: 'ssh'
-    ubuntu.vm.provision 'shell', inline: $worker_script
+  get_workers.map do |worker_name, data|
+    config.vm.define worker_name do |ubuntu|
+      ubuntu.vm.network 'private_network', ip: data[:ip_addr]
+      ubuntu.vm.network 'forwarded_port', guest: 22, host: data[:ssh_port], id: 'ssh'
+      ubuntu.vm.provision 'shell', inline: gen_script(worker_name, worker: true)
+    end
   end
 
-  config.vm.define 'ubuntu-22.04', primary: true do |ubuntu|
+  vm_name = 'ubuntu-22.04'
+  config.vm.define vm_name, primary: true do |ubuntu|
     ubuntu.vm.network 'forwarded_port', guest: 1511, host: ENV.fetch('VM_PORT_SITE', 1511)
     ubuntu.vm.network 'forwarded_port', guest: 8443, host: ENV.fetch('VM_PORT_WS',   8443)
     ubuntu.vm.network 'forwarded_port', guest: 5432, host: ENV.fetch('VM_PORT_DB',  16442)
     ubuntu.vm.network 'forwarded_port', guest: 7000, host: ENV.fetch('VM_PORT_SAML', 7000)
     ubuntu.vm.network 'forwarded_port', guest:   22, host: ENV.fetch('VM_PORT_SSH',  2222), id: 'ssh'
-    ubuntu.vm.provision 'shell', inline: $script
+    ubuntu.vm.provision 'shell', inline: gen_script(vm_name)
   end
 
   config.vm.provider 'virtualbox' do |vb, override|

generate_workers.py

@@ -0,0 +1,70 @@
+#!/usr/bin/python3
+
+import argparse
+import glob
+import ipaddress
+import json
+import os
+from collections import OrderedDict
+from typing import Union, cast
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        description='Script to generate configuration for '
+        'development worker machines')
+
+    parser.add_argument('-n', '--num', default=1, type=int,
+                        help='Number of worker machines to configure')
+    parser.add_argument('--ip-range', default='192.168.56.0/24', type=ipaddress.ip_network,
+                        help='IP address range for workers')
+    parser.add_argument('--base-port', default=2240, type=int,
+                        help='Base ssh port (ports will be assigned incrementally)')
+
+    return parser.parse_args()
+
+
+def main():
+    args = get_args()
+    workers = OrderedDict()
+    rootdir = os.path.dirname(os.path.realpath(__file__))
+    workerfile = os.path.join(rootdir, '.vagrant', 'workers.json')
+
+    if len(glob.glob(os.path.join(rootdir, '.vagrant', 'machines', '*', '*', 'action_provision'))):
+        if input("Warning: There are existing vagrant machines in this project that may conflict"
+                 " with new configuration. Are you sure you would like to proceed? [y/N] "
+                 )[0].lower() != 'y':
+            return
+
+    if os.path.isfile(workerfile):
+        if input('Overwrite existing worker configuration? [y/N] ')[0].lower() != 'y':
+            return
+
+    ips = cast(Union[ipaddress.IPv4Network, ipaddress.IPv6Network], args.ip_range).hosts()
+    if isinstance(ips, list):
+        ips = iter(ips)
+
+    for i in range(1, args.num+1):
+        ip = next(ips, None)
+        while (ip is not None
+               and (ip.is_reserved or isinstance(ip, ipaddress.IPv4Address)
+                    and str(ip).endswith('.1'))):
+            ip = next(ips, None)
+
+        if ip is None:
+            raise IndexError("IP range insufficient for requested number of workers")
+
+        data = OrderedDict()
+        data['ip_addr'] = str(ip)
+        data['ssh_port'] = args.base_port + i
+        workers[f'worker-{i}'] = data
+
+    os.makedirs(os.path.dirname(workerfile), exist_ok=True)
+    with open(workerfile, 'w') as file:
+        json.dump(workers, file, indent=4)
+
+    print('Wrote new configuration to ' + workerfile)
+
+
+if __name__ == '__main__':
+    main()