WIP: Playbook to manually start noble migration

Fixes #7416.

admin/securedrop_admin/__init__.py

@@ -1110,6 +1110,19 @@ def get_logs(args: argparse.Namespace) -> int:
     return 0
 
 
+@update_check_required("noble-migration")
+def noble_migration(args: argparse.Namespace) -> int:
+    """Upgrade to Ubuntu Noble"""
+    sdlog.info("Beginning the upgrade to Ubuntu Noble")
+    ansible_cmd = ansible_command() + [
+        os.path.join(args.ansible_path, "securedrop-noble-migration.yml"),
+    ]
+
+    subprocess.check_call(ansible_cmd, cwd=args.ansible_path)
+    sdlog.info("Upgrade to Ubuntu Noble complete!")
+    return 0
+
+
 def set_default_paths(args: argparse.Namespace) -> argparse.Namespace:
     if not args.ansible_path:
         args.ansible_path = args.root + "/install_files/ansible-base"
@@ -1209,6 +1222,9 @@ class ArgParseFormatterCombo(
     parse_logs = subparsers.add_parser("logs", help=get_logs.__doc__)
     parse_logs.set_defaults(func=get_logs)
 
+    parse_noble_migration = subparsers.add_parser("noble_migration", help=noble_migration.__doc__)
+    parse_noble_migration.set_defaults(func=noble_migration)
+
     parse_reset_ssh = subparsers.add_parser("reset_admin_access", help=reset_admin_access.__doc__)
     parse_reset_ssh.set_defaults(func=reset_admin_access)
 

install_files/ansible-base/roles/noble-migration/tasks/main.yml

@@ -0,0 +1,92 @@
+---
+
+- name: Check migration JSON on mon server
+  ansible.builtin.slurp:
+    src: /etc/securedrop-noble-migration-state.json
+  register: migration_json
+  ignore_errors: yes
+
+- name: Skip migration if already done
+  set_fact:
+    already_finished: "{{ not migration_json.failed and (migration_json.content | b64decode | from_json)['finished'] == 'Done' }}"
+
+- name: Perform migration
+  when: not already_finished
+  block:
+    - name: Instruct upgrade to begin
+      ansible.builtin.copy:
+        # It's okay to enable both app and mon here to simplify the logic,
+        # as this only affects the server the file is updated on.
+        content: |
+          {
+            "app": {"enabled": true, "bucket": 5},
+            "mon": {"enabled": true, "bucket": 5}
+          }
+        dest: /usr/share/securedrop/noble-upgrade.json
+
+    # Start the systemd service manually to avoid waiting for the timer to pick it up
+    - name: Start upgrade systemd service
+      ansible.builtin.systemd:
+        name: securedrop-noble-migration-upgrade
+        state: started
+
+    # Wait until we've finished the PendingUpdates stage. It's highly unlikely
+    # we'll ever successfully complete this stage because as soon as the script
+    # reaches finishes that stage, it reboots. Most likely this step will fail
+    # as unreachable, which we ignore and wait_for_connection.
+    - name: Wait for pending updates to be applied
+      ansible.builtin.wait_for:
+        path: /etc/securedrop-noble-migration-state.json
+        search_regex: '"finished":"PendingUpdates"'
+        sleep: 1
+        timeout: 300
+      ignore_unreachable: yes
+      ignore_errors: yes
+
+    - name: Wait for the first reboot
+      ansible.builtin.wait_for_connection:
+        connect_timeout: 20
+        sleep: 5
+        delay: 10
+        timeout: 300
+
+    # Start the systemd service manually to avoid waiting for the timer to pick it up
+    - name: Resume upgrade systemd service
+      ansible.builtin.systemd:
+        name: securedrop-noble-migration-upgrade
+        state: started
+
+    - debug:
+        msg: "The upgrade is in progress; it may take up to 30 minutes."
+
+    # Same as above, this will most likely fail as unreachable when the server
+    # actually reboots.
+    - name: Wait for system upgrade to noble
+      ansible.builtin.wait_for:
+        path: /etc/securedrop-noble-migration-state.json
+        search_regex: '"finished":"Reboot"'
+        sleep: 5
+        # Should finish in less than 30 minutes
+        timeout: 1800
+      ignore_unreachable: yes
+      ignore_errors: yes
+
+    - name: Wait for the second reboot
+      ansible.builtin.wait_for_connection:
+        connect_timeout: 20
+        sleep: 5
+        delay: 10
+        timeout: 300
+
+    - name: Re-resume upgrade systemd service
+      ansible.builtin.systemd:
+        name: securedrop-noble-migration-upgrade
+        state: started
+
+    # This final check should actually succeed.
+    - name: Wait for migration to complete
+      ansible.builtin.wait_for:
+        path: /etc/securedrop-noble-migration-state.json
+        search_regex: '"finished":"Done"'
+        sleep: 5
+        timeout: 300

install_files/ansible-base/securedrop-noble-migration.yml

@@ -0,0 +1,66 @@
+---
+- name: Disable OSSEC notifications
+  hosts: securedrop_monitor_server
+  max_fail_percentage: 0
+  any_errors_fatal: yes
+  environment:
+    LC_ALL: C
+  tasks:
+    - name: Disable OSSEC notifications
+      ansible.builtin.lineinfile:
+        path: /var/ossec/etc/ossec.conf
+        regexp: '<email_alert_level>7</email_alert_level>'
+        line: '<email_alert_level>15</email_alert_level>'
+      register: ossec_config
+
+    - name: Restart OSSEC service
+      ansible.builtin.systemd:
+        name: ossec
+        state: restarted
+      when: ossec_config.changed
+  become: yes
+
+- name: Perform upgrade on application server
+  hosts: securedrop_application_server
+  max_fail_percentage: 0
+  any_errors_fatal: yes
+  environment:
+    LC_ALL: C
+  roles:
+    - role: noble-migration
+      tags: noble-migration
+  become: yes
+
+- name: Perform upgrade on monitor server
+  hosts: securedrop_monitor_server
+  max_fail_percentage: 0
+  any_errors_fatal: yes
+  environment:
+    LC_ALL: C
+  roles:
+    - role: noble-migration
+      tags: noble-migration
+  become: yes
+
+# This is not really necessary since the mon migration will restore the old
+# configuration back, but let's include it for completeness.
+- name: Restore OSSEC notifications
+  hosts: securedrop_monitor_server
+  max_fail_percentage: 0
+  any_errors_fatal: yes
+  environment:
+    LC_ALL: C
+  tasks:
+    - name: Re-enable OSSEC email alerts
+      ansible.builtin.lineinfile:
+        path: /var/ossec/etc/ossec.conf
+        regexp: '<email_alert_level>15</email_alert_level>'
+        line: '<email_alert_level>7</email_alert_level>'
+      register: ossec_config
+
+    - name: Restart OSSEC service
+      ansible.builtin.systemd:
+        name: ossec
+        state: restarted
+      when: ossec_config.changed
+  become: yes