From e217766d31730eb2874ec76eca0bbfa077dbeb1c Mon Sep 17 00:00:00 2001 From: Kunal Mehta Date: Fri, 24 Jan 2025 13:33:52 -0500 Subject: [PATCH] Enable manual noble upgrade Admins can run `./securedrop-admin noble_migration` to trigger a manual noble migration. At a high level the playbook: * disables OSSEC notifications * triggers the app upgrade, waiting through two reboots * triggers the mon upgrade, again waiting through reboots * re-enables OSSEC notifications The most complicated part is how we for the reboots. We first have a `wait_for` that looks for a specific stage in the state file. Because the upgrade script writes the state file and then immediately reboots, it should never actually succeed and fail because the connection is interrupted. So we set `ignore_unreachable` and `ignore_errors`, and the next block is `wait_for_connection` for the server to come back up. There is a delay before we begin checking just in case the wait_for did succeed and we need to wait for the reboot to happen. Because of this sequencing, there isn't any support for the playbook failing mid-host and restarting it. It is probably unnecessary since, once started, the upgrade will automatically finish by itself. The script does support one host already being upgraded and the other still needing migration. So if e.g. app migration fails, you can manually fix the host, let it auto finish the upgrade, and then re-run the playbook to migrate mon. Fixes #7416. --- admin/securedrop_admin/__init__.py | 16 ++++ .../roles/noble-migration/tasks/main.yml | 92 +++++++++++++++++++ .../securedrop-noble-migration.yml | 66 +++++++++++++ 3 files changed, 174 insertions(+) create mode 100644 install_files/ansible-base/roles/noble-migration/tasks/main.yml create mode 100644 install_files/ansible-base/securedrop-noble-migration.yml diff --git a/admin/securedrop_admin/__init__.py b/admin/securedrop_admin/__init__.py index 5c197337a1..c1375c3ec8 100755 --- a/admin/securedrop_admin/__init__.py +++ b/admin/securedrop_admin/__init__.py @@ -1110,6 +1110,19 @@ def get_logs(args: argparse.Namespace) -> int: return 0 +@update_check_required("noble_migration") +def noble_migration(args: argparse.Namespace) -> int: + """Upgrade to Ubuntu Noble""" + sdlog.info("Beginning the upgrade to Ubuntu Noble") + ansible_cmd = ansible_command() + [ + os.path.join(args.ansible_path, "securedrop-noble-migration.yml"), + ] + + subprocess.check_call(ansible_cmd, cwd=args.ansible_path) + sdlog.info("Upgrade to Ubuntu Noble complete!") + return 0 + + def set_default_paths(args: argparse.Namespace) -> argparse.Namespace: if not args.ansible_path: args.ansible_path = args.root + "/install_files/ansible-base" @@ -1209,6 +1222,9 @@ class ArgParseFormatterCombo( parse_logs = subparsers.add_parser("logs", help=get_logs.__doc__) parse_logs.set_defaults(func=get_logs) + parse_noble_migration = subparsers.add_parser("noble_migration", help=noble_migration.__doc__) + parse_noble_migration.set_defaults(func=noble_migration) + parse_reset_ssh = subparsers.add_parser("reset_admin_access", help=reset_admin_access.__doc__) parse_reset_ssh.set_defaults(func=reset_admin_access) diff --git a/install_files/ansible-base/roles/noble-migration/tasks/main.yml b/install_files/ansible-base/roles/noble-migration/tasks/main.yml new file mode 100644 index 0000000000..b9dc47a63d --- /dev/null +++ b/install_files/ansible-base/roles/noble-migration/tasks/main.yml @@ -0,0 +1,92 @@ +--- + +- name: Check migration JSON on mon server + ansible.builtin.slurp: + src: /etc/securedrop-noble-migration-state.json + register: migration_json + ignore_errors: yes + +- name: Skip migration if already done + set_fact: + already_finished: "{{ not migration_json.failed and (migration_json.content | b64decode | from_json)['finished'] == 'Done' }}" + +- name: Perform migration + when: not already_finished + block: + - name: Instruct upgrade to begin + ansible.builtin.copy: + # It's okay to enable both app and mon here to simplify the logic, + # as this only affects the server the file is updated on. + content: | + { + "app": {"enabled": true, "bucket": 5}, + "mon": {"enabled": true, "bucket": 5} + } + dest: /usr/share/securedrop/noble-upgrade.json + + # Start the systemd service manually to avoid waiting for the timer to pick it up + - name: Start upgrade systemd service + ansible.builtin.systemd: + name: securedrop-noble-migration-upgrade + state: started + + # Wait until we've finished the PendingUpdates stage. It's highly unlikely + # we'll ever successfully complete this stage because as soon as the script + # reaches finishes that stage, it reboots. Most likely this step will fail + # as unreachable, which we ignore and wait_for_connection. + - name: Wait for pending updates to be applied + ansible.builtin.wait_for: + path: /etc/securedrop-noble-migration-state.json + search_regex: '"finished":"PendingUpdates"' + sleep: 1 + timeout: 300 + ignore_unreachable: yes + ignore_errors: yes + + - name: Wait for the first reboot + ansible.builtin.wait_for_connection: + connect_timeout: 20 + sleep: 5 + delay: 10 + timeout: 300 + + # Start the systemd service manually to avoid waiting for the timer to pick it up + - name: Resume upgrade systemd service + ansible.builtin.systemd: + name: securedrop-noble-migration-upgrade + state: started + + - debug: + msg: "The upgrade is in progress; it may take up to 30 minutes." + + # Same as above, this will most likely fail as unreachable when the server + # actually reboots. + - name: Wait for system upgrade to noble + ansible.builtin.wait_for: + path: /etc/securedrop-noble-migration-state.json + search_regex: '"finished":"Reboot"' + sleep: 5 + # Should finish in less than 30 minutes + timeout: 1800 + ignore_unreachable: yes + ignore_errors: yes + + - name: Wait for the second reboot + ansible.builtin.wait_for_connection: + connect_timeout: 20 + sleep: 5 + delay: 10 + timeout: 300 + + - name: Re-resume upgrade systemd service + ansible.builtin.systemd: + name: securedrop-noble-migration-upgrade + state: started + + # This final check should actually succeed. + - name: Wait for migration to complete + ansible.builtin.wait_for: + path: /etc/securedrop-noble-migration-state.json + search_regex: '"finished":"Done"' + sleep: 5 + timeout: 300 diff --git a/install_files/ansible-base/securedrop-noble-migration.yml b/install_files/ansible-base/securedrop-noble-migration.yml new file mode 100644 index 0000000000..adc49ffce7 --- /dev/null +++ b/install_files/ansible-base/securedrop-noble-migration.yml @@ -0,0 +1,66 @@ +--- +- name: Disable OSSEC notifications + hosts: securedrop_monitor_server + max_fail_percentage: 0 + any_errors_fatal: yes + environment: + LC_ALL: C + tasks: + - name: Disable OSSEC notifications + ansible.builtin.lineinfile: + path: /var/ossec/etc/ossec.conf + regexp: '7' + line: '15' + register: ossec_config + + - name: Restart OSSEC service + ansible.builtin.systemd: + name: ossec + state: restarted + when: ossec_config.changed + become: yes + +- name: Perform upgrade on application server + hosts: securedrop_application_server + max_fail_percentage: 0 + any_errors_fatal: yes + environment: + LC_ALL: C + roles: + - role: noble-migration + tags: noble-migration + become: yes + +- name: Perform upgrade on monitor server + hosts: securedrop_monitor_server + max_fail_percentage: 0 + any_errors_fatal: yes + environment: + LC_ALL: C + roles: + - role: noble-migration + tags: noble-migration + become: yes + +# This is not really necessary since the mon migration will restore the old +# configuration back, but let's include it for completeness. +- name: Restore OSSEC notifications + hosts: securedrop_monitor_server + max_fail_percentage: 0 + any_errors_fatal: yes + environment: + LC_ALL: C + tasks: + - name: Re-enable OSSEC email alerts + ansible.builtin.lineinfile: + path: /var/ossec/etc/ossec.conf + regexp: '(.*?)' + line: '7' + register: ossec_config + + - name: Restart OSSEC service + ansible.builtin.systemd: + name: ossec + state: restarted + when: ossec_config.changed + become: yes