Skip to content

Commit 10841ee

Browse files
jackhodgkissjackhodgkiss
committed
feat: support migration from Consul to Raft 
1 parent f6990a9 commit 10841ee

File tree

6 files changed

+240
-0
lines changed

6 files changed

+240
-0
lines changed

‎.github/workflows/pull_request.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ jobs:
2626
- openbao
2727
- openbao_ha
2828
- vault
29+
- vault_raft_migration
2930
steps:
3031
- name: Github Checkout 🛎
3132
uses: actions/checkout@v4

‎roles/vault/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@ s (default: Omitted)
5151
* `vault_write_keys_file`: Whether to write the root token and unseal keys to a file. Default `false`
5252
* `vault_write_keys_file_host`: Host on which to write root token and unseal keys. Default `localhost`
5353
* `vault_write_keys_file_path`: Path of file to write root token and unseal keys. Default `vault-keys.json`
54+
* `vault_storage_type`: The type of storage to be used by vault either `Consul` or `Raft`. Note if vault is already deployed with `Consul` then you can change this inconjunction with `vault_migrate_consul_to_raft` to perform a one-way migration from `Consul` to `Raft`. Default `consul`.
55+
* `vault_migrate_consul_to_raft`: if set true perform the necessary steps to convert the storage backend from `Consul` to `Raft`. Default `false`
56+
* `vault_raft_leaders`: List of IPs belonging to Raft leaders. Expected that the first and only entry is the IP address of the first Vault instance as this would be initialised whereas as the others will not.
5457

5558
Root and unseal keys
5659
--------------------

‎roles/vault/defaults/main.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,10 @@ vault_tls_cert: ""
2424

2525
vault_config_dir: ""
2626

27+
vault_raft_leaders: []
28+
29+
vault_storage_type: "consul"
30+
2731
vault_config: >
2832
{
2933
"cluster_name": "{{ vault_cluster_name }}",
@@ -49,10 +53,17 @@ vault_config: >
4953
{% endif %}
5054
}],
5155
"storage": {
56+
{% if vault_storage_type == 'consul' %}
5257
"consul": {
5358
"address": "127.0.0.1:{{ consul_bind_port }}",
5459
"path": "vault/"
5560
}
61+
{% elif vault_storage_type == 'raft' %}
62+
"raft": {
63+
"node_id": "raft_{{ inventory_hostname }}",
64+
"path": "/vault/file"
65+
}
66+
{% endif %}
5667
},
5768
"telemetry": {
5869
"prometheus_retention_time": "30s",
@@ -97,3 +108,16 @@ vault_write_keys_file: false
97108
vault_write_keys_file_host: localhost
98109
# Path of file to write root token and unseal keys.
99110
vault_write_keys_file_path: vault-keys.json
111+
112+
vault_migrate_consul_to_raft: false
113+
114+
vault_raft_migration_config: >
115+
storage_source "consul" {
116+
address = "{{ consul_bind_ip }}:{{ consul_bind_port }}"
117+
path = "vault/"
118+
}
119+
storage_destination "raft" {
120+
node_id = "raft_{{ inventory_hostname }}"
121+
path = "/vault/file"
122+
}
123+
cluster_addr = "{{ vault_protocol }}://{{ vault_bind_address }}:8201"

‎roles/vault/tasks/main.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,5 +8,9 @@
88
- name: "Deploy Consul"
99
import_tasks: consul.yml
1010

11+
- name: "Migrate from Consul to Raft"
12+
import_tasks: raft_migration.yml
13+
when: vault_migrate_consul_to_raft | bool
14+
1115
- name: "Deploy Vault"
1216
import_tasks: vault.yml

‎roles/vault/tasks/raft_migration.yml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
---
2+
- name: Take Consul snapshot
3+
ansible.builtin.command: >
4+
docker exec {{ consul_docker_name }} /bin/sh -c "consul snapshot save /consul/data/backup_{{ ansible_date_time.iso8601_basic_short }}.snap"
5+
become: true
6+
register: consul_snapshot
7+
changed_when: true
8+
9+
- name: Write migration configuration
10+
ansible.builtin.copy:
11+
content: "{{ vault_raft_migration_config }}"
12+
dest: "{{ vault_config_dir }}/migration.hcl"
13+
mode: "0644"
14+
become: true
15+
16+
- name: Perform migration
17+
ansible.builtin.command: >
18+
docker exec {{ vault_docker_name }} /bin/sh -c "vault operator migrate -config /vault/config/migration.hcl"
19+
become: true
20+
changed_when: true
21+
22+
- name: Change ownership of raft data directory
23+
ansible.builtin.command: >
24+
docker exec {{ vault_docker_name }} /bin/sh -c "chown -R vault:vault /vault/file"
25+
become: true
26+
changed_when: true

‎tests/test_vault_raft_migration.yml‎

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
---
2+
- name: Prepare for vault role
3+
gather_facts: true
4+
hosts: consul
5+
vars:
6+
vault_config_dir: "/etc/vault"
7+
vault_log_keys: true
8+
vault_protocol: http
9+
vault_set_keys_fact: true
10+
vault_write_keys_file: true
11+
tasks:
12+
- name: Ensure /etc/vault exists
13+
file:
14+
path: /etc/vault
15+
state: directory
16+
mode: "0700"
17+
become: true
18+
19+
- name: Include vault role
20+
include_role:
21+
name: vault
22+
23+
- name: Include vault role (idempotence test)
24+
include_role:
25+
name: vault
26+
27+
- name: Include Vault keys
28+
ansible.builtin.include_vars:
29+
file: "vault-keys.json"
30+
name: vault_keys
31+
32+
- name: Unseal vault
33+
include_role:
34+
name: vault_unseal
35+
vars:
36+
vault_unseal_keys: "{{ vault_keys.keys_base64 }}"
37+
38+
- name: Configure PKI - create root/intermediate and generate certificates
39+
vars:
40+
vault_pki_certificate_subject:
41+
- role: 'ServerCert'
42+
common_name: "OS-CERT-TEST"
43+
extra_params:
44+
ttl: "8760h"
45+
ip_sans: "127.0.0.1"
46+
alt_names: "example.com"
47+
exclude_cn_from_sans: true
48+
vault_pki_certificates_directory: "/tmp/"
49+
vault_pki_generate_certificates: true
50+
vault_pki_intermediate_ca_name: "OS-TLS-INT"
51+
vault_pki_intermediate_create: true
52+
vault_pki_intermediate_roles:
53+
- name: "ServerCert"
54+
config:
55+
max_ttl: 8760h
56+
ttl: 8760h
57+
allow_any_name: true
58+
allow_ip_sans: true
59+
require_cn: false
60+
server_flag: true
61+
key_type: rsa
62+
key_bits: 4096
63+
country: ["UK"]
64+
locality: ["Bristol"]
65+
organization: ["StackHPC"]
66+
ou: ["HPC"]
67+
vault_pki_root_ca_name: "OS-TLS-ROOT"
68+
vault_pki_root_create: true
69+
vault_pki_write_certificate_files: true
70+
vault_pki_write_int_ca_to_file: true
71+
vault_pki_write_pem_bundle: false
72+
vault_pki_write_root_ca_to_file: true
73+
vault_token: "{{ vault_keys.root_token }}"
74+
block:
75+
- name: Configure PKI - create root/intermediate and generate certificates
76+
include_role:
77+
name: vault_pki
78+
79+
- name: Configure PKI - create root/intermediate and generate certificates (idempotence test)
80+
include_role:
81+
name: vault_pki
82+
83+
- name: Configure PKI - generate certificate pem bundle
84+
vars:
85+
vault_pki_certificate_subject:
86+
- role: 'ServerCert'
87+
common_name: "OS-CERT-TEST2"
88+
extra_params:
89+
ttl: "8760h"
90+
ip_sans: "192.168.38.72"
91+
exclude_cn_from_sans: true
92+
vault_pki_certificates_directory: "/tmp/"
93+
vault_pki_generate_certificates: true
94+
vault_pki_intermediate_ca_name: "OS-TLS-INT"
95+
vault_pki_intermediate_create: false
96+
vault_pki_root_ca_name: "OS-TLS-ROOT"
97+
vault_pki_root_create: false
98+
vault_pki_write_certificate_files: true
99+
vault_pki_write_pem_bundle: true
100+
vault_token: "{{ vault_keys.root_token }}"
101+
block:
102+
- name: Configure PKI - generate certificate pem bundle
103+
include_role:
104+
name: vault_pki
105+
106+
- name: Configure PKI - generate certificate pem bundle (idempotence test)
107+
include_role:
108+
name: vault_pki
109+
110+
- name: Validate if certificates exist
111+
stat:
112+
path: "/tmp/{{ item }}"
113+
register: stat_result
114+
failed_when: not stat_result.stat.exists
115+
loop:
116+
- OS-CERT-TEST.crt
117+
- OS-CERT-TEST2.pem
118+
119+
- name: Concatenate CAs
120+
shell: |
121+
cat /tmp/OS-TLS-ROOT.pem /tmp/OS-TLS-INT.crt > /tmp/CA-CHAIN.pem
122+
args:
123+
executable: /bin/bash
124+
become: true
125+
changed_when: true
126+
127+
- name: Verify certificate chain
128+
command: |
129+
openssl verify -CAfile /tmp/CA-CHAIN.pem
130+
/tmp/{{ item }}
131+
register: verify_result
132+
failed_when: verify_result.rc != 0
133+
loop:
134+
- OS-CERT-TEST.crt
135+
- OS-CERT-TEST2.pem
136+
changed_when: false
137+
138+
- name: Migrate vault to raft
139+
include_role:
140+
name: vault
141+
vars:
142+
vault_storage_type: raft
143+
vault_migrate_consul_to_raft: true
144+
145+
- name: Unseal vault
146+
include_role:
147+
name: vault_unseal
148+
vars:
149+
vault_unseal_keys: "{{ vault_keys.keys_base64 }}"
150+
151+
- name: Validate vault is using raft
152+
ansible.builtin.command: >
153+
docker exec -e VAULT_ADDR=http://127.0.0.1:8200 vault /bin/sh -c "vault status -format=json"
154+
register: vault_status
155+
become: true
156+
changed_when: false
157+
158+
- name: Validate vault is using raft
159+
ansible.builtin.assert:
160+
that:
161+
- vault_status.stdout | from_json | json_query('storage_type') == 'raft'
162+
fail_msg: "Vault is not using raft storage backend"
163+
success_msg: "Vault is using raft storage backend"
164+
165+
- name: Read CA certificate from vault
166+
community.hashi_vault.vault_read:
167+
url: http://127.0.0.1:8200
168+
path: OS-TLS-ROOT/cert/ca
169+
token: "{{ vault_keys.root_token }}"
170+
register: vault_ca_cert
171+
172+
- name: Read CA from file
173+
ansible.builtin.slurp:
174+
src: /tmp/OS-TLS-ROOT.pem
175+
register: ca_chain
176+
177+
- name: Validate ROOT CA
178+
ansible.builtin.assert:
179+
that:
180+
- vault_ca_cert.data.data.certificate == (ca_chain.content | b64decode).rstrip('\n')
181+
fail_msg: "ROOT CA certificate do not match"
182+
success_msg: "ROOT CA certificate do match"

0 commit comments

Comments
 (0)