From 8d36fafee7f0a9ef011e15f0ddf32336d0e060a6 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Tue, 29 Oct 2019 15:53:05 +0000 Subject: [PATCH 1/5] Deploy fastpath Push sshfeeder pubkey on collectors --- ansible/deploy-fastpath.yml | 6 +++ ansible/inventory | 3 ++ ansible/roles/fastpath/tasks/main.yml | 42 +++++++++++++++++++ .../roles/fastpath/templates/fastpath.conf.j2 | 7 ++++ 4 files changed, 58 insertions(+) create mode 100644 ansible/deploy-fastpath.yml create mode 100644 ansible/roles/fastpath/tasks/main.yml create mode 100644 ansible/roles/fastpath/templates/fastpath.conf.j2 diff --git a/ansible/deploy-fastpath.yml b/ansible/deploy-fastpath.yml new file mode 100644 index 00000000..eadd362a --- /dev/null +++ b/ansible/deploy-fastpath.yml @@ -0,0 +1,6 @@ +--- +- import_playbook: ansible-version.yml + +- hosts: fastpath.ooni.nu + roles: + - role: fastpath diff --git a/ansible/inventory b/ansible/inventory index 5c8615fe..bb5964b4 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -201,6 +201,9 @@ staticiforme.torproject.org # tor LDAP [no_nodeexp:children] no_passwd # no passwd => no nodeexp user ;-) +[fastpath] +fastpath.ooni.nu + ######################################################################## # TO DELETE. # Stopped VMs that should be deleted from GH and DNS after some grace period: diff --git a/ansible/roles/fastpath/tasks/main.yml b/ansible/roles/fastpath/tasks/main.yml new file mode 100644 index 00000000..884174a6 --- /dev/null +++ b/ansible/roles/fastpath/tasks/main.yml @@ -0,0 +1,42 @@ +--- + +# Deploy the fastpath +# +# Deploy .deb package to the fastpath host +# +# Fetch SSH pubkey and deploy it on the collector hosts +# +# Set the hostname of the collectors in the fastpath config file + +# Usage: place the .deb file in files/ +# update the version number as needed + +- name: copy fastpath .deb package + copy: + src: files/fastpath_0.1_all.deb + dest: /root/ + owner: root + group: root + mode: '0644' + +# Leave a copy of the .deb in /root as an emergency backup for rollbacks + +- name: install .deb using dpkg + apt: + deb: '{{ item }}' + state: present + with_items: + - /root/fastpath_0.1_all.deb + +- name: Fetch sshfeeder pubkey from fastpath host + slurp: + src: /var/lib/fastpath/ssh/id_ed25519.pub + register: sshfeeder_pubkey_b64encoded + +- name: generate fastpath.conf file with collectors + template: + owner: "root" + group: "root" + mode: '0644' + src: "fastpath.conf.j2" + dest: "/etc/fastpath.conf" diff --git a/ansible/roles/fastpath/templates/fastpath.conf.j2 b/ansible/roles/fastpath/templates/fastpath.conf.j2 new file mode 100644 index 00000000..a6f4de97 --- /dev/null +++ b/ansible/roles/fastpath/templates/fastpath.conf.j2 @@ -0,0 +1,7 @@ +# Deployed by ansible +# deploy-fastpath.yml +# roles/fastpath/tasks/main.yml +# roles/fastpath/templates/fastpath.conf.j2 + +[DEFAULT] +collectors = {{ groups['have_collector'] | join(' ') }} From 7502aefe6967f70f324f22aebe1fa9dd81994ba1 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Tue, 29 Oct 2019 18:50:17 +0000 Subject: [PATCH 2/5] Update key --- ansible/roles/ooni-collector/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/ooni-collector/defaults/main.yml b/ansible/roles/ooni-collector/defaults/main.yml index 7dd37401..08a07307 100644 --- a/ansible/roles/ooni-collector/defaults/main.yml +++ b/ansible/roles/ooni-collector/defaults/main.yml @@ -7,5 +7,5 @@ collector_tag: '20190221-0980b82a' collector_data_dir: '{{ CHANGE_ME }}' # it varies across collectors collector_domain: '{{ CHANGE_ME }}' with_nginx: yes -sshfeeder_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAID3MZP0tYUzaZOZa8JhRIx3PiTXwXfwQdTPa/RwDmwcA" +sshfeeder_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHAxyZVlDHc50fim5qWkf+yQhVpVR6d4btNBE8xb4XCd fastpath" ... From 26819d0efb35caeea9665d7b018ecd47fa1cc7ff Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Wed, 30 Oct 2019 12:42:42 +0000 Subject: [PATCH 3/5] Remove sshfeeder key slurp --- ansible/roles/fastpath/tasks/main.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ansible/roles/fastpath/tasks/main.yml b/ansible/roles/fastpath/tasks/main.yml index 884174a6..77d59c89 100644 --- a/ansible/roles/fastpath/tasks/main.yml +++ b/ansible/roles/fastpath/tasks/main.yml @@ -28,11 +28,6 @@ with_items: - /root/fastpath_0.1_all.deb -- name: Fetch sshfeeder pubkey from fastpath host - slurp: - src: /var/lib/fastpath/ssh/id_ed25519.pub - register: sshfeeder_pubkey_b64encoded - - name: generate fastpath.conf file with collectors template: owner: "root" From fff0d16938f65daf2c0519775417d29803080a35 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Wed, 30 Oct 2019 12:42:59 +0000 Subject: [PATCH 4/5] Add sshfeeder to collector group --- ansible/roles/ooni-collector/tasks/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ansible/roles/ooni-collector/tasks/main.yml b/ansible/roles/ooni-collector/tasks/main.yml index e750cd5a..e8ec5705 100644 --- a/ansible/roles/ooni-collector/tasks/main.yml +++ b/ansible/roles/ooni-collector/tasks/main.yml @@ -144,6 +144,8 @@ - name: Setup the sshfeeder account user: name: sshfeeder + group: sshfeeder + groups: "{{ collector_user }}" createhome: yes append: yes shell: /bin/bash From 9bb02b6b5621f43dee657860a5ecd34b5b053c62 Mon Sep 17 00:00:00 2001 From: Federico Ceratto Date: Wed, 27 Nov 2019 12:39:02 +0000 Subject: [PATCH 5/5] Add known_hosts creation, minor fixes --- ansible/inventory | 3 -- ansible/roles/fastpath/tasks/main.yml | 39 +++++++++++-------- .../roles/fastpath/templates/fastpath.conf.j2 | 4 +- .../roles/fastpath/templates/known_collectors | 4 ++ ansible/roles/ooni-collector/tasks/main.yml | 2 + 5 files changed, 31 insertions(+), 21 deletions(-) create mode 100644 ansible/roles/fastpath/templates/known_collectors diff --git a/ansible/inventory b/ansible/inventory index bb5964b4..5c8615fe 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -201,9 +201,6 @@ staticiforme.torproject.org # tor LDAP [no_nodeexp:children] no_passwd # no passwd => no nodeexp user ;-) -[fastpath] -fastpath.ooni.nu - ######################################################################## # TO DELETE. # Stopped VMs that should be deleted from GH and DNS after some grace period: diff --git a/ansible/roles/fastpath/tasks/main.yml b/ansible/roles/fastpath/tasks/main.yml index 77d59c89..edf6aeb3 100644 --- a/ansible/roles/fastpath/tasks/main.yml +++ b/ansible/roles/fastpath/tasks/main.yml @@ -11,27 +11,34 @@ # Usage: place the .deb file in files/ # update the version number as needed -- name: copy fastpath .deb package - copy: - src: files/fastpath_0.1_all.deb - dest: /root/ - owner: root - group: root - mode: '0644' - -# Leave a copy of the .deb in /root as an emergency backup for rollbacks +#- name: pull fastpath .deb package +# copy: +# src: files/fastpath_0.1_all.deb +# dest: /root/ +# owner: root +# group: root +# mode: '0644' -- name: install .deb using dpkg - apt: - deb: '{{ item }}' - state: present - with_items: - - /root/fastpath_0.1_all.deb +#- name: install .deb using dpkg +# apt: +# deb: '{{ item }}' +# state: present +# with_items: +# - /root/fastpath_0.1_all.deb -- name: generate fastpath.conf file with collectors +- name: generate fastpath.conf file with active collectors names in it template: owner: "root" group: "root" mode: '0644' src: "fastpath.conf.j2" dest: "/etc/fastpath.conf" + +- name: fill active collectors SSH pubkeys into fastpath's known_hosts + template: + owner: "fastpath" + group: "fastpath" + mode: '0644' + src: "known_collectors" + dest: /var/lib/fastpath/ssh/known_hosts + diff --git a/ansible/roles/fastpath/templates/fastpath.conf.j2 b/ansible/roles/fastpath/templates/fastpath.conf.j2 index a6f4de97..dc06bb67 100644 --- a/ansible/roles/fastpath/templates/fastpath.conf.j2 +++ b/ansible/roles/fastpath/templates/fastpath.conf.j2 @@ -1,7 +1,7 @@ -# Deployed by ansible +# Deployed by https://github.com/ooni/sysadmin/blob/master/ansible/roles/fastpath # deploy-fastpath.yml # roles/fastpath/tasks/main.yml # roles/fastpath/templates/fastpath.conf.j2 [DEFAULT] -collectors = {{ groups['have_collector'] | join(' ') }} +collectors = {{ groups['active_collector'] | join(' ') }} diff --git a/ansible/roles/fastpath/templates/known_collectors b/ansible/roles/fastpath/templates/known_collectors new file mode 100644 index 00000000..2f5b326e --- /dev/null +++ b/ansible/roles/fastpath/templates/known_collectors @@ -0,0 +1,4 @@ +# Deployed by https://github.com/ooni/sysadmin/blob/master/ansible/roles/fastpath/templates/known_collectors +{% for host in groups.active_collector %} +{{ lookup('pipe', 'grep -F -e {}, {}/../ext/known_hosts'.format(host, inventory_dir)) }} +{% endfor %} diff --git a/ansible/roles/ooni-collector/tasks/main.yml b/ansible/roles/ooni-collector/tasks/main.yml index e8ec5705..3dfeef67 100644 --- a/ansible/roles/ooni-collector/tasks/main.yml +++ b/ansible/roles/ooni-collector/tasks/main.yml @@ -151,9 +151,11 @@ shell: /bin/bash home: "/home/sshfeeder" system: no + tags: collector_ssh - name: Set authorized key for sshfeeder authorized_key: user: sshfeeder state: present key: "{{ sshfeeder_key }}" + tags: collector_ssh