Skip to content

Commit f3f60b9

Browse files
sjpbsjpb
authored
Support manila fileshares (cephfs) (#344)
* add support for manila to common environment * use manila share for /scratch in stackhpc env w/ tests * make home volume optional in skeleton TF * remove share creation from CI TF * add manila UI for caas * support manila- or nfs/volume- based home dirs in caas * remove manila config from UI * add optional platform-lifecycle manila share for homedirs for caas * add home and project manila config for caas * tweak home volume size UI description to account for shares * fix caas manila config typo * tidy PR diff * bump fatimage to include manila client * Revert commit "tweak home volume size UI description to account for shares" This reverts commit 3d9cfba. * add manila UI for caas * add defaults for new caas manila extravars, where possible * make cluster_home_manila_share_type optional for when default share type is defined * address review comments * bump manila requirement after role release * default usage of home manila share to match project share for caas * remove caas manila-specific ui-meta
1 parent b902115 commit f3f60b9

File tree

17 files changed

+122
-25
lines changed

17 files changed

+122
-25
lines changed

‎ansible/fatimage.yml

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,16 @@
4141
tasks_from: client-install.yml
4242
when: "'freeipa_client' in group_names"
4343

44-
# - import_playbook: filesystems.yml
45-
- name: nfs
44+
# - import_playbook: filesystems.yml:
45+
- name: Install nfs packages
4646
dnf:
4747
name: nfs-utils
48+
when: "'nfs' in group_names"
49+
- name: Install Manila client packages
50+
include_role:
51+
name: stackhpc.os-manila-mount
52+
tasks_from: install.yml
53+
when: "'manila' in group_names"
4854

4955
- import_playbook: extras.yml
5056

‎ansible/filesystems.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,11 @@
1616
tasks:
1717
- include_role:
1818
name: stackhpc.nfs
19+
20+
- name: Setup Manila share mounts
21+
hosts: manila
22+
become: true
23+
tags: manila
24+
tasks:
25+
- include_role:
26+
name: stackhpc.os-manila-mount

‎ansible/roles/cluster_infra/templates/resources.tf.j2

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,22 @@ resource "openstack_blockstorage_volume_v3" "state" {
7979
size = "{{ state_volume_size }}"
8080
}
8181

82+
{% if cluster_home_manila_share | bool %}
83+
resource "openstack_sharedfilesystem_share_v2" "home" {
84+
name = "{{ cluster_name }}-home"
85+
description = "Home for cluster"
86+
share_proto = "CEPHFS"
87+
share_type = {{ '"' + cluster_home_manila_share_type + '"' | default('null') }}
88+
size = "{{ home_volume_size }}"
89+
}
90+
91+
resource "openstack_sharedfilesystem_share_access_v2" "home" {
92+
share_id = openstack_sharedfilesystem_share_v2.home.id
93+
access_type = "cephx"
94+
access_to = "cluster_{{ cluster_id }}"
95+
access_level = "rw"
96+
}
97+
{% else %}
8298
resource "openstack_blockstorage_volume_v3" "home" {
8399
name = "{{ cluster_name }}-home"
84100
description = "Home for control node"
@@ -89,6 +105,7 @@ resource "openstack_blockstorage_volume_v3" "home" {
89105
{% endif %}
90106
{% endif %}
91107
}
108+
{% endif %}
92109

93110
######
94111
###### Cluster network
@@ -334,13 +351,15 @@ resource "openstack_compute_instance_v2" "control" {
334351
uuid = openstack_blockstorage_volume_v3.state.id
335352
}
336353

354+
{% if not cluster_home_manila_share | bool %}
337355
# home volume:
338356
block_device {
339357
destination_type = "volume"
340358
source_type = "volume"
341359
boot_index = -1
342360
uuid = openstack_blockstorage_volume_v3.home.id
343361
}
362+
{% endif %}
344363

345364
# Use cloud-init to a) inject SSH keys b) configure volumes
346365
user_data = <<-EOF
@@ -359,12 +378,14 @@ resource "openstack_compute_instance_v2" "control" {
359378
- {{ ssh_key }}
360379
{%- endfor %}
361380
bootcmd:
362-
%{for volume in [openstack_blockstorage_volume_v3.state, openstack_blockstorage_volume_v3.home]}
381+
%{for volume in [openstack_blockstorage_volume_v3.state, {% if not cluster_home_manila_share | bool %} openstack_blockstorage_volume_v3.home {% endif %}]}
363382
- BLKDEV=$(readlink -f $(ls /dev/disk/by-id/*${substr(volume.id, 0, 20)}* | head -n1 )); blkid -o value -s TYPE $BLKDEV || mke2fs -t ext4 -L ${lower(split(" ", volume.description)[0])} $BLKDEV
364383
%{endfor}
365384
mounts:
366385
- [LABEL=state, {{ appliances_state_dir }}, auto]
386+
{% if not cluster_home_manila_share | bool %}
367387
- [LABEL=home, /exports/home, auto]
388+
{% endif %}
368389
EOF
369390
}
370391

‎environments/.caas/inventory/extra_groups

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,7 @@ cluster
77
[zenith:children]
88
grafana
99
openondemand
10+
11+
[manila:children]
12+
login
13+
compute

‎environments/.caas/inventory/group_vars/all/cluster.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,7 @@ openondemand_servername_default: "{{ hostvars[groups['openstack'][0]].cluster_ga
2020
openondemand_servername: "{{ zenith_fqdn_ood | default(openondemand_servername_default) }}"
2121

2222
appliances_state_dir: /var/lib/state
23+
24+
# Defaults for caas-provided extravars:
25+
cluster_project_manila_share: false
26+
cluster_home_manila_share: "{{ cluster_project_manila_share }}"

‎environments/.caas/inventory/group_vars/all/manila.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
caas_manila_home:
2+
share_name: "{{ cluster_name }}-home"
3+
mount_path: /home
4+
mount_user: root
5+
mount_group: root
6+
mount_mode: u=rwX,go=rX
7+
8+
cluster_project_manila_share_name: azimuth-project-share
9+
caas_manila_project:
10+
share_name: "{{ cluster_project_manila_share_name }}"
11+
mount_path: /project
12+
mount_user: root
13+
mount_group: root
14+
mount_mode: ugo=rwX
15+
16+
os_manila_mount_shares: "{{ ([caas_manila_home] if cluster_home_manila_share | bool else []) + ([caas_manila_project] if cluster_project_manila_share | bool else []) }}"

‎environments/.caas/inventory/group_vars/all/nfs.yml

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,20 @@
11
nfs_server: "{{ nfs_server_default }}"
22

3-
nfs_configurations:
4-
- comment: Export /exports/home from Slurm control node as /home
5-
nfs_enable:
6-
server: "{{ inventory_hostname in groups['control'] }}"
7-
clients: "{{ inventory_hostname in groups['cluster'] and inventory_hostname not in groups['control'] }}"
8-
nfs_export: "/exports/home" # assumes skeleton TF is being used
9-
nfs_client_mnt_point: "/home"
3+
caas_nfs_ood_state:
104
- comment: Export /var/lib/state from Slurm control node to OOD
115
nfs_enable:
126
server: "{{ inventory_hostname in groups['control'] }}"
137
clients: "{{ inventory_hostname in groups['openondemand'] }}"
148
nfs_export: "{{ appliances_state_dir }}"
159
nfs_client_mnt_point: "{{ appliances_state_dir }}"
1610
nfs_client_mnt_options: "x-systemd.required-by=zenith-ood.service,x-systemd.before=zenith-ood.service"
11+
12+
caas_nfs_home:
13+
- comment: Export /exports/home from Slurm control node as /home
14+
nfs_enable:
15+
server: "{{ inventory_hostname in groups['control'] }}"
16+
clients: "{{ inventory_hostname in groups['cluster'] and inventory_hostname not in groups['control'] }}"
17+
nfs_export: "/exports/home" # assumes skeleton TF is being used
18+
nfs_client_mnt_point: "/home"
19+
20+
nfs_configurations: "{{ caas_nfs_ood_state + (caas_nfs_home if not cluster_home_manila_share | bool else []) }}"

‎environments/.caas/ui-meta/slurm-infra.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ parameters:
5050

5151
- name: home_volume_size
5252
label: Home volume size (GB)
53-
description: The size of the cloud volume to use for home directories.
53+
description: The size of the cloud volume or share to use for home directories.
5454
kind: cloud.volume_size
5555
immutable: true
5656
options:

‎environments/.stackhpc/inventory/extra_groups

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,8 @@ cluster
2222
# [resolv_conf:children]
2323
# freeipa_client
2424
# --- end of FreeIPA example ---
25+
26+
[manila:children]
27+
# Allows demo; also installs manila client in fat image
28+
login
29+
compute

‎environments/.stackhpc/terraform/main.tf

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,14 @@
11
# This terraform configuration uses the "skeleton" terraform, so that is checked by CI.
22

3+
terraform {
4+
required_version = ">= 0.14"
5+
required_providers {
6+
openstack = {
7+
source = "terraform-provider-openstack/openstack"
8+
}
9+
}
10+
}
11+
312
variable "environment_root" {
413
type = string
514
description = "Path to environment root, automatically set by activate script"
@@ -13,7 +22,7 @@ variable "cluster_name" {
1322
variable "cluster_image" {
1423
description = "single image for all cluster nodes - a convenience for CI"
1524
type = string
16-
default = "openhpc-240116-1156-aa8dba7d" # https://github.com/stackhpc/ansible-slurm-appliance/pull/351
25+
default = "openhpc-240116-1604-b3563a08" # https://github.com/stackhpc/ansible-slurm-appliance/pull/344
1726
# default = "Rocky-8-GenericCloud-Base-8.9-20231119.0.x86_64.qcow2"
1827
}
1928

‎environments/common/inventory/group_vars/all/manila.yml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Default configuration for manila file shares, see
2+
# https://github.com/stackhpc/ansible-role-os-manila-mount
3+
# for all variable definitions, and override in your environment.
4+
5+
os_manila_mount_shares: []
6+
# - share_name:
7+
# share_user:
8+
# mount_path:
9+
# mount_user:
10+
# mount_group:
11+
# mount_mode:
12+
13+
# os_manila_mount_ceph_version: nautilus # role default for RockyLinux 8

‎environments/common/layouts/everything

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,3 +66,6 @@ openhpc
6666

6767
[proxy]
6868
# Hosts to configure http/s proxies - see ansible/roles/proxy/README.md
69+
70+
[manila]
71+
# Hosts to configure for manila fileshares

‎environments/skeleton/{{cookiecutter.environment}}/terraform/nodes.tf

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
locals {
22
user_data_path = "${var.environment_root}/cloud_init/${var.cluster_name}-%s.userdata.yml"
3+
control_volumes = concat([openstack_blockstorage_volume_v3.state], var.home_volume_size > 0 ? [openstack_blockstorage_volume_v3.home][0] : [])
34
}
45

56

@@ -100,20 +101,14 @@ resource "openstack_compute_instance_v2" "control" {
100101
delete_on_termination = true
101102
}
102103

103-
# state volume:
104-
block_device {
105-
destination_type = "volume"
106-
source_type = "volume"
107-
boot_index = -1
108-
uuid = openstack_blockstorage_volume_v3.state.id
109-
}
110-
111-
# home volume:
112-
block_device {
104+
dynamic "block_device" {
105+
for_each = local.control_volumes
106+
content {
113107
destination_type = "volume"
114108
source_type = "volume"
115109
boot_index = -1
116-
uuid = openstack_blockstorage_volume_v3.home.id
110+
uuid = block_device.value.id # actually openstack_blockstorage_volume_v3 id
111+
}
117112
}
118113

119114
network {
@@ -130,13 +125,15 @@ resource "openstack_compute_instance_v2" "control" {
130125
fqdn: ${var.cluster_name}-${each.key}.${var.cluster_name}.${var.cluster_domain_suffix}
131126
132127
bootcmd:
133-
%{for volume in [openstack_blockstorage_volume_v3.state, openstack_blockstorage_volume_v3.home]}
128+
%{for volume in local.control_volumes}
134129
- BLKDEV=$(readlink -f $(ls /dev/disk/by-id/*${substr(volume.id, 0, 20)}* | head -n1 )); blkid -o value -s TYPE $BLKDEV || mke2fs -t ext4 -L ${lower(split(" ", volume.description)[0])} $BLKDEV
135130
%{endfor}
136131
137132
mounts:
138133
- [LABEL=state, ${var.state_dir}]
134+
%{if var.home_volume_size > 0}
139135
- [LABEL=home, /exports/home]
136+
%{endif}
140137
EOF
141138

142139
}

‎environments/skeleton/{{cookiecutter.environment}}/terraform/variables.tf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ variable "state_volume_size" {
7070
variable "home_volume_size" {
7171
type = number
7272
description = "Size of state volume on control node, in GB"
73-
default = 100 # GB
73+
default = 100 # GB, 0 means no home volume
7474
}
7575

7676
variable "vnic_type" {

‎environments/skeleton/{{cookiecutter.environment}}/terraform/volumes.tf

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@ resource "openstack_blockstorage_volume_v3" "state" {
55
}
66

77
resource "openstack_blockstorage_volume_v3" "home" {
8+
9+
count = var.home_volume_size > 0 ? 1 : 0
10+
811
name = "${var.cluster_name}-home"
912
description = "Home for control node" # first word used to label filesystem
1013
size = var.home_volume_size

‎requirements.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
ansible==6.0.0
22
openstacksdk
33
python-openstackclient
4+
python-manilaclient
45
jmespath
56
passlib[bcrypt]==1.7.4
67
cookiecutter

‎requirements.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ roles:
2020
- src: https://github.com/OSC/ood-ansible.git
2121
name: osc.ood
2222
version: v3.0.6
23+
- src: https://github.com/stackhpc/ansible-role-os-manila-mount.git
24+
name: stackhpc.os-manila-mount
25+
version: v24.1.0
2326

2427
collections:
2528
- name: containers.podman

0 commit comments

Comments
 (0)