RFC 2228: Changes for BeeGFS re-deployment (#1)

* Umount meta data device only in case of reformat

Prior to this change, running the playbook a second time will not work
because the meta.yml playbook will try to unmount the meta data device
while the service is still running and also it is unnecessary.
Additionally add explicit filesystem and mount options for the metadata
devices.
Small bugfix for newer ansbile file module and allow the user allow for
meta data device tunings

* RFC-2228: Changes related to BeeGFS 2.7.5

- Replace beegfs-admon with beegfs-mon
- Add quota support
- small bugfixes related to RHEL 8.x

README.md

@@ -44,11 +44,11 @@ And a corresponding playbook as this (`beegfs.yml`):
       - cluster_beegfs_mgmt
       - cluster_beegfs_mds
       - cluster_beegfs_oss
-      - cluster_beegfs_client 
+      - cluster_beegfs_client
       roles:
       - role: stackhpc.beegfs
         beegfs_enable:
-          admon: false
+          mon: false
           mgmt: "{{ inventory_hostname in groups['cluster_beegfs_mgmt'] }}"
           meta: "{{ inventory_hostname in groups['cluster_beegfs_mds'] }}"
           oss: "{{ inventory_hostname in groups['cluster_beegfs_oss'] }}"
@@ -89,7 +89,7 @@ under `beegfs_enable` to `true` or `false` where:
 - `mds`: Metadata storage server nodes
 - `oss`: Object storage server nodes
 - `client`: Clients of the BeeGFS storage cluster
-- `admon`: NOT IMPLEMENTED
+- `mon`: Monitoring service
 
 This role is dependent upon each node's hostname resolving to the IP address
 used to reach the management host, as configured via `beegfs_host_mgmt`. In

defaults/main.yml

@@ -4,13 +4,16 @@ beegfs_enable:
   mgmt: false
   meta: false
   oss: false
-  admon: false
+  mon: false
   client: false
   tuning: false
 
 # Can be present or absent
 beegfs_state: present
 
+# with what to multiply the nodeid to get the target id in case a node is explictly set
+beegfs_target_id_multiplier: 100
+
 # Kernel modules to ensure are loaded prior to BeeGFS deploy/startup
 beegfs_kmod_preload: []
 
@@ -57,7 +60,7 @@ beegfs_rdma: true
 
 # Default filesystem options
 beegfs_filesystem_opts: "-K -d su=128k,sw=8 -l version=2,su=128k -isize=512"
-beegfs_mount_opts: "noatime,nodiratime,logbufs=8,logbsize=256k,largeio,inode64,swalloc,allocsize=131072k,nobarrier"
+beegfs_mount_opts: "uqnoenforce,gqnoenforce,noatime,nodiratime,logbufs=8,logbsize=256k,largeio,inode64,swalloc,allocsize=131072k"
 beegfs_force_format: false
 beegfs_fstype: "xfs"
 
@@ -81,6 +84,8 @@ beegfs_mgmt_host:  # Example: "{{ groups['cluster_beegfs_mgmt'] | first }}"
 beegfs_meta_path: "/data/beegfs/beegfs_meta"
 beegfs_meta_dev:  # /dev/sdb
 beegfs_meta_fstype:  # Example: "ext4"
+beegfs_meta_filesystem_opts: "-i 2048 -I 512 -J size=400 -O dir_index,filetype"
+beegfs_meta_mount_opts: "noatime,nodiratime,nobarrier,auto"
 
 # Client mount configs
 beegfs_client: []
@@ -96,4 +101,25 @@ beegfs_package_action: "{{ 'latest' if beegfs_update | bool else 'present' }}"
 
 # Path to the kernel module
 beegfs_kernel_module: "/lib/modules/{{ ansible_kernel }}/updates/fs/beegfs_autobuild/beegfs.ko"
+
+# tuneNumWorkers settings for meta server
+beegfs_meta_tuneNumWorkers: 0
+
+# tuneNumWorkers settings for meta server
+beegfs_meta_tuneTargetChooser: randomized
+
+beegfs_enable_quota: false
+beegfs_client_scope_config: true
+
+beegfs_client_ofed_include_path:
+
+beegfs_node_num_id:
+
+beegfs_mon_db_hostname:
+
+beegfs_mon_db_database:
+
+beegfs_mon_db_port:
+
+beegfs_oss_max_sectors_kb: 256
 ...

handlers/main.yml

@@ -24,13 +24,13 @@
   with_items: "{{ beegfs_oss }}"
   when: beegfs_enable.oss | bool
 
-- name: Restart BeeGFS admon service
+- name: Restart BeeGFS mon service
   systemd:
-    name: beegfs-admon
+    name: beegfs-mon
     daemon-reload: true
     state: restarted
   become: true
-  when: beegfs_enable.admon | bool
+  when: beegfs_enable.mon | bool
 
 - name: Restart BeeGFS client service
   systemd:

molecule/default/playbook.yml

@@ -4,7 +4,7 @@
   roles:
     - role: ansible-role-beegfs
       beegfs_enable:
-        admon: false
+        mon: false
         mgmt: "{{ inventory_hostname in groups['cluster_beegfs_mgmt'] }}"
         meta: "{{ inventory_hostname in groups['cluster_beegfs_mds'] }}"
         oss: "{{ inventory_hostname in groups['cluster_beegfs_oss'] }}"

molecule/vagrant-all/playbook.yml

@@ -4,7 +4,7 @@
   roles:
     - role: ansible-role-beegfs
       beegfs_enable:
-        admon: false
+        mon: false
         mgmt: "{{ inventory_hostname in groups['cluster_beegfs_mgmt'] }}"
         meta: "{{ inventory_hostname in groups['cluster_beegfs_mds'] }}"
         oss: "{{ inventory_hostname in groups['cluster_beegfs_oss'] }}"

molecule/vagrant-ubuntu-16.04/playbook.yml

@@ -5,7 +5,7 @@
     - role: ansible-role-beegfs
       beegfs_enable:
         mgmt: "{{ inventory_hostname in groups['cluster_beegfs_mgmt'] }}"
-        admon: false
+        mon: false
         meta: "{{ inventory_hostname in groups['cluster_beegfs_mds'] }}"
         oss: "{{ inventory_hostname in groups['cluster_beegfs_oss'] }}"
         tuning: "{{ inventory_hostname in groups['cluster_beegfs_oss'] }}"

molecule/vagrant-ubuntu-18.04/playbook.yml

@@ -4,7 +4,7 @@
   roles:
     - role: ansible-role-beegfs
       beegfs_enable:
-        admon: false
+        mon: false
         mgmt: "{{ inventory_hostname in groups['cluster_beegfs_mgmt'] }}"
         meta: "{{ inventory_hostname in groups['cluster_beegfs_mds'] }}"
         oss: "{{ inventory_hostname in groups['cluster_beegfs_oss'] }}"

tasks/client.yml

@@ -1,12 +1,12 @@
 ---
 - name: Install packages for BeeGFS client
   package:
-    name: "{{ item }}"
-    state: "{{ beegfs_package_action }}"
-  with_items:
+    name:
+    - elfutils-libelf-devel
     - beegfs-client
     - beegfs-helperd
     - beegfs-utils
+    state: "present"
   become: true
   notify: Restart BeeGFS client service
 
@@ -16,18 +16,18 @@
     regexp: "^buildArgs="
     line: "{{ item.line }}"
   when: item.condition
+  become: true
   with_items:
-    - { line: "buildArgs=-j8 BEEGFS_OPENTK_IBVERBS=1", condition: "{{ beegfs_rdma | bool }}" }
-    - { line: "buildArgs=-j8", condition: "{{ not beegfs_rdma | bool }}" }
+    - { line: "buildArgs=-j8 OFED_INCLUDE_PATH={{ beegfs_client_ofed_include_path }}", condition: "{{ beegfs_client_ofed_include_path }}" }
+    - { line: "buildArgs=-j8", condition: "{{ not beegfs_client_ofed_include_path }}" }
   notify: Restart BeeGFS client service
-  become: true
 
 - name: Ensure kernel development headers are present
   package:
-    name: "{{ beegfs_distro_vars[ansible_os_family]['kernel_dev_pkg'] }}-{{ ansible_kernel }}"
+    name: "{{ beegfs_distro_vars[ansible_os_family]['kernel_dev_pkg'] }}"
     state: present
-  notify: Restart BeeGFS client service
   become: true
+  notify: Restart BeeGFS client service
 
 - name: Ensure gcc is installed
   package:
@@ -36,15 +36,9 @@
   become: true
   notify: Restart BeeGFS client service
 
-- name: Rebuild the BeeGFS client kernel module
-  command: /etc/init.d/beegfs-client rebuild
-  args:
-    creates: "{{ beegfs_kernel_module }}"
-  become: true
-  notify: Restart BeeGFS client service
-
 - name: Ensure the BeeGFS mount point exists
   file:
+    mode: 0755
     path: "{{ client_path }}"
     state: directory
   become: true
@@ -60,47 +54,77 @@
 
 - name: Make of copy of BeeGFS client config file if it doesn't exist
   copy:
+    mode: 0644
     remote_src: true
     src: /etc/beegfs/beegfs-client.conf
-    dest: "/etc/beegfs/beegfs-client-{{ client_mgmt_host }}.conf"
+    dest: "/etc/beegfs/{{ beegfs_client_config_file }}"
     force: false
   become: true
+  when: beegfs_client_scope_config | bool
   notify: Restart BeeGFS client service
 
 - name: Configure beegfs-client config file to point to the specified management host
   lineinfile:
-    dest: "/etc/beegfs/beegfs-client-{{ client_mgmt_host }}.conf"
+    dest: "/etc/beegfs/{{ beegfs_client_config_file }}"
     regexp: "^sysMgmtdHost"
     line: "sysMgmtdHost = {{ client_mgmt_host }}"
   become: true
+  notify: Restart BeeGFS client service
 
 - name: Configure beegfs-client config file to use the specified port
   lineinfile:
-    dest: "/etc/beegfs/beegfs-client-{{ client_mgmt_host }}.conf"
+    dest: "/etc/beegfs/{{ beegfs_client_config_file }}"
     regexp: "^connClientPortUDP"
     line: "connClientPortUDP = {{ client_port }}"
   become: true
+  notify: Restart BeeGFS client service
 
 - name: Configure beegfs-client config file to ensure the specifed connection interface is used
   lineinfile:
-    dest: "/etc/beegfs/beegfs-client-{{ client_mgmt_host }}.conf"
+    dest: "/etc/beegfs/{{ beegfs_client_config_file }}"
     regexp: "^connInterfacesFile"
     line: "connInterfacesFile = /etc/beegfs/connInterfacesFile"
   become: true
+  notify: Restart BeeGFS client service
 
-- name: Copy default beegfs-client config file to enable beegfs_ctl in the management host
-  copy:
-    remote_src: true
-    dest: /etc/beegfs/beegfs-client.conf
-    src: "/etc/beegfs/beegfs-client-{{ client_mgmt_host }}.conf"
-    force: true
-  when: client_mgmt_host == inventory_hostname
-  become: true
 
 - name: Copy beegfs-client.service.d/ to /etc/systemd/system/
   copy:
     src: beegfs-client.service.d/
     dest: /etc/systemd/system/beegfs-client.service.d/
   notify: Restart BeeGFS client service
   become: true
-...
+
+
+- name: Configure beegfs-client config file to enable quota support
+  lineinfile:
+    dest: "/etc/beegfs/{{ beegfs_client_config_file }}"
+    regexp: "^quotaEnabled"
+    line: "quotaEnabled = {{ (beegfs_enable_quota | bool) | ternary('true', 'false') }}"
+  become: true
+  notify: Restart BeeGFS client service
+
+
+- name: Configure beegfs-client config file to enable/disable RDMA
+  lineinfile:
+    dest: "/etc/beegfs/{{ beegfs_client_config_file }}"
+    regexp: "^connUseRDMA"
+    line: "connUseRDMA = {{ (beegfs_rdma | bool) | ternary('true', 'false') }}"
+  become: true
+  notify: Restart BeeGFS client service
+
+- name: Rebuild the BeeGFS client kernel module
+  command: /etc/init.d/beegfs-client rebuild
+  args:
+    creates: "{{ beegfs_kernel_module }}"
+  become: true
+  notify: Restart BeeGFS client service
+
+- name: Copy default beegfs-client config file to enable beegfs_ctl in the management host
+  copy:
+    remote_src: true
+    dest: /etc/beegfs/beegfs-client.conf
+    src: "/etc/beegfs/beegfs-client-{{ client_mgmt_host }}.conf"
+    force: true
+  when: client_mgmt_host == inventory_hostname and not (beegfs_client_scope_config | bool)
+  become: true