Merge pull request #6 from Mikroways/feature/better-prometheus-exporter

Feature/better prometheus exporter

.envrc

@@ -1,3 +1,4 @@
-layout pyenv 3.9.9
+use asdf
+layout python
 export PY_COLORS=1
 export ANSIBLE_FORCE_COLOR=1

.github/workflows/test.yml

@@ -1,17 +1,6 @@
 ---
 name: Tests
 on:
-  push:
-    paths:
-      - defaults/**
-      - handlers/**
-      - meta/**
-      - molecule/**
-      - tasks/**
-      - templates/**
-      - vars/**
-      - .github/**
-      - tox.ini
   pull_request:
     branches:
       - main

README.md

@@ -6,6 +6,8 @@ I've also found an inspiration from other existing roles as: [arsillio.restic](h
 but evolving from cron to systemd was my objective, so I've decided to write
 this new role as a composition of ideas.
 
+> This documentation needs to be updated with changes introduced since 2.0.x
+
 ## Description
 
 This role installs [Restic](https://github.com/restic/restic) and configures as

defaults/main.yml

@@ -1,34 +1,51 @@
 ---
 # defaults file for skeleton
 restic_url: '{{ restic_url_default }}'
-restic_version: '0.12.0'
+restic_version: '0.16.0'
 restic_download_path: '/opt/restic'
 restic_install_path: '/usr/bin'
 restic_cache_dir: /var/cache/restic
+restic_verify_pgp_url: https://restic.net/gpg-key-alex.asc
+restic_verify_dir: '{{ restic_cache_dir }}/.verify'
 restic_script_dir: '{{ restic_download_path }}/scripts'
 restic_repos: {}
 restic_backups: []
+restic_prune_repositories: []
+restic_prune_enabled: false
 restic_create_systemd_timer: false
 restic_skip_handlers: false
 restic_skip_checksum_verify: true
 
-restic_dir_owner: '{{ ansible_user | default(ansible_user_id) }}'
-restic_dir_group: '{{ ansible_user | default(ansible_user_id) }}'
+restic_dir_owner: 'root'
+restic_dir_group: 'root'
 
 restic_access_template: restic_access.j2
 restic_backup_template: restic_backup.j2
-restic_prune_template: restic_prune.j2
+restic_forget_template: restic_forget.j2
 restic_stats_template: restic_stats.j2
 restic_systemd_timer_template: systemd_timer.unit.j2
+restic_systemd_forget_service_template: systemd_forget.unit.j2
+restic_systemd_forget_timer_template: systemd_forget.timer.j2
+restic_systemd_prune_service_template: systemd_forget.unit.j2
+restic_systemd_prune_timer_template: systemd_forget.timer.j2
 restic_systemd_service_template: systemd_service.unit.j2
 restic_systemd_failure_service_template: systemd_failure.unit.j2
 restic_systemd_failure_service_name: '[email protected]'
 
-restic_prometehus_exporter_enabled: false
-restic_prometehus_exporter_template: restic_prometehus_exporter.j2
-restic_prometehus_exporter_script: '{{ restic_script_dir }}/restic_prometehus_exporter.sh'
-restic_prometehus_exporter_metrics_basedir: /var/lib/node-exporter
-
+restic_prometheus_exporter_enabled: false
+restic_prometheus_exporter_template: restic_prometheus_exporter.j2
+restic_prometheus_exporter_script: '{{ restic_script_dir }}/restic-prometheus-exporter'
+restic_prometheus_exporter_metrics_basedir: /var/lib/node_exporter
+restic_prometheus_exporter_repositories: []
+restic_systemd_prometheus_service_template: restic-prometheus-service.unit.j2
+restic_systemd_prometheus_service_prefix_name: 'restic-prometheus_exporter@'
+restic_systemd_prometheus_service_name: '{{ restic_systemd_prometheus_service_prefix_name}}.service'
+restic_systemd_prometheus_timer_template: restic-prometheus-timer.unit.j2
+restic_prune_template: restic-prune.j2
+restic_systemd_prune_service_template: restic-prune-service.unit.j2
+restic_systemd_prune_service_prefix_name: 'restic-prune@'
+restic_systemd_prune_service_name: '{{ restic_systemd_prune_service_prefix_name}}.service'
+restic_systemd_prune_timer_template: restic-prune-timer.unit.j2
 restic_failure_template: restic_failure_unit.j2
 restic_failure_script: '{{ restic_script_dir }}/unit-failure'
 restic_failure_mail_to: "root"

molecule/default/molecule.yml

@@ -8,6 +8,9 @@ platforms:
     image: 'geerlingguy/docker-ubuntu2004-ansible:latest'
     privileged: true
     pre_build_image: true
+  - name: ubuntu-2204
+    image: 'geerlingguy/docker-ubuntu2204-ansible:latest'
+    privileged: true
   - name: debian-11
     image: 'geerlingguy/docker-debian11-ansible:latest'
     privileged: true
@@ -18,6 +21,10 @@ lint: |
   ansible-lint .
 provisioner:
   name: ansible
+  inventory:
+    host_vars:
+      ubuntu-2004:
+        restic_skip_checksum_verify: false
   playbooks:
     converge: playbook.yml
 scenario:

molecule/default/playbook.yml

@@ -12,7 +12,7 @@
     restic_skip_handlers: true
     restic_download_path: ~/restic
     restic_install_path: ~/restic
-    restic_prometehus_exporter_enabled: True
+    restic_prometheus_exporter_enabled: True
     restic_create_systemd_timer: false
     restic_repos:
       local:
@@ -61,7 +61,7 @@
         - acl  # needed to run 1 task with become_user: "{{ restic_non_root_setup_user }}"
   vars:
     restic_skip_handlers: true
-    restic_prometehus_exporter_enabled: false
+    restic_prometheus_exporter_enabled: false
     restic_create_systemd_timer: false
     restic_download_path: /home/restic/workDir
     restic_install_path: /home/restic/workDir

requirements.txt

@@ -1,7 +1,8 @@
 jmespath
+urllib3<2
 tox==4.0.0b2
 ansible==5.*
-molecule[docker]
+molecule-plugins[docker]
 docker == 5.*
 ansible-lint == 5.*
 yamllint == 1.26.*

tasks/install.yml

@@ -3,7 +3,7 @@
   get_url:
     url: '{{ restic_url }}'
     dest: '{{ restic_download_path }}/restic.bz2'
-    checksum: "sha256:{{ restic_url_checksums }}"
+    checksum: "sha256:{{ restic_url_checksums }}/SHA256SUMS"
     mode: "0644"
   register: get_url_restic
   when: restic_skip_checksum_verify
@@ -17,12 +17,18 @@
   register: get_url_restic
   when: not restic_skip_checksum_verify
 
+- name: Check download
+  stat:
+    path: '{{ restic_download_path }}/restic.bz2'
+  register: file_status
+
 - name: Install restic
   shell: |
-    bzip2 -dc {{ get_url_restic.dest }} > {{ restic_bin_bath }} &&
-    rm -f {{ get_url_restic.dest }}
+    bzip2 -dc {{ restic_download_path }}/restic.bz2 > {{ restic_bin_bath }} &&
+    rm -f {{ restic_download_path }}/restic.bz2
   register: restic_install_cmd
   changed_when: restic_install_cmd.rc == 0
+  when: file_status.stat.exists
 
 - name: Ensure permissions are correct
   file:

tasks/main.yml

@@ -48,13 +48,27 @@
 
 - name: Verify SHA256SUMS
   import_tasks: 'verify.yml'
-  when: not restic_executable.stat.exists or not restic_installed.stat.exists
-    or restic_executable.stat.size == 0 and not restic_skip_checksum_verify
+  when: ( not restic_executable.stat.exists or not restic_installed.stat.exists
+    or restic_executable.stat.size == 0 ) and not restic_skip_checksum_verify
 
-- name: Install restic
+- name: Install restic and dependencies
   import_tasks: 'install.yml'
   when: not restic_executable.stat.exists or not restic_installed.stat.exists
     or restic_executable.stat.size == 0
 
 - name: Configure restic scripts and scheduler
   import_tasks: 'restic.yml'
+
+- name: Configure restic exporter
+  import_tasks: 'prune.yml'
+  when:
+    - restic_prune_enabled
+    - restic_prune_repositories is defined
+    - restic_prune_repositories | length > 0
+
+- name: Configure restic exporter
+  import_tasks: 'prometheus.yml'
+  when:
+    - restic_prometheus_exporter_enabled
+    - restic_prometheus_exporter_repositories is defined
+    - restic_prometheus_exporter_repositories | length > 0

tasks/prometheus.yml

@@ -0,0 +1,85 @@
+---
+- name: Install dependencies
+  block:
+      - name: update apt cache
+        ansible.builtin.apt:
+          update_cache: yes
+        when: ansible_pkg_mgr == "apt"
+      - name: update yum cache
+        ansible.builtin.yum:
+          update_cache: yes
+        when: ansible_pkg_mgr == "yum"
+      - name: update dnf cache
+        ansible.builtin.dnf:
+          update_cache: yes
+        when: ansible_pkg_mgr == "dnf"
+      - name: install jq
+        ansible.builtin.package:
+          name: jq
+
+- name: Ensure prometehus metrics file can be created
+  file:
+    state: 'directory'
+    path: '{{ restic_prometheus_exporter_metrics_basedir }}'
+    mode: '0755'
+    owner: 'root'
+    group: 'root'
+
+- name: Configure prometehus exporter script
+  template:
+    src: '{{ restic_prometheus_exporter_template }}'
+    dest: '{{ restic_prometheus_exporter_script }}'
+    mode: '0700'
+    owner: '{{ _restic_user }}'
+    group: '{{ _restic_group }}'
+
+- name: Create dummy_list of restic_credentials
+  set_fact:
+    exporters_repos: >
+      {{ exporters_repos | default([]) +
+        [ item | combine({"repo": item.name, "src": '/'}) ] }}
+  with_items: '{{ restic_prometheus_exporter_repositories }}'
+
+- name: Create restic access credentials
+  template:
+    src: '{{ restic_access_template }}'
+    dest: '{{ restic_script_dir }}/access-prometheus-{{ item.name }}.sh'
+    mode: '0400'
+    owner: '{{ _restic_user }}'
+    group: '{{ _restic_group }}'
+      #  no_log: true
+  with_items: '{{ exporters_repos }}'
+
+- name: "Add systemd service restic-prometheus"
+  template:
+    src: "{{ restic_systemd_prometheus_service_template }}"
+    dest: "/etc/systemd/system/{{ restic_systemd_prometheus_service_name }}"
+    mode: "0644"
+    owner: root
+    group: root
+  notify:
+    - Reload systemd
+
+- name: "Add systemd timer restic-prometheus {{ item.name }}"
+  template:
+    src: "{{ restic_systemd_prometheus_timer_template }}"
+    dest: "/etc/systemd/system/{{ restic_systemd_prometheus_service_prefix_name }}{{ item.name}}.timer"
+    mode: "0644"
+    owner: root
+    group: root
+  notify:
+    - Reload systemd
+  when:
+    - item.schedules is defined
+    - item.schedules | length > 0
+  with_items: "{{ restic_prometheus_exporter_repositories }}"
+
+- name: "Enable systemd restic exporter for {{ item.name }} timer"
+  ansible.builtin.systemd:
+    name: "{{ restic_systemd_prometheus_service_prefix_name }}{{ item.name }}.timer"
+    enabled: true
+    state: "restarted"
+  when:
+    - item.schedules is defined
+    - item.schedules | length > 0
+  with_items: "{{ restic_prometheus_exporter_repositories }}"

tasks/prune.yml

@@ -0,0 +1,63 @@
+---
+- name: Create dummy_list of restic_credentials for restic_prune_repositories
+  set_fact:
+    prune_repos: >
+      {{ prune_repos | default([]) +
+        [ item | combine({"repo": item.name, "src": '/'}) ] }}
+  with_items: '{{ restic_prune_repositories }}'
+
+- name: Create restic access credentials
+  template:
+    src: '{{ restic_access_template }}'
+    dest: '{{ restic_script_dir }}/access-prune-{{ item.name }}.sh'
+    mode: '0400'
+    owner: '{{ _restic_user }}'
+    group: '{{ _restic_group }}'
+  no_log: true
+  with_items: '{{ prune_repos }}'
+
+- name: Configure restic prune script
+  template:
+    src: '{{ restic_prune_template }}'
+    dest: '{{ restic_script_dir }}/backup-prune-{{ item.name }}.sh'
+    mode: '0700'
+    owner: '{{ _restic_user }}'
+    group: '{{ _restic_group }}'
+  with_items: '{{ prune_repos }}'
+  when:
+    - item.name is defined
+    - item.repo in restic_repos
+
+- name: "Add systemd service restic-prune"
+  template:
+    src: "{{ restic_systemd_prune_service_template }}"
+    dest: "/etc/systemd/system/{{ restic_systemd_prune_service_name }}"
+    mode: "0644"
+    owner: root
+    group: root
+  notify:
+    - Reload systemd
+
+- name: "Add systemd timer restic-prune {{ item.name }}"
+  template:
+    src: "{{ restic_systemd_prune_timer_template }}"
+    dest: "/etc/systemd/system/{{ restic_systemd_prune_service_prefix_name }}{{ item.name}}.timer"
+    mode: "0644"
+    owner: root
+    group: root
+  notify:
+    - Reload systemd
+  when:
+    - item.schedules is defined
+    - item.schedules | length > 0
+  with_items: "{{ restic_prune_repositories }}"
+
+- name: "Enable systemd restic-prune for {{ item.name }} timer"
+  ansible.builtin.systemd:
+    name: "{{ restic_systemd_prune_service_prefix_name }}{{ item.name }}.timer"
+    enabled: true
+    state: "restarted"
+  when:
+    - item.schedules is defined
+    - item.schedules | length > 0
+  with_items: "{{ restic_prune_repositories }}"