From ff754423dc2e44cd5d2c54d8c0951e68b3d463ef Mon Sep 17 00:00:00 2001 From: Sam Schmit-Van Werweke Date: Sun, 2 Apr 2023 16:26:15 +0200 Subject: [PATCH] Transition to k3s installation and uninstall scripts (#108) --- .github/ISSUE_TEMPLATE.md | 1 - .gitignore | 1 + README.md | 1 + inventory/sample/group_vars/all.yml | 3 +- inventory/sample/hosts.ini | 1 + molecule/default/molecule.yml | 14 +++ roles/download/tasks/main.yml | 36 ------- roles/k3s/master/defaults/main.yml | 1 - ...h_k3s_init_logs.yml => fetch_k3s_logs.yml} | 12 +-- roles/k3s/master/tasks/main.yml | 96 +++---------------- roles/k3s/master/templates/k3s.service.j2 | 24 ----- roles/k3s/node/tasks/main.yml | 25 +++-- roles/k3s/node/templates/k3s.service.j2 | 24 ----- roles/reset/tasks/main.yml | 61 +++--------- roles/reset/tasks/umount_with_children.yml | 16 ---- site.yml | 2 - 16 files changed, 64 insertions(+), 254 deletions(-) delete mode 100644 roles/download/tasks/main.yml rename roles/k3s/master/tasks/{fetch_k3s_init_logs.yml => fetch_k3s_logs.yml} (60%) delete mode 100644 roles/k3s/master/templates/k3s.service.j2 delete mode 100644 roles/k3s/node/templates/k3s.service.j2 delete mode 100644 roles/reset/tasks/umount_with_children.yml diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 986c9cbb8..e17814846 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -33,7 +33,6 @@ Hardware: ```yml k3s_version: "" ansible_user: NA -systemd_dir: "" flannel_iface: "" diff --git a/.gitignore b/.gitignore index 78f3d0bbe..e0540ca20 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .env/ +.vagrant *.log ansible.cfg diff --git a/README.md b/README.md index ceff956d5..230f77bd5 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Build a Kubernetes cluster using Ansible with k3s. The goal is easily install a - [x] Debian (tested on version 11) - [x] Ubuntu (tested on version 22.04) - [x] Rocky (tested on version 9) +- [x] Alpine (tested on version 3.17) on processor architecture: diff --git a/inventory/sample/group_vars/all.yml b/inventory/sample/group_vars/all.yml index dbe104821..7ba76db3c 100644 --- a/inventory/sample/group_vars/all.yml +++ b/inventory/sample/group_vars/all.yml @@ -1,8 +1,7 @@ --- -k3s_version: v1.25.9+k3s1 +k3s_version: v1.25.11+k3s1 # this is the user that has ssh access to these machines ansible_user: ansibleuser -systemd_dir: /etc/systemd/system # Set your timezone system_timezone: "Your/Timezone" diff --git a/inventory/sample/hosts.ini b/inventory/sample/hosts.ini index 7045423d8..f40de20f1 100644 --- a/inventory/sample/hosts.ini +++ b/inventory/sample/hosts.ini @@ -6,6 +6,7 @@ [node] 192.168.30.41 192.168.30.42 +192.168.30.43 # only required if proxmox_lxc_configure: true # must contain all proxmox instances that have a master or worker node diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml index 4a07503aa..7cab2954c 100644 --- a/molecule/default/molecule.yml +++ b/molecule/default/molecule.yml @@ -3,6 +3,7 @@ dependency: name: galaxy driver: name: vagrant + provision: true platforms: - name: control1 @@ -70,6 +71,19 @@ platforms: - network_name: private_network ip: 192.168.30.42 + - name: node3 + box: generic/alpine317 + instance_raw_config_args: + - 'vm.provision :shell, inline: "apk add --update --no-cache python3"' + memory: 2048 + cpus: 2 + groups: + - k3s_cluster + - node + interfaces: + - network_name: private_network + ip: 192.168.30.43 + provisioner: name: ansible playbooks: diff --git a/roles/download/tasks/main.yml b/roles/download/tasks/main.yml deleted file mode 100644 index 1450fd86e..000000000 --- a/roles/download/tasks/main.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- - -- name: Download k3s binary x64 - get_url: - url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s - checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-amd64.txt - dest: /usr/local/bin/k3s - owner: root - group: root - mode: 0755 - when: ansible_facts.architecture == "x86_64" - -- name: Download k3s binary arm64 - get_url: - url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s-arm64 - checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-arm64.txt - dest: /usr/local/bin/k3s - owner: root - group: root - mode: 0755 - when: - - ( ansible_facts.architecture is search("arm") and - ansible_facts.userspace_bits == "64" ) or - ansible_facts.architecture is search("aarch64") - -- name: Download k3s binary armhf - get_url: - url: https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/k3s-armhf - checksum: sha256:https://github.com/k3s-io/k3s/releases/download/{{ k3s_version }}/sha256sum-arm.txt - dest: /usr/local/bin/k3s - owner: root - group: root - mode: 0755 - when: - - ansible_facts.architecture is search("arm") - - ansible_facts.userspace_bits == "32" diff --git a/roles/k3s/master/defaults/main.yml b/roles/k3s/master/defaults/main.yml index 9e2fe633e..8cc13abf1 100644 --- a/roles/k3s/master/defaults/main.yml +++ b/roles/k3s/master/defaults/main.yml @@ -11,6 +11,5 @@ server_init_args: >- {% else %} --server https://{{ hostvars[groups['master'][0]].k3s_node_ip | split(",") | first | ansible.utils.ipwrap }}:6443 {% endif %} - --token {{ k3s_token }} {% endif %} {{ extra_server_args | default('') }} diff --git a/roles/k3s/master/tasks/fetch_k3s_init_logs.yml b/roles/k3s/master/tasks/fetch_k3s_logs.yml similarity index 60% rename from roles/k3s/master/tasks/fetch_k3s_init_logs.yml rename to roles/k3s/master/tasks/fetch_k3s_logs.yml index 060f26a89..b8476cad4 100644 --- a/roles/k3s/master/tasks/fetch_k3s_init_logs.yml +++ b/roles/k3s/master/tasks/fetch_k3s_logs.yml @@ -1,12 +1,12 @@ --- -# Download logs of k3s-init.service from the nodes to localhost. +# Download logs of k3s.service from the nodes to localhost. # Note that log_destination must be set. -- name: Fetch k3s-init.service logs +- name: Fetch k3s.service logs ansible.builtin.command: - cmd: journalctl --all --unit=k3s-init.service + cmd: journalctl --all --unit=k3s.service changed_when: false - register: k3s_init_log + register: k3s_log - name: Create {{ log_destination }} delegate_to: localhost @@ -22,7 +22,7 @@ become: false ansible.builtin.template: src: content.j2 - dest: "{{ log_destination }}/k3s-init@{{ ansible_hostname }}.log" + dest: "{{ log_destination }}/k3s@{{ ansible_hostname }}.log" mode: 0644 vars: - content: "{{ k3s_init_log.stdout }}" + content: "{{ k3s_log.stdout }}" diff --git a/roles/k3s/master/tasks/main.yml b/roles/k3s/master/tasks/main.yml index 76b910c39..c9561b8d4 100644 --- a/roles/k3s/master/tasks/main.yml +++ b/roles/k3s/master/tasks/main.yml @@ -1,17 +1,18 @@ --- -- name: Clean previous runs of k3s-init - systemd: - name: k3s-init - state: stopped - failed_when: false +- name: Download install script + ansible.builtin.get_url: + url: https://get.k3s.io + dest: "{{ ansible_user_dir }}/install_k3s.sh" + mode: 744 -- name: Clean previous runs of k3s-init - command: systemctl reset-failed k3s-init +- name: Install k3s + command: + cmd: "./install_k3s.sh {{ server_init_args }}" + environment: + INSTALL_K3S_VERSION: "{{ k3s_version }}" + K3S_TOKEN: "{{ k3s_token }}" failed_when: false - changed_when: false - args: - warn: false # The ansible systemd module does not support reset-failed - name: Deploy vip manifest include_tasks: vip.yml @@ -19,17 +20,9 @@ - name: Deploy metallb manifest include_tasks: metallb.yml -- name: Init cluster inside the transient k3s-init service - command: - cmd: "systemd-run -p RestartSec=2 \ - -p Restart=on-failure \ - --unit=k3s-init \ - k3s server {{ server_init_args }}" - creates: "{{ systemd_dir }}/k3s.service" - - name: Verification block: - - name: Verify that all nodes actually joined (check k3s-init.service if this fails) + - name: Verify that all nodes actually joined (check k3s.service if this fails) command: cmd: k3s kubectl get nodes -l "node-role.kubernetes.io/master=true" -o=jsonpath="{.items[*].metadata.name}" register: nodes @@ -38,63 +31,14 @@ delay: 10 changed_when: false always: - - name: Save logs of k3s-init.service - include_tasks: fetch_k3s_init_logs.yml + - name: Save logs of k3s.service + include_tasks: fetch_k3s_logs.yml when: log_destination vars: log_destination: >- {{ lookup('ansible.builtin.env', 'ANSIBLE_K3S_LOG_DIR', default=False) }} - - name: Kill the temporary service used for initialization - systemd: - name: k3s-init - state: stopped - failed_when: false when: not ansible_check_mode -- name: Copy K3s service file - register: k3s_service - template: - src: "k3s.service.j2" - dest: "{{ systemd_dir }}/k3s.service" - owner: root - group: root - mode: 0644 - -- name: Enable and check K3s service - systemd: - name: k3s - daemon_reload: yes - state: restarted - enabled: yes - -- name: Wait for node-token - wait_for: - path: /var/lib/rancher/k3s/server/node-token - -- name: Register node-token file access mode - stat: - path: /var/lib/rancher/k3s/server - register: p - -- name: Change file access node-token - file: - path: /var/lib/rancher/k3s/server - mode: "g+rx,o+rx" - -- name: Read node-token from master - slurp: - src: /var/lib/rancher/k3s/server/node-token - register: node_token - -- name: Store Master node-token - set_fact: - token: "{{ node_token.content | b64decode | regex_replace('\n', '') }}" - -- name: Restore node-token file access - file: - path: /var/lib/rancher/k3s/server - mode: "{{ p.stat.mode }}" - - name: Create directory .kube file: path: "{{ ansible_user_dir }}/.kube" @@ -124,18 +68,6 @@ # would be undefined. This will not be the case during playbook execution. # noqa jinja[invalid] -- name: Create kubectl symlink - file: - src: /usr/local/bin/k3s - dest: /usr/local/bin/kubectl - state: link - -- name: Create crictl symlink - file: - src: /usr/local/bin/k3s - dest: /usr/local/bin/crictl - state: link - - name: Get contents of manifests folder find: paths: /var/lib/rancher/k3s/server/manifests diff --git a/roles/k3s/master/templates/k3s.service.j2 b/roles/k3s/master/templates/k3s.service.j2 deleted file mode 100644 index ae5cb484e..000000000 --- a/roles/k3s/master/templates/k3s.service.j2 +++ /dev/null @@ -1,24 +0,0 @@ -[Unit] -Description=Lightweight Kubernetes -Documentation=https://k3s.io -After=network-online.target - -[Service] -Type=notify -ExecStartPre=-/sbin/modprobe br_netfilter -ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/local/bin/k3s server {{ extra_server_args | default("") }} -KillMode=process -Delegate=yes -# Having non-zero Limit*s causes performance problems due to accounting overhead -# in the kernel. We recommend using cgroups to do container-local accounting. -LimitNOFILE=1048576 -LimitNPROC=infinity -LimitCORE=infinity -TasksMax=infinity -TimeoutStartSec=0 -Restart=always -RestartSec=5s - -[Install] -WantedBy=multi-user.target diff --git a/roles/k3s/node/tasks/main.yml b/roles/k3s/node/tasks/main.yml index 0ce8e08d0..6d14e4e7a 100644 --- a/roles/k3s/node/tasks/main.yml +++ b/roles/k3s/node/tasks/main.yml @@ -1,16 +1,15 @@ --- -- name: Copy K3s service file - template: - src: "k3s.service.j2" - dest: "{{ systemd_dir }}/k3s-node.service" - owner: root - group: root - mode: 0755 +- name: Download install script + ansible.builtin.get_url: + url: https://get.k3s.io + dest: "{{ ansible_user_dir }}/install_k3s.sh" + mode: 744 -- name: Enable and check K3s service - systemd: - name: k3s-node - daemon_reload: yes - state: restarted - enabled: yes +- name: Install k3s + command: + cmd: "./install_k3s.sh {{ extra_agent_args | default('') }}" + environment: + INSTALL_K3S_VERSION: "{{ k3s_version }}" + K3S_TOKEN: "{{ k3s_token }}" + K3S_URL: https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443 diff --git a/roles/k3s/node/templates/k3s.service.j2 b/roles/k3s/node/templates/k3s.service.j2 deleted file mode 100644 index 67abadbb5..000000000 --- a/roles/k3s/node/templates/k3s.service.j2 +++ /dev/null @@ -1,24 +0,0 @@ -[Unit] -Description=Lightweight Kubernetes -Documentation=https://k3s.io -After=network-online.target - -[Service] -Type=notify -ExecStartPre=-/sbin/modprobe br_netfilter -ExecStartPre=-/sbin/modprobe overlay -ExecStart=/usr/local/bin/k3s agent --server https://{{ apiserver_endpoint | ansible.utils.ipwrap }}:6443 --token {{ hostvars[groups['master'][0]]['token'] | default(k3s_token) }} {{ extra_agent_args | default("") }} -KillMode=process -Delegate=yes -# Having non-zero Limit*s causes performance problems due to accounting overhead -# in the kernel. We recommend using cgroups to do container-local accounting. -LimitNOFILE=1048576 -LimitNPROC=infinity -LimitCORE=infinity -TasksMax=infinity -TimeoutStartSec=0 -Restart=always -RestartSec=5s - -[Install] -WantedBy=multi-user.target diff --git a/roles/reset/tasks/main.yml b/roles/reset/tasks/main.yml index cd3bf720c..8c378b005 100644 --- a/roles/reset/tasks/main.yml +++ b/roles/reset/tasks/main.yml @@ -1,54 +1,21 @@ --- -- name: Disable services - systemd: - name: "{{ item }}" - state: stopped - enabled: no - failed_when: false - with_items: - - k3s - - k3s-node - - k3s-init - -- name: RUN pkill -9 -f "k3s/data/[^/]+/bin/containerd-shim-runc" - register: pkill_containerd_shim_runc - command: pkill -9 -f "k3s/data/[^/]+/bin/containerd-shim-runc" - changed_when: "pkill_containerd_shim_runc.rc == 0" - failed_when: false +- name: Check if uninstall script exists + stat: + path: /usr/local/bin/k3s-uninstall.sh + register: uninstalscript -- name: Umount k3s filesystems - include_tasks: umount_with_children.yml - with_items: - - /run/k3s - - /var/lib/kubelet - - /run/netns - - /var/lib/rancher/k3s - - /var/lib/kubelet/pods - - /var/lib/kubelet/plugins - - /run/netns/cni- - loop_control: - loop_var: mounted_fs +- name: Uninstall k3s (server) + command: /usr/local/bin/k3s-uninstall.sh + when: uninstalscript.stat.exists -- name: Remove service files, binaries and data - file: - name: "{{ item }}" - state: absent - with_items: - - /usr/local/bin/k3s - - "{{ systemd_dir }}/k3s.service" - - "{{ systemd_dir }}/k3s-node.service" - - /etc/rancher/k3s - - /run/k3s - - /run/flannel - - /etc/rancher/ - - /var/lib/kubelet - - /var/lib/rancher/k3s - - /var/lib/rancher/ - - /var/lib/cni/ +- name: Check if agent uninstall script exists + stat: + path: /usr/local/bin/k3s-agent-uninstall.sh + register: agentuninstalscript -- name: Reload daemon_reload - systemd: - daemon_reload: yes +- name: Uninstall k3s (agent) + command: /usr/local/bin/k3s-agent-uninstall.sh + when: agentuninstalscript.stat.exists - name: Remove tmp directory used for manifests file: diff --git a/roles/reset/tasks/umount_with_children.yml b/roles/reset/tasks/umount_with_children.yml deleted file mode 100644 index 5883b70a6..000000000 --- a/roles/reset/tasks/umount_with_children.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -- name: Get the list of mounted filesystems - shell: set -o pipefail && cat /proc/mounts | awk '{ print $2}' | grep -E "^{{ mounted_fs }}" - register: get_mounted_filesystems - args: - executable: /bin/bash - failed_when: false - changed_when: get_mounted_filesystems.stdout | length > 0 - check_mode: false - -- name: Umount filesystem - mount: - path: "{{ item }}" - state: unmounted - with_items: - "{{ get_mounted_filesystems.stdout_lines | reverse | list }}" diff --git a/site.yml b/site.yml index 5104cce28..82f9485a9 100644 --- a/site.yml +++ b/site.yml @@ -15,8 +15,6 @@ when: proxmox_lxc_configure - role: prereq become: true - - role: download - become: true - role: raspberrypi become: true