diff --git a/salt/sys-cacher/README.md b/salt/sys-cacher/README.md index 7c3bfccc..8da2f298 100644 --- a/salt/sys-cacher/README.md +++ b/salt/sys-cacher/README.md @@ -31,6 +31,10 @@ This change will be done automatically for every template that exists and is not Whonix based. No changes are made to Whonix templates, and updates to those templates will not be cached. +The caching proxy supports Debian derivatives (not Whonix) and Arch Linux. +Fedora support was dropped due to unreliability of the mirror mechanism of +zchunk checksums when caching packages. + ## Installation Installation may take a long time as it will target all templates unless you @@ -174,6 +178,13 @@ sudo qubesctl --skip-dom0 --targets=QUBE state.apply sys-cacher.uninstall-client qvm-tags del QUBE updatevm-sys-cacher ``` +If you tagged manually a qube that is unsupported, updates for that qube will +fail. Get a full list of unsupported qubes (**warning**: there may be false +positives of supported qubes being listed): +```sh +sudo qubesctl --show-output state.apply sys-cacher.list-extra-tag +``` + ## Credits - [Unman](https://github.com/unman/shaker/tree/main/cacher) diff --git a/salt/sys-cacher/files/admin/list-extra-tag.sh b/salt/sys-cacher/files/admin/list-extra-tag.sh new file mode 100755 index 00000000..f5c2e4ad --- /dev/null +++ b/salt/sys-cacher/files/admin/list-extra-tag.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +# SPDX-FileCopyrightText: 2023 - 2024 Benjamin Grande M. S. +# +# SPDX-License-Identifier: AGPL-3.0-or-later + +set -eu + +get_os_distro(){ + distro_qube="${1}" + os_distro="$(qvm-features "${distro_qube}" os-distribution || true)" +} + +tagged="$(qvm-ls --no-spinner --raw-list --tags updatevm-sys-cacher | tr "\n" " ")" + +wanted="" +for qube in ${tagged}; do + get_os_distro "${qube}" + case "${os_distro}" in + debian|ubuntu|linuxmint|kali|kicksecure|arch) + continue + ;; + "") + ## AppVMs and DispVMs do not report the features, discover from + ## their templates. + klass="$(qvm-prefs "${qube}" klass)" + case "${klass}" in + TemplateVM|StandaloneVM) + ## WARN: creates false positives in case qube never did an update to + ## report the OS ID, thus reporting both supported qubes that are + ## not updated yet and unsupported that didn't update yet also. + wanted="${wanted:+"${wanted} "}${qube}" + ;; + AppVM|DispVM) + case "${klass}" in + AppVM) + template="$(qvm-prefs "${qube}" template)" + ;; + DispVM) + app="$(qvm-prefs "${qube}" template)" + template="$(qvm-prefs "${app}" template)" + ;; + esac + get_os_distro "${template}" + case "${os_distro}" in + debian|ubuntu|linuxmint|kali|kicksecure|arch) + continue + ;; + ## Qube is not supported. + *) wanted="${wanted:+"${wanted} "}${qube}";; + esac + ;; + esac + ;; + ## Qube is not supported. + *) wanted="${wanted:+"${wanted} "}${qube}";; + esac +done + +echo "${wanted}" | tr " " "\n" diff --git a/salt/sys-cacher/files/admin/tag.sh b/salt/sys-cacher/files/admin/tag.sh index 13e7dafd..49756b07 100755 --- a/salt/sys-cacher/files/admin/tag.sh +++ b/salt/sys-cacher/files/admin/tag.sh @@ -10,7 +10,19 @@ exclude="$(qvm-ls --no-spinner --raw-list --tags whonix-updatevm \ | sed "s/^./--exclude &/" | tr "\n" " ")" # shellcheck disable=SC2086 -wanted="$(qvm-ls --no-spinner --raw-data --fields=NAME,CLASS --all ${exclude} \ - | awk -v class="TemplateVM" -F "|" '$2 ~ class {print $1}')" +templates="$(qvm-ls --no-spinner --raw-data --fields=NAME,CLASS --all ${exclude} \ + | awk -v class="TemplateVM" -F "|" '$2 ~ class {print $1}' \ + | tr "\n" " ")" -echo "${wanted}" +wanted="" +for qube in ${templates}; do + os_distro="$(qvm-features "${qube}" os-distribution || true)" + case "${os_distro}" in + debian|ubuntu|linuxmint|kali|arch) + wanted="${wanted:+"${wanted} "}${qube}" + ;; + *) continue + esac +done + +echo "${wanted}" | tr " " "\n" diff --git a/salt/sys-cacher/files/client/bin/apt-cacher-ng-repo b/salt/sys-cacher/files/client/bin/apt-cacher-ng-repo index 612bdb7e..7749a9e8 100755 --- a/salt/sys-cacher/files/client/bin/apt-cacher-ng-repo +++ b/salt/sys-cacher/files/client/bin/apt-cacher-ng-repo @@ -85,15 +85,23 @@ check_netvm_cacher(){ proxy_conf="proxy=${proxy_addr}" } +reject_os(){ + echo "${0##*/} does not support your Operating System distribution." >&2 + exit 1 +} + +# shellcheck disable=SC2317 set_proxy_os(){ if test -e /etc/fedora-release; then ## Fedora + ## Uninstall because it leads to many zchunk checksum mismatch problems. + action="uninstall" + echo "${0##*/} doesn't work well on Fedora, uninstalling." >&2 if test -w /etc/dnf/dnf.conf; then set_proxy_marker /etc/dnf/dnf.conf "zchunk=False ${proxy_conf}" fi - if test -n "${proxy_addr}"; then cat >/etc/yum.conf.d/qubes-proxy.conf </etc/apt/apt.conf.d/50cacher-proxy <&2 - exit 1 + ## Gentoo: upstream does not have a good solution: + ## https://wiki.gentoo.org/wiki/Local_distfiles_cache#Configuring_for_Gentoo + reject_os fi } diff --git a/salt/sys-cacher/install-client.top b/salt/sys-cacher/install-client.top index 7d76814d..4a50cd05 100644 --- a/salt/sys-cacher/install-client.top +++ b/salt/sys-cacher/install-client.top @@ -1,10 +1,10 @@ {# -SPDX-FileCopyrightText: 2023 Benjamin Grande M. S. +SPDX-FileCopyrightText: 2023 - 2024 Benjamin Grande M. S. SPDX-License-Identifier: AGPL-3.0-or-later #} base: - 'I@qubes:type:template and not P@nodename:host and not P@nodename:whonix.*': + 'I@qubes:type:template and ( ( G@os_family:Debian and not P@nodename:host and not P@nodename:whonix.* ) or G@os_family:Arch )': - match: compound - sys-cacher.install-client diff --git a/salt/sys-cacher/list-extra-tag.sls b/salt/sys-cacher/list-extra-tag.sls new file mode 100644 index 00000000..eb7fbb1d --- /dev/null +++ b/salt/sys-cacher/list-extra-tag.sls @@ -0,0 +1,10 @@ +{# +SPDX-FileCopyrightText: 2023 - 2024 Benjamin Grande M. S. + +SPDX-License-Identifier: AGPL-3.0-or-later +#} + +{% set extraneous = salt['cmd.script']('salt://' ~ slsdotpath ~ '/files/admin/list-extra-tag.sh') -%} +"{{ slsdotpath }}-list-extra-tag": + cmd.run: + - name: echo {{ extraneous.stdout.split("\n") }} diff --git a/salt/sys-cacher/list-extra-tag.top b/salt/sys-cacher/list-extra-tag.top new file mode 100644 index 00000000..bd99589a --- /dev/null +++ b/salt/sys-cacher/list-extra-tag.top @@ -0,0 +1,10 @@ +{# +SPDX-FileCopyrightText: 2023 - 2024 Benjamin Grande M. S. + +SPDX-License-Identifier: AGPL-3.0-or-later +#} + +base: + 'dom0': + - match: nodegroup + - sys-cacher.list-extra-tag diff --git a/salt/sys-cacher/tag.sls b/salt/sys-cacher/tag.sls index ee7b6f97..32696c22 100644 --- a/salt/sys-cacher/tag.sls +++ b/salt/sys-cacher/tag.sls @@ -6,7 +6,7 @@ SPDX-License-Identifier: AGPL-3.0-or-later {% set templates = salt['cmd.script']('salt://' ~ slsdotpath ~ '/files/admin/tag.sh') -%} {% for tpl in templates.stdout.split("\n") -%} -"{{ slsdotpath }}-tag-for-{{ tpl }}": +"{{ slsdotpath }}-add-tag-of-{{ tpl }}": qvm.tags: - name: {{ tpl }} - add: diff --git a/salt/sys-cacher/uninstall-client.sls b/salt/sys-cacher/uninstall-client.sls index 02a461d3..dad6e837 100644 --- a/salt/sys-cacher/uninstall-client.sls +++ b/salt/sys-cacher/uninstall-client.sls @@ -4,13 +4,22 @@ SPDX-FileCopyrightText: 2023 - 2024 Benjamin Grande M. S. /dev/null') -%} +"{{ slsdotpath }}-install-client-scripts": + file.recurse: + - name: /usr/bin/ + - source: salt://{{ slsdotpath }}/files/client/bin/ + - file_mode: "0755" + - group: root + - user: root + - makedirs: True + "{{ slsdotpath }}-uninstall-client-repository-modifications": cmd.run: + - require: + - file: "{{ slsdotpath }}-install-client-scripts" - name: apt-cacher-ng-repo uninstall - stateful: True - runas: root -{% endif -%} "{{ slsdotpath }}-uninstall-client-scripts": file.absent: diff --git a/salt/sys-cacher/uninstall-client.top b/salt/sys-cacher/uninstall-client.top index 79b789cb..06cbd2a8 100644 --- a/salt/sys-cacher/uninstall-client.top +++ b/salt/sys-cacher/uninstall-client.top @@ -5,6 +5,6 @@ SPDX-License-Identifier: AGPL-3.0-or-later #} base: - 'qubes:type:template': - - match: pillar + 'I@qubes:type:template and ( ( G@os_family:Debian and not P@nodename:host and not P@nodename:whonix.* ) or G@os_family:Arch )': + - match: compound - sys-cacher.uninstall-client diff --git a/salt/sys-cacher/untag.sls b/salt/sys-cacher/untag.sls index 1ba8ae8f..e2e77aca 100644 --- a/salt/sys-cacher/untag.sls +++ b/salt/sys-cacher/untag.sls @@ -8,7 +8,7 @@ SPDX-License-Identifier: AGPL-3.0-or-later {% if wanted -%} {% for tpl in wanted.split("\n") %} -"{{ tpl }}-cacher-untag": +"{{ slsdotpath }}-del-tag-of-{{ tpl }}": qvm.tags: - name: {{ tpl }} - del: