From 7e8f8afdf533483b156a7c8abcda8a72855313a0 Mon Sep 17 00:00:00 2001 From: Gabriel Mainberger Date: Thu, 4 Jan 2024 14:00:46 +0100 Subject: [PATCH] Do not prune the namespace if the SUP is installed in cattle-system Most of the things in cattle-system are not Project syn managed and this can cause an unexpected loss of not Project Syn managed things. --- .cruft.json | 2 +- Makefile.vars.mk | 1 + component/main.jsonnet | 15 +- tests/cattle-system.yml | 3 + .../apps/system-upgrade-controller.yaml | 0 .../00_namespace.yaml | 8 + .../01_serviceaccount.yaml | 8 + .../02_clusterrolebinding.yaml | 16 + .../03_configmap.yaml | 18 + .../04_deployment.yaml | 72 +++ .../system-upgrade-controller/05_plans.yaml | 0 .../06_dashboard.yaml | 555 ++++++++++++++++++ 12 files changed, 696 insertions(+), 2 deletions(-) create mode 100644 tests/cattle-system.yml create mode 100644 tests/golden/cattle-system/system-upgrade-controller/apps/system-upgrade-controller.yaml create mode 100644 tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/00_namespace.yaml create mode 100644 tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/01_serviceaccount.yaml create mode 100644 tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/02_clusterrolebinding.yaml create mode 100644 tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/03_configmap.yaml create mode 100644 tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/04_deployment.yaml create mode 100644 tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/05_plans.yaml create mode 100644 tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/06_dashboard.yaml diff --git a/.cruft.json b/.cruft.json index 85a1631..938b903 100644 --- a/.cruft.json +++ b/.cruft.json @@ -7,7 +7,7 @@ "name": "system-upgrade-controller", "slug": "system-upgrade-controller", "parameter_key": "system_upgrade_controller", - "test_cases": "defaults", + "test_cases": "defaults cattle-system", "add_lib": "y", "add_pp": "n", "add_golden": "y", diff --git a/Makefile.vars.mk b/Makefile.vars.mk index dc9eb37..b2ad019 100644 --- a/Makefile.vars.mk +++ b/Makefile.vars.mk @@ -57,3 +57,4 @@ KUBENT_IMAGE ?= ghcr.io/doitintl/kube-no-trouble:latest KUBENT_DOCKER ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --entrypoint=/app/kubent $(KUBENT_IMAGE) instance ?= defaults +test_instances = tests/defaults.yml tests/cattle-system.yml diff --git a/component/main.jsonnet b/component/main.jsonnet index 779733f..0d56a7c 100644 --- a/component/main.jsonnet +++ b/component/main.jsonnet @@ -21,7 +21,20 @@ local sucImage = else '%(registry)s/%(repository)s:%(tag)s' % params.images.system_upgrade_controller; -local namespace = kube.Namespace(params.namespace); +local cattleSystemNamespaceNoPrune = + if params.namespace == 'cattle-system' then + { + metadata+: { + annotations+: { + 'argocd.argoproj.io/sync-options': 'Prune=false', + }, + }, + } + else + {}; + +local namespace = kube.Namespace(params.namespace) + + cattleSystemNamespaceNoPrune; local serviceaccount = kube.ServiceAccount(params.service_account) { metadata+: { diff --git a/tests/cattle-system.yml b/tests/cattle-system.yml new file mode 100644 index 0000000..00d65d2 --- /dev/null +++ b/tests/cattle-system.yml @@ -0,0 +1,3 @@ +parameters: + system_upgrade_controller: + namespace: cattle-system diff --git a/tests/golden/cattle-system/system-upgrade-controller/apps/system-upgrade-controller.yaml b/tests/golden/cattle-system/system-upgrade-controller/apps/system-upgrade-controller.yaml new file mode 100644 index 0000000..e69de29 diff --git a/tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/00_namespace.yaml b/tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/00_namespace.yaml new file mode 100644 index 0000000..770b46e --- /dev/null +++ b/tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/00_namespace.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Namespace +metadata: + annotations: + argocd.argoproj.io/sync-options: Prune=false + labels: + name: cattle-system + name: cattle-system diff --git a/tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/01_serviceaccount.yaml b/tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/01_serviceaccount.yaml new file mode 100644 index 0000000..92332f5 --- /dev/null +++ b/tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/01_serviceaccount.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + annotations: {} + labels: + name: system-upgrade + name: system-upgrade + namespace: cattle-system diff --git a/tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/02_clusterrolebinding.yaml b/tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/02_clusterrolebinding.yaml new file mode 100644 index 0000000..8d5bdaa --- /dev/null +++ b/tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/02_clusterrolebinding.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + annotations: {} + labels: + name: system-upgrade + name: system-upgrade + namespace: cattle-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: system-upgrade + namespace: cattle-system diff --git a/tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/03_configmap.yaml b/tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/03_configmap.yaml new file mode 100644 index 0000000..deff944 --- /dev/null +++ b/tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/03_configmap.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +data: + SYSTEM_UPGRADE_CONTROLLER_DEBUG: 'false' + SYSTEM_UPGRADE_CONTROLLER_THREADS: '2' + SYSTEM_UPGRADE_JOB_ACTIVE_DEADLINE_SECONDS: '900' + SYSTEM_UPGRADE_JOB_BACKOFF_LIMIT: '3' + SYSTEM_UPGRADE_JOB_IMAGE_PULL_POLICY: Always + SYSTEM_UPGRADE_JOB_KUBECTL_IMAGE: rancher/kubectl:v1.17.0 + SYSTEM_UPGRADE_JOB_PRIVILEGED: 'true' + SYSTEM_UPGRADE_JOB_TTL_SECONDS_AFTER_FINISH: '900' + SYSTEM_UPGRADE_PLAN_POLLING_INTERVAL: 15m +kind: ConfigMap +metadata: + annotations: {} + labels: + name: default-controller-env + name: default-controller-env + namespace: cattle-system diff --git a/tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/04_deployment.yaml b/tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/04_deployment.yaml new file mode 100644 index 0000000..72a36d2 --- /dev/null +++ b/tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/04_deployment.yaml @@ -0,0 +1,72 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: {} + labels: + app.kubernetes.io/instance: c-green-test-1234 + app.kubernetes.io/managed-by: syn + app.kubernetes.io/name: system-upgrade-controller + name: system-upgrade-controller + name: system-upgrade-controller + namespace: cattle-system +spec: + minReadySeconds: 30 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + upgrade.cattle.io/controller: system-upgrade-controller + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + upgrade.cattle.io/controller: system-upgrade-controller + spec: + affinity: {} + containers: + - args: [] + env: + - name: SYSTEM_UPGRADE_CONTROLLER_NAME + valueFrom: + fieldRef: + fieldPath: metadata.labels['upgrade.cattle.io/controller'] + - name: SYSTEM_UPGRADE_CONTROLLER_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + envFrom: + - configMapRef: + name: default-controller-env + image: docker.io/rancher/system-upgrade-controller:v0.13.1 + imagePullPolicy: IfNotPresent + name: system-upgrade-controller + ports: [] + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 250m + memory: 64Mi + stdin: false + tty: false + volumeMounts: + - mountPath: /etc/ssl + name: etc-ssl + - mountPath: /tmp + name: tmp + imagePullSecrets: [] + initContainers: [] + serviceAccountName: system-upgrade + terminationGracePeriodSeconds: 30 + volumes: + - hostPath: + path: /etc/ssl + type: Directory + name: etc-ssl + - emptyDir: {} + name: tmp diff --git a/tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/05_plans.yaml b/tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/05_plans.yaml new file mode 100644 index 0000000..e69de29 diff --git a/tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/06_dashboard.yaml b/tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/06_dashboard.yaml new file mode 100644 index 0000000..3f2fd74 --- /dev/null +++ b/tests/golden/cattle-system/system-upgrade-controller/system-upgrade-controller/06_dashboard.yaml @@ -0,0 +1,555 @@ +apiVersion: integreatly.org/v1alpha1 +kind: GrafanaDashboard +metadata: + labels: + app: platform-grafana + name: system-upgrade-controller + namespace: cattle-system +spec: + json: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 3, + "links": [ + + ], + "panels": [ + { + "aliasColors": { + "Jobs failed": "dark-red", + "Nodes completed": "dark-green", + "Running Jobs": "dark-purple" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "RANCHER_MONITORING", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null as zero", + "options": { + "dataLinks": [ + + ] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "count(kube_job_status_succeeded{namespace=\"cattle-system\"} == 1) by (node)", + "instant": false, + "interval": "", + "legendFormat": "Nodes completed", + "refId": "A" + }, + { + "expr": "sum(kube_node_info)", + "instant": false, + "interval": "", + "legendFormat": "Nodes", + "refId": "B" + }, + { + "expr": "count(kube_job_status_failed{namespace=\"cattle-system\"} > 1) by (node)", + "interval": "", + "legendFormat": "Jobs failed", + "refId": "C" + }, + { + "expr": "sum(kube_job_status_active{namespace=\"cattle-system\"} == 1) by (node)", + "interval": "", + "legendFormat": "Running Jobs", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Upgrade Jobs completed", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Cordoned nodes": "dark-red", + "Total nodes": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "RANCHER_MONITORING", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "hiddenSeries": false, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "dataLinks": [ + + ] + }, + "percentage": false, + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_node_spec_unschedulable)", + "interval": "", + "legendFormat": "Cordoned nodes", + "refId": "A" + }, + { + "expr": "count(kube_node_info)", + "interval": "", + "legendFormat": "Total nodes", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Cordoned Nodes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [ + + ], + "datasource": "RANCHER_MONITORING", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "hideTimeOverride": false, + "id": 6, + "interval": "", + "options": { + + }, + "pageSize": null, + "repeat": null, + "showHeader": true, + "sort": { + "col": 15, + "desc": true + }, + "styles": [ + { + "alias": "Node Hashes", + "align": "left", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "link": false, + "mappingType": 1, + "pattern": "Metric", + "preserveFormat": false, + "rangeMaps": [ + + ], + "sanitize": false, + "thresholds": [ + + ], + "type": "string", + "unit": "short", + "valueMaps": [ + + ] + }, + { + "alias": "", + "align": "auto", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Time", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "/label_k3s:*|Value|kubernetes_.*|__name__|instance|app_kubernetes_io_.*|job|helm.*/", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "Node", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "node", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + }, + { + "alias": "Upgrade Hash", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "/label_plan_upgrade_cattle_io_.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "kube_node_labels", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Node table", + "transform": "table", + "type": "table" + }, + { + "columns": [ + + ], + "datasource": "RANCHER_MONITORING", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 8, + "options": { + + }, + "pageSize": null, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "/__name__|endpoint|job|namespace|pod|service/", + "thresholds": [ + + ], + "type": "hidden", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ + + ], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "suc_package_upgraded", + "format": "table", + "instant": true, + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Updated Packages", + "transform": "table", + "type": "table" + } + ], + "refresh": "5s", + "schemaVersion": 21, + "style": "dark", + "tags": [ + + ], + "templating": { + "list": [ + + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "SUC overview", + "uid": "Ut3fVneZz", + "version": 1 + } + name: system-upgrade-controller.json