feat(deploy): enhance values and add monitoring (#445)

* feat(deploy): enhance values and add monitoring

* feat(deploy): add app_env_type to match newest image

* feat(deploy): add db-monitoring, enhance portals install

deploy/README.md

@@ -353,15 +353,19 @@ helm upgrade --install cnpg cnpg/cloudnative-pg --namespace postgresql-system --
 # redis operator
 helm repo add ot-helm https://ot-container-kit.github.io/helm-charts/
 helm upgrade --install redis-operator ot-helm/redis-operator --namespace redis-system --create-namespace
-```
 
-<!-- TODO: Add monitoring -->
+# monitoring
+helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
+helm upgrade --install prometheus prometheus-community/kube-prometheus-stack --namespace monitoring-system --create-namespace -f ../deploy/deployments/addons/prometheus-values.yaml
+kubectl apply -f deployments/addons/cilium-pod-monitor.yaml
+kubectl apply -f deployments/addons/pgsql-operator-pod-monitor.yaml
+```
 
 <!-- TODO: Add logging -->
 
 <!-- TODO: Configure addons -->
 
-<!-- TODO: Add horizontal and vertical autoscaler -->
+<!-- TODO: Add cluster autoscaler -->
 
 <!-- TODO: Add access for more users -->
 
@@ -408,3 +412,11 @@ curl --request POST --url https://api.cloudflare.com/client/v4/zones/<CLOUDFLARE
 ```
 
 Ready, you can connect to portals on your configured url.
+
+### Load Test
+
+If you want to load test the application you could use wrk.
+
+```sh
+docker run -it --name load-test --rm alpine:latest /bin/sh -c "apk update && apk add wrk && apk add curl && ulimit -n 65535 && wrk -t12 -c400 -d120s https://<YOUR_PORTALS_DOMAIN>/login"
+```

deploy/deployments/addons/cilium-pod-monitor.yaml

@@ -0,0 +1,23 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: cilium-agent
+  namespace: cilium-system
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: cilium-agent
+  podMetricsEndpoints:
+    - port: prometheus
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: cilium-operator
+  namespace: cilium-system
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: cilium-operator
+  podMetricsEndpoints:
+    - port: prometheus

deploy/deployments/addons/cilium-values.yaml

@@ -41,3 +41,8 @@ image:
 operator:
   rollOutPods: true
   priorityClassName: "system-node-critical"
+  prometheus:
+    enabled: true
+
+prometheus:
+  enabled: true

deploy/deployments/addons/ingress-nginx-values.yaml

@@ -1,10 +1,18 @@
 controller:
   config:
     use-proxy-protocol: true
+    #proxy-real-ip-cidr: "173.245.48.0/20,103.21.244.0/22,103.22.200.0/22,103.31.4.0/22,141.101.64.0/18,108.162.192.0/18,190.93.240.0/20,188.114.96.0/20,197.234.240.0/22,198.41.128.0/17,162.158.0.0/15,104.16.0.0/13,104.24.0.0/14,172.64.0.0/13,131.0.72.0/22,2400:cb00::/32,2606:4700::/32,2803:f800::/32,2405:b500::/32,2405:8100::/32,2a06:98c0::/29,2c0f:f248::/32"
+    #proxy-real-ip-cidr: "10.0.0.8/32"
+    #use-forwarded-headers: true
+    #forwarded-for-header: "CF-Connecting-IP"
   replicaCount: 2
   service:
     annotations:
       load-balancer.hetzner.cloud/location: fsn1
       load-balancer.hetzner.cloud/use-private-ip: true
       load-balancer.hetzner.cloud/name: <YOUR_LB_NAME>
       load-balancer.hetzner.cloud/uses-proxyprotocol: true
+  metrics:
+    enabled: true
+    serviceMonitor:
+      enabled: true

deploy/deployments/addons/pgsql-operator-pod-monitor.yaml

@@ -0,0 +1,11 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: cnpg-controller-manager
+  namespace: postgresql-system
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: cloudnative-pg
+  podMetricsEndpoints:
+    - port: metrics

deploy/deployments/addons/prometheus-values.yaml

@@ -0,0 +1,100 @@
+# see https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/values.yaml
+alertmanager:
+  config:
+    route:
+      routes:
+        - receiver: "null"
+          matchers:
+            - alertname =~ "InfoInhibitor|Watchdog"
+        - receiver: telegram
+          matchers:
+            - severity =~ "critical|warning|info"
+    receivers:
+      - name: "null"
+      - name: telegram
+        telegram_configs:
+          - bot_token: <YOUR_BOT_TOKEN>
+            api_url: https://api.telegram.org
+            chat_id: <YOUR_CHAT_ID>
+  ingress:
+    enabled: false
+    ingressClassName: nginx
+    hosts:
+      - <YOUR_URL>
+    annotations:
+      cert-manager.io/cluster-issuer: "letsencrypt-staging"
+    tls:
+      - secretName: alertmanager-ingress-tls
+        hosts:
+          - <YOUR_URL>
+grafana:
+  adminPassword: <YOUR_PASSWORD>
+  ingress:
+    enabled: true
+    ingressClassName: nginx
+    annotations:
+      cert-manager.io/cluster-issuer: "letsencrypt-staging"
+    hosts:
+      - <YOUR_URL>
+    tls:
+      - secretName: grafana-ingress-tls
+        hosts:
+          - <YOUR_URL>
+kube-state-metrics:
+  prometheus:
+    monitor:
+      enabled: true
+      relabelings:
+        - action: replace
+          regex: (.*)
+          replacement: $1
+          sourceLabels:
+            - __meta_kubernetes_pod_node_name
+          targetLabel: kubernetes_node
+nodeExporter:
+  serviceMonitor:
+    relabelings:
+      - action: replace
+        regex: (.*)
+        replacement: $1
+        sourceLabels:
+          - __meta_kubernetes_pod_node_name
+        targetLabel: kubernetes_node
+prometheus-node-exporter:
+  prometheus:
+    monitor:
+      enabled: true
+      relabelings:
+        - action: replace
+          regex: (.*)
+          replacement: $1
+          sourceLabels:
+            - __meta_kubernetes_pod_node_name
+          targetLabel: kubernetes_node
+kubeProxy:
+  enabled: false # kube proxy not used because of proxy replacement by cilium
+prometheus:
+  ingress:
+    enabled: false
+    ingressClassName: nginx
+    annotations:
+      cert-manager.io/cluster-issuer: "letsencrypt-staging"
+    hosts:
+      - <YOUR_URL>
+    tls:
+      - secretName: prometheus-ingress-tls
+        hosts:
+          - <YOUR_URL>
+  prometheusSpec:
+    retention: 14d
+    storageSpec:
+      volumeClaimTemplate:
+        spec:
+          accessModes: ["ReadWriteOnce"]
+          resources:
+            requests:
+              storage: 50Gi
+    ruleSelectorNilUsesHelmValues: false
+    serviceMonitorSelectorNilUsesHelmValues: false
+    podMonitorSelectorNilUsesHelmValues: false
+    probeSelectorNilUsesHelmValues: false

deploy/deployments/portals/pgsql.yaml

@@ -8,7 +8,39 @@ spec:
   primaryUpdateStrategy: unsupervised
   storage:
     size: 10Gi
+  postgresql:
+    parameters:
+      max_connections: "300"
+      shared_buffers: "80MB"
   bootstrap:
     initdb:
       database: app
       owner: app
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: portals-db
+  namespace: portals
+spec:
+  selector:
+    matchLabels:
+      cnpg.io/cluster: portals-db
+  podMetricsEndpoints:
+    - port: metrics
+#---
+#apiVersion: postgresql.cnpg.io/v1
+#kind: Pooler
+#metadata:
+#  name: portals-db-pooler
+#  namespace: portals
+#spec:
+#  cluster:
+#    name: portals-db
+#  instances: 3
+#  type: rw
+#  pgbouncer:
+#    poolMode: transaction
+#    parameters:
+#      max_client_conn: "1000"
+#      default_pool_size: "20"

deploy/deployments/portals/portals-values.yaml

@@ -4,6 +4,7 @@ environment:
   APP_DEBUG: "false"
   APP_ENV: production
   APP_FORCE_HTTPS: "true"
+  APP_EVENT_TYPE: "demo"
   TUTOR_PASSWORD: password
   ADMIN_PASSWORD: admin
   DB_HOST: portals-db-rw.portals.svc
@@ -18,10 +19,16 @@ ingress:
     - portals.fsr5.de
   annotations:
     cert-manager.io/cluster-issuer: "letsencrypt-prod"
-replicaCount: 3
+    nginx.ingress.kubernetes.io/limit-rps: "500"
+replicaCount: 5
 migrateJob:
   onInstall: true
   onUpgrade: false
   seed: true
   tutorsCsvConfigMapName: portals-tutors-csv
   studentsCsvConfigMapName: portals-students-csv
+hpa:
+  enabled: true
+  minReplicas: 5
+  maxReplicas: 10
+  averageCPUUtilization: 90