Add tracking autoscaler and increment janitor release (#173)

* upgrade memory for tracking-consumer * add a default tracking-consumer HPA that acts very similarly to zip-consumer * update redis janitor to 0.3.0 and clean both predict and track queues
vanvalenlab · Sep 7, 2019 · df3675f · df3675f
1 parent c11339c
commit df3675f
Show file tree

Hide file tree

Showing 4 changed files with 23 additions and 6 deletions.
diff --git a/conf/helmfile.d/0220.redis-janitor.yaml b/conf/helmfile.d/0220.redis-janitor.yaml
@@ -30,7 +30,7 @@ releases:
 
       image:
         repository: "vanvalenlab/kiosk-redis-janitor"
-        tag: "0.2"
+        tag: "0.3.0"
         pullPolicy: "Always"
 
       serviceAccount:
@@ -49,6 +49,7 @@ releases:
         #   memory: 64Mi
 
       env:
+        QUEUES: "predict,track"
         DEBUG: "true"
         INTERVAL: "5"
         REDIS_HOST: "redis"

diff --git a/conf/helmfile.d/0250.tracking-consumer.yaml b/conf/helmfile.d/0250.tracking-consumer.yaml
@@ -38,7 +38,7 @@ releases:
       resources:
         requests:
           cpu: 300m
-          memory: 128Mi
+          memory: 256Mi
         # limits:
         #   cpu: 100m
         #   memory: 1024Mi

diff --git a/conf/helmfile.d/0600.prometheus-operator.yaml b/conf/helmfile.d/0600.prometheus-operator.yaml
@@ -193,6 +193,18 @@ releases:
               labels:
                 namespace: deepcell
                 service: zip-scaling-service
+            - record: tracking_consumer_key_ratio
+              expr: |-
+                avg_over_time(redis_script_value{key="track_image_keys"}[15s])
+                / on()
+                (
+                  avg_over_time(kube_deployment_spec_replicas{deployment="tracking-consumer"}[15s])
+                  +
+                  1
+                )
+              labels:
+                namespace: deepcell
+                service: tracking-scaling-service
 
       ##
       global:

diff --git a/conf/patches/hpa.yaml b/conf/patches/hpa.yaml
@@ -114,7 +114,11 @@ spec:
   minReplicas: 1
   maxReplicas: $GPU_MAX_TIMES_FIFTY
   metrics:
-  - type: Resource
-    resource:
-      name: cpu
-      targetAverageUtilization: 80
+  - type: Object
+    object:
+      metricName: tracking_consumer_key_ratio
+      target:
+        apiVersion: v1
+        kind: Namespace
+        name: tracking_consumer_key_ratio
+      targetValue: 1