From df3675fda48c73981475b96b7968f90e9f059b6b Mon Sep 17 00:00:00 2001
From: willgraf <7930703+willgraf@users.noreply.github.com>
Date: Fri, 6 Sep 2019 17:20:38 -0700
Subject: [PATCH] Add tracking autoscaler and increment janitor release (#173)

* upgrade memory for tracking-consumer

* add a default tracking-consumer HPA that acts very similarly to zip-consumer

* update redis janitor to 0.3.0 and clean both predict and track queues
---
 conf/helmfile.d/0220.redis-janitor.yaml       |  3 ++-
 conf/helmfile.d/0250.tracking-consumer.yaml   |  2 +-
 conf/helmfile.d/0600.prometheus-operator.yaml | 12 ++++++++++++
 conf/patches/hpa.yaml                         | 12 ++++++++----
 4 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/conf/helmfile.d/0220.redis-janitor.yaml b/conf/helmfile.d/0220.redis-janitor.yaml
index 972ad8be..7a2873b9 100644
--- a/conf/helmfile.d/0220.redis-janitor.yaml
+++ b/conf/helmfile.d/0220.redis-janitor.yaml
@@ -30,7 +30,7 @@ releases:
 
       image:
         repository: "vanvalenlab/kiosk-redis-janitor"
-        tag: "0.2"
+        tag: "0.3.0"
         pullPolicy: "Always"
 
       serviceAccount:
@@ -49,6 +49,7 @@ releases:
         #   memory: 64Mi
 
       env:
+        QUEUES: "predict,track"
         DEBUG: "true"
         INTERVAL: "5"
         REDIS_HOST: "redis"
diff --git a/conf/helmfile.d/0250.tracking-consumer.yaml b/conf/helmfile.d/0250.tracking-consumer.yaml
index ec9124ae..11fe8dc2 100644
--- a/conf/helmfile.d/0250.tracking-consumer.yaml
+++ b/conf/helmfile.d/0250.tracking-consumer.yaml
@@ -38,7 +38,7 @@ releases:
       resources:
         requests:
           cpu: 300m
-          memory: 128Mi
+          memory: 256Mi
         # limits:
         #   cpu: 100m
         #   memory: 1024Mi
diff --git a/conf/helmfile.d/0600.prometheus-operator.yaml b/conf/helmfile.d/0600.prometheus-operator.yaml
index 2c443ac6..886b9c92 100644
--- a/conf/helmfile.d/0600.prometheus-operator.yaml
+++ b/conf/helmfile.d/0600.prometheus-operator.yaml
@@ -193,6 +193,18 @@ releases:
               labels:
                 namespace: deepcell
                 service: zip-scaling-service
+            - record: tracking_consumer_key_ratio
+              expr: |-
+                avg_over_time(redis_script_value{key="track_image_keys"}[15s])
+                / on()
+                (
+                  avg_over_time(kube_deployment_spec_replicas{deployment="tracking-consumer"}[15s])
+                  +
+                  1
+                )
+              labels:
+                namespace: deepcell
+                service: tracking-scaling-service
 
       ##
       global:
diff --git a/conf/patches/hpa.yaml b/conf/patches/hpa.yaml
index 35b86159..742780b7 100644
--- a/conf/patches/hpa.yaml
+++ b/conf/patches/hpa.yaml
@@ -114,7 +114,11 @@ spec:
   minReplicas: 1
   maxReplicas: $GPU_MAX_TIMES_FIFTY
   metrics:
-  - type: Resource
-    resource:
-      name: cpu
-      targetAverageUtilization: 80
+  - type: Object
+    object:
+      metricName: tracking_consumer_key_ratio
+      target:
+        apiVersion: v1
+        kind: Namespace
+        name: tracking_consumer_key_ratio
+      targetValue: 1