Add TZ support; simplify scaling

clickup · Mar 21, 2024 · 479b0f3 · 479b0f3
1 parent e6f5167
commit 479b0f3
Show file tree

Hide file tree

Showing 7 changed files with 180 additions and 80 deletions.
diff --git a/docs/classes/CiStorage.md b/docs/classes/CiStorage.md
@@ -53,7 +53,7 @@ Construct.constructor
 
 #### Defined in
 
-[src/CiStorage.ts:161](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L161)
+[src/CiStorage.ts:184](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L184)
 
 ## Properties
 
@@ -63,7 +63,7 @@ Construct.constructor
 
 #### Defined in
 
-[src/CiStorage.ts:150](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L150)
+[src/CiStorage.ts:173](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L173)
 
 ___
 
@@ -73,7 +73,7 @@ ___
 
 #### Defined in
 
-[src/CiStorage.ts:151](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L151)
+[src/CiStorage.ts:174](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L174)
 
 ___
 
@@ -83,7 +83,7 @@ ___
 
 #### Defined in
 
-[src/CiStorage.ts:152](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L152)
+[src/CiStorage.ts:175](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L175)
 
 ___
 
@@ -93,7 +93,7 @@ ___
 
 #### Defined in
 
-[src/CiStorage.ts:153](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L153)
+[src/CiStorage.ts:176](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L176)
 
 ___
 
@@ -110,7 +110,7 @@ ___
 
 #### Defined in
 
-[src/CiStorage.ts:154](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L154)
+[src/CiStorage.ts:177](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L177)
 
 ___
 
@@ -120,7 +120,7 @@ ___
 
 #### Defined in
 
-[src/CiStorage.ts:155](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L155)
+[src/CiStorage.ts:178](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L178)
 
 ___
 
@@ -130,7 +130,7 @@ ___
 
 #### Defined in
 
-[src/CiStorage.ts:156](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L156)
+[src/CiStorage.ts:179](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L179)
 
 ___
 
@@ -140,7 +140,7 @@ ___
 
 #### Defined in
 
-[src/CiStorage.ts:157](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L157)
+[src/CiStorage.ts:180](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L180)
 
 ___
 
@@ -150,7 +150,7 @@ ___
 
 #### Defined in
 
-[src/CiStorage.ts:158](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L158)
+[src/CiStorage.ts:181](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L181)
 
 ___
 
@@ -160,7 +160,7 @@ ___
 
 #### Defined in
 
-[src/CiStorage.ts:159](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L159)
+[src/CiStorage.ts:182](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L182)
 
 ___
 
@@ -170,7 +170,7 @@ ___
 
 #### Defined in
 
-[src/CiStorage.ts:162](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L162)
+[src/CiStorage.ts:185](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L185)
 
 ___
 
@@ -180,7 +180,7 @@ ___
 
 #### Defined in
 
-[src/CiStorage.ts:163](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L163)
+[src/CiStorage.ts:186](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L186)
 
 ___
 
@@ -190,4 +190,4 @@ ___
 
 #### Defined in
 
-[src/CiStorage.ts:164](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L164)
+[src/CiStorage.ts:187](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L187)
diff --git a/docs/interfaces/CiStorageProps.md b/docs/interfaces/CiStorageProps.md
@@ -17,7 +17,7 @@ VPC to use by this construct.
 
 #### Defined in
 
-[src/CiStorage.ts:52](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L52)
+[src/CiStorage.ts:56](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L56)
 
 ___
 
@@ -30,7 +30,7 @@ instances.
 
 #### Defined in
 
-[src/CiStorage.ts:55](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L55)
+[src/CiStorage.ts:59](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L59)
 
 ___
 
@@ -42,7 +42,7 @@ Id of the Security Group to set for the created instances.
 
 #### Defined in
 
-[src/CiStorage.ts:57](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L57)
+[src/CiStorage.ts:61](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L61)
 
 ___
 
@@ -61,7 +61,7 @@ A Hosted Zone to register the host instances in.
 
 #### Defined in
 
-[src/CiStorage.ts:59](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L59)
+[src/CiStorage.ts:63](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L63)
 
 ___
 
@@ -74,7 +74,19 @@ must pre-exist.
 
 #### Defined in
 
-[src/CiStorage.ts:67](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L67)
+[src/CiStorage.ts:71](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L71)
+
+___
+
+### timeZone
+
+• `Optional` **timeZone**: `string`
+
+Time zone for instances, example: America/Los_Angeles.
+
+#### Defined in
+
+[src/CiStorage.ts:73](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L73)
 
 ___
 
@@ -93,16 +105,18 @@ Configuration for self-hosted runner instances in the pool.
 | `imageSsmName` | `string` | SSM parameter name which holds the reference to an instance image. |
 | `volumeGb` | `number` | Size of the root volume. |
 | `instanceRequirements` | [`InstanceRequirementsProperty`, ...InstanceRequirementsProperty[]] | The list of requirements to choose Spot Instances. |
-| `scale` | \{ `onDemandPercentageAboveBaseCapacity`: `number` ; `maxActiveRunnersPercent`: `number` ; `minIdleRunnersCount`: `number` ; `maxCapacity`: `number` ; `maxInstanceLifetime`: `Duration`  } | Scaling options. |
+| `scale` | \{ `onDemandPercentageAboveBaseCapacity`: `number` ; `maxActiveRunnersPercent`: \{ `periodSec`: `number` ; `value`: `number`  } ; `minCapacity`: \{ `id`: `string` ; `value`: `number` ; `cron`: \{ `timeZone?`: `string`  } & `CronOptions`  }[] ; `maxCapacity`: `number` ; `maxInstanceLifetime`: `Duration`  } | Scaling options. |
 | `scale.onDemandPercentageAboveBaseCapacity` | `number` | The percentages of On-Demand Instances and Spot Instances for your additional capacity. |
-| `scale.maxActiveRunnersPercent` | `number` | Maximum percentage of active runners. If the number of active runners grows beyond this threshold, the autoscaling group will launch new instances until the percentage drops. |
-| `scale.minIdleRunnersCount` | `number` | Minimal number of idle runners to keep. If the auto scaling group has less than this number of idle runners, the new instances will be created. |
+| `scale.maxActiveRunnersPercent` | \{ `periodSec`: `number` ; `value`: `number`  } | Maximum percentage of active runners. If the MAX metric of number of active runners within the recent periodSec interval grows beyond this threshold, the autoscaling group will launch new instances until the percentage drops, or maxCapacity is reached. |
+| `scale.maxActiveRunnersPercent.periodSec` | `number` | Calculate MAX metric within that period. The higher is the value, the slower will the capacity lower (but it doesn't affect how fast will it increase). |
+| `scale.maxActiveRunnersPercent.value` | `number` | Value to use for the target percentage of active (busy) runners. |
+| `scale.minCapacity` | \{ `id`: `string` ; `value`: `number` ; `cron`: \{ `timeZone?`: `string`  } & `CronOptions`  }[] | Minimal number of idle runners to keep, depending on the daytime. If the auto scaling group has less than this number of instances, the new instances will be created. |
 | `scale.maxCapacity` | `number` | Maximum total number of instances. |
 | `scale.maxInstanceLifetime` | `Duration` | Re-create instances time to time. |
 
 #### Defined in
 
-[src/CiStorage.ts:69](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L69)
+[src/CiStorage.ts:75](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L75)
 
 ___
 
@@ -128,4 +142,4 @@ runner has its localhost ports redirected to that instance.
 
 #### Defined in
 
-[src/CiStorage.ts:107](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L107)
+[src/CiStorage.ts:130](https://github.com/clickup/ci-storage-cdk/blob/master/src/CiStorage.ts#L130)
diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@clickup/ci-storage-cdk",
   "description": "A CDK construct to deploy ci-storage infrastructure",
-  "version": "2.10.292",
+  "version": "2.10.293",
   "license": "MIT",
   "keywords": [
     "cdk",

diff --git a/src/CiStorage.ts b/src/CiStorage.ts
@@ -1,9 +1,13 @@
 import { ArnFormat, Duration, Stack, Tags } from "aws-cdk-lib";
-import type { CfnAutoScalingGroup } from "aws-cdk-lib/aws-autoscaling";
+import type {
+  CfnAutoScalingGroup,
+  CronOptions,
+} from "aws-cdk-lib/aws-autoscaling";
 import {
   AutoScalingGroup,
   GroupMetrics,
   OnDemandAllocationStrategy,
+  Schedule,
   SpotAllocationStrategy,
   UpdatePolicy,
 } from "aws-cdk-lib/aws-autoscaling";
@@ -65,6 +69,8 @@ export interface CiStorageProps {
   /** A name of secret in Secrets Manager which holds GitHub PAT. This secret
    * must pre-exist. */
   ghTokenSecretName: string;
+  /** Time zone for instances, example: America/Los_Angeles. */
+  timeZone?: string;
   /** Configuration for self-hosted runner instances in the pool. */
   runner: {
     /** "{owner}/{repository}" which this self-hosted runners pool serves. */
@@ -87,14 +93,31 @@ export interface CiStorageProps {
       /** The percentages of On-Demand Instances and Spot Instances for your
        * additional capacity. */
       onDemandPercentageAboveBaseCapacity: number;
-      /** Maximum percentage of active runners. If the number of active runners
-       * grows beyond this threshold, the autoscaling group will launch new
-       * instances until the percentage drops. */
-      maxActiveRunnersPercent: number;
-      /** Minimal number of idle runners to keep. If the auto scaling group has
-       * less than this number of idle runners, the new instances will be
-       * created. */
-      minIdleRunnersCount: number;
+      /** Maximum percentage of active runners. If the MAX metric of number of
+       * active runners within the recent periodSec interval grows beyond this
+       * threshold, the autoscaling group will launch new instances until the
+       * percentage drops, or maxCapacity is reached. */
+      maxActiveRunnersPercent: {
+        /** Calculate MAX metric within that period. The higher is the value,
+         * the slower will the capacity lower (but it doesn't affect how fast
+         * will it increase). */
+        periodSec: number;
+        /** Value to use for the target percentage of active (busy) runners. */
+        value: number;
+      };
+      /** Minimal number of idle runners to keep, depending on the daytime. If
+       * the auto scaling group has less than this number of instances, the new
+       * instances will be created. */
+      minCapacity: Array<{
+        /** Alpha-numeric id of this schedule. */
+        id: string;
+        /** Value to assign to minCapacity when reaching the schedule time. Note
+         * that it doesn't apply retrospectively, i.e. there is no processing of
+         * past-due schedules in AWS. */
+        value: number;
+        /** Schedule info. Time zone example: America/Los_Angeles. */
+        cron: { timeZone?: string } & CronOptions;
+      }>;
       /** Maximum total number of instances. */
       maxCapacity: number;
       /** Re-create instances time to time. */
@@ -257,6 +280,8 @@ export class CiStorage extends Construct {
             ghDockerComposeDirectoryUrl:
               props.runner.ghDockerComposeDirectoryUrl,
             keyPairPrivateKeySecretName: this.keyPairPrivateKeySecretName,
+            timeZone: props.timeZone,
+            mount: undefined,
           }),
         ),
       );
@@ -291,7 +316,6 @@ export class CiStorage extends Construct {
       this.autoScalingGroup = new AutoScalingGroup(this, id.pascal, {
         autoScalingGroupName: keyNamer.pathKebabFrom(scope),
         vpc: this.vpc,
-        minCapacity: 1, // props.runner.scale.minIdleRunnersCount,
         maxCapacity: props.runner.scale.maxCapacity,
         maxInstanceLifetime: props.runner.scale.maxInstanceLifetime,
         mixedInstancesPolicy: {
@@ -314,27 +338,29 @@ export class CiStorage extends Construct {
         groupMetrics: [GroupMetrics.all()],
         updatePolicy: UpdatePolicy.rollingUpdate(),
       });
-      const namespace = "ci-storage/metrics";
+      Tags.of(this.autoScalingGroup).add(
+        "Name",
+        namer(keyNamer, "runner").kebab,
+      );
       this.autoScalingGroup.scaleToTrackMetric("ActiveRunnersPercent", {
         metric: new Metric({
-          namespace,
+          namespace: "ci-storage/metrics",
           metricName: "ActiveRunnersPercent",
           dimensionsMap: { GH_REPOSITORY: props.runner.ghRepository },
-          period: Duration.seconds(10),
-          statistic: "max",
-        }),
-        targetValue: props.runner.scale.maxActiveRunnersPercent,
-      });
-      this.autoScalingGroup.scaleToTrackMetric("IdleRunnersCountInverse", {
-        metric: new Metric({
-          namespace,
-          metricName: "IdleRunnersCountInverse",
-          dimensionsMap: { GH_REPOSITORY: props.runner.ghRepository },
-          period: Duration.seconds(10),
+          period: Duration.seconds(
+            props.runner.scale.maxActiveRunnersPercent.periodSec,
+          ),
           statistic: "max",
         }),
-        targetValue: 1000000 - props.runner.scale.minIdleRunnersCount,
+        targetValue: props.runner.scale.maxActiveRunnersPercent.value,
       });
+      for (const { id, value, cron } of props.runner.scale.minCapacity) {
+        this.autoScalingGroup.scaleOnSchedule(id, {
+          minCapacity: value,
+          timeZone: cron.timeZone ?? props.timeZone,
+          schedule: Schedule.cron(cron),
+        });
+      }
     }
 
     {
@@ -394,6 +420,7 @@ export class CiStorage extends Construct {
               ghDockerComposeDirectoryUrl:
                 props.host.ghDockerComposeDirectoryUrl,
               keyPairPrivateKeySecretName: this.keyPairPrivateKeySecretName,
+              timeZone: props.timeZone,
               mount: { volumeId: volume.attrVolumeId, path: "/mnt" },
             }),
           ),

diff --git a/src/__tests__/CiStorage.test.ts b/src/__tests__/CiStorage.test.ts
@@ -29,6 +29,7 @@ class CiStorageStack extends Stack {
         zoneName: "test-zoneName",
       },
       ghTokenSecretName: "ci-storage/gh-token",
+      timeZone: "America/Los_Angeles",
       runner: {
         ghRepository: "time-loop/slapdash",
         ghDockerComposeDirectoryUrl:
@@ -43,8 +44,22 @@ class CiStorageStack extends Stack {
         ],
         scale: {
           onDemandPercentageAboveBaseCapacity: 10,
-          maxActiveRunnersPercent: 80,
-          minIdleRunnersCount: 5,
+          maxActiveRunnersPercent: {
+            periodSec: 600,
+            value: 70,
+          },
+          minCapacity: [
+            {
+              id: "CaWorkDayStarts",
+              value: 10,
+              cron: { hour: "8" },
+            },
+            {
+              id: "CaWorkDayEnds",
+              value: 5,
+              cron: { hour: "18" },
+            },
+          ],
           maxCapacity: 20,
           maxInstanceLifetime: Duration.days(1),
         },