ci: add concurrency groups

I've been running into rate limit and deployment size limitations so I'm adding concurrency limits.
redpanda-data · Jun 21, 2024 · 67c9516 · 67c9516
1 parent 8b843d4
commit 67c9516
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 1 deletion.
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -4,6 +4,8 @@ agents:
 steps:
   - label: aws ubuntu
     key: aws-up-ubuntu
+    concurrency_group: aws-ub
+    concurrency: 1
     command: make ci-aws-rp -e DEPLOYMENT_ID=ci-bs-ub-`tr -dc a-z0-9 </dev/urandom | head -c 4` -e DISTRO=ubuntu-focal --keep-going
     plugins:
       - seek-oss/aws-sm#v2.3.2:
@@ -18,6 +20,8 @@ steps:
             - AWS_DEFAULT_REGION
   - label: aws ubuntu tiered
     key: aws-up-ubuntu-tiered
+    concurrency_group: aws-ub
+    concurrency: 1
     command: make ci-aws-rp-tiered -e DEPLOYMENT_ID=ci-ts-ub-`tr -dc a-z0-9 </dev/urandom | head -c 4` -e DISTRO=ubuntu-focal --keep-going
     plugins:
       - seek-oss/aws-sm#v2.3.2:
@@ -32,6 +36,8 @@ steps:
             - AWS_DEFAULT_REGION
   - label: aws ubuntu tiered large
     key: aws-up-ubuntu-ts-large
+    concurrency_group: aws-ub
+    concurrency: 1
     command: make ci-aws-rp-tiered -e DEPLOYMENT_ID=ci-ts-ub-lg-`tr -dc a-z0-9 </dev/urandom | head -c 4` -e DISTRO=ubuntu-focal -e INSTANCE_TYPE_AWS=is4gen.4xlarge -e MACHINE_ARCH=arm64 --keep-going
     plugins:
       - seek-oss/aws-sm#v2.3.2:
@@ -46,6 +52,8 @@ steps:
             - AWS_DEFAULT_REGION
   - label: aws fedora
     key: aws-up-fedora
+    concurrency_group: aws-fd
+    concurrency: 1
     command: make ci-aws-rp -e DEPLOYMENT_ID=ci-bs-fd-`tr -dc a-z0-9 </dev/urandom | head -c 4` -e DISTRO=Fedora-Cloud-Base-37 --keep-going
     plugins:
       - seek-oss/aws-sm#v2.3.2:
@@ -60,6 +68,8 @@ steps:
             - AWS_DEFAULT_REGION
   - label: aws fedora connect
     key: aws-up-fed-con
+    concurrency_group: aws-fd
+    concurrency: 1
     command: make ci-aws-rp-connect -e DEPLOYMENT_ID=ci-cn-fd-`tr -dc a-z0-9 </dev/urandom | head -c 4` -e DISTRO=Fedora-Cloud-Base-37 --keep-going
     plugins:
       - seek-oss/aws-sm#v2.3.2:
@@ -75,6 +85,8 @@ steps:
             - CONNECT_RPM_TOKEN
   - label: aws fedora tiered
     key: aws-up-fedora-tiered
+    concurrency_group: aws-fd
+    concurrency: 1
     command: make ci-aws-rp-tiered -e DEPLOYMENT_ID=ci-ts-fd-`tr -dc a-z0-9 </dev/urandom | head -c 4` -e DISTRO=Fedora-Cloud-Base-37 --keep-going
     plugins:
       - seek-oss/aws-sm#v2.3.2:
@@ -89,6 +101,8 @@ steps:
             - AWS_DEFAULT_REGION
   - label: aws fedora tiered connect
     key: aws-up-fed-cts
+    concurrency_group: aws-fd
+    concurrency: 1
     command: make ci-aws-rp-ts-connect -e DEPLOYMENT_ID=ci-ct-fd-`tr -dc a-z0-9 </dev/urandom | head -c 4` -e DISTRO=Fedora-Cloud-Base-37 --keep-going
     plugins:
       - seek-oss/aws-sm#v2.3.2:
@@ -106,6 +120,8 @@ steps:
             - CONNECT_RPM_TOKEN
   - label: aws fedora tiered large
     key: aws-up-fedora-ts-large
+    concurrency_group: aws-fd
+    concurrency: 1
     command: make ci-aws-rp-tiered -e DEPLOYMENT_ID=ci-ts-fd-lg-`tr -dc a-z0-9 </dev/urandom | head -c 4` -e DISTRO=Fedora-Cloud-Base-37 -e INSTANCE_TYPE_AWS=is4gen.4xlarge -e MACHINE_ARCH=arm64 --keep-going
     plugins:
       - seek-oss/aws-sm#v2.3.2:
@@ -120,6 +136,8 @@ steps:
             - AWS_DEFAULT_REGION
   - label: gcp ubuntu basic
     key: gcp-up-ubuntu
+    concurrency_group: gcp-ub
+    concurrency: 1
     command: make ci-gcp-rp -e GCP_IMAGE="ubuntu-os-cloud/ubuntu-2204-lts" -e DEPLOYMENT_ID="ci-bs-ub-`tr -dc a-z0-9 </dev/urandom | head -c 4`" -e GCP_CREDS="$DEVEX_GCP_CREDS_BASE64" --keep-going
     plugins:
       - seek-oss/aws-sm#v2.3.2:
@@ -132,6 +150,8 @@ steps:
             - DEVEX_GCP_CREDS_BASE64
   - label: gcp ubuntu tiered
     key: gcp-up-ubuntu-tiered
+    concurrency_group: gcp-ub
+    concurrency: 1
     command: make ci-gcp-rp-tiered -e GCP_IMAGE="ubuntu-os-cloud/ubuntu-2204-lts" -e DEPLOYMENT_ID="ci-ts-ub-`tr -dc a-z0-9 </dev/urandom | head -c 4`" -e GCP_CREDS="$DEVEX_GCP_CREDS_BASE64" --keep-going
     plugins:
       - seek-oss/aws-sm#v2.3.2:
@@ -145,6 +165,8 @@ steps:
             - DEVEX_GCP_CREDS_BASE64
   - label: gcp fedora basic
     key: gcp-up-fedora
+    concurrency_group: gcp-fd
+    concurrency: 1
     command: make ci-gcp-rp -e GCP_IMAGE="fedora-cloud/fedora-cloud-37" -e DEPLOYMENT_ID="ci-bs-fd-`tr -dc a-z0-9 </dev/urandom | head -c 4`" -e GCP_CREDS="$DEVEX_GCP_CREDS_BASE64" --keep-going
     plugins:
       - seek-oss/aws-sm#v2.3.2:
@@ -157,6 +179,8 @@ steps:
             - DEVEX_GCP_CREDS_BASE64
   - label: gcp fedora tiered
     key: gcp-up-fedora-tiered
+    concurrency_group: gcp-fd
+    concurrency: 1
     command: make ci-gcp-rp-tiered -e GCP_IMAGE="fedora-cloud/fedora-cloud-37" -e DEPLOYMENT_ID="ci-ud-fd-`tr -dc a-z0-9 </dev/urandom | head -c 4`" -e GCP_CREDS="$DEVEX_GCP_CREDS_BASE64" --keep-going
     plugins:
       - seek-oss/aws-sm#v2.3.2:
@@ -169,6 +193,8 @@ steps:
             - DEVEX_GCP_CREDS_BASE64
   - label: unstable aws fedora tiered
     key: aws-us-fedora-tiered
+    concurrency_group: unstable
+    concurrency: 1
     command: make ci-aws-rp-tiered -e DEPLOYMENT_ID=ci-ts-fd-us-`tr -dc a-z0-9 </dev/urandom | head -c 4` -e DISTRO=Fedora-Cloud-Base-37 -e IS_USING_UNSTABLE=true --keep-going
     plugins:
       - seek-oss/aws-sm#v2.3.2:
@@ -183,6 +209,8 @@ steps:
             - AWS_DEFAULT_REGION
   - label: unstable aws fedora tiered large
     key: aws-us-fedora-ts-large
+    concurrency_group: unstable
+    concurrency: 1
     command: make ci-aws-rp-tiered -e DEPLOYMENT_ID=ci-ts-fd-us-lg-`tr -dc a-z0-9 </dev/urandom | head -c 4` -e DISTRO=Fedora-Cloud-Base-37 -e IS_USING_UNSTABLE=true -e INSTANCE_TYPE_AWS=is4gen.4xlarge -e MACHINE_ARCH=arm64 --keep-going
     plugins:
       - seek-oss/aws-sm#v2.3.2:
@@ -197,6 +225,8 @@ steps:
             - AWS_DEFAULT_REGION
   - label: unstable aws ubuntu tiered
     key: aws-us-ubuntu-tiered
+    concurrency_group: unstable
+    concurrency: 1
     command: make ci-aws-rp-tiered -e DEPLOYMENT_ID=ci-ts-ub-us-`tr -dc a-z0-9 </dev/urandom | head -c 4` -e DISTRO=ubuntu-focal -e IS_USING_UNSTABLE=true --keep-going
     plugins:
       - seek-oss/aws-sm#v2.3.2:
@@ -211,6 +241,8 @@ steps:
             - AWS_DEFAULT_REGION
   - label: unstable aws ubuntu tiered large
     key: aws-us-ubuntu-ts-large
+    concurrency_group: unstable
+    concurrency: 1
     command: make ci-aws-rp-tiered -e DEPLOYMENT_ID=ci-ts-ub-us-lg-`tr -dc a-z0-9 </dev/urandom | head -c 4` -e DISTRO=ubuntu-focal -e IS_USING_UNSTABLE=true -e INSTANCE_TYPE_AWS=is4gen.4xlarge -e MACHINE_ARCH=arm64 --keep-going
     plugins:
       - seek-oss/aws-sm#v2.3.2:

diff --git a/Makefile b/Makefile
@@ -175,7 +175,9 @@ destroy-aws:
 		-var='hosts_file=$(ANSIBLE_INVENTORY)' \
 		-var='machine_architecture=$(MACHINE_ARCH)' \
 		-var='enable_connect=$(ENABLE_CONNECT)' \
-		-var='instance_type=$(INSTANCE_TYPE)'
+		-var='broker_instance_type=$(INSTANCE_TYPE_AWS)' \
+		-var='client_instance_type=$(INSTANCE_TYPE_AWS)' \
+		-var='prometheus_instance_type=$(INSTANCE_TYPE_AWS)'
 
 .PHONY: build-aws-proxy
 build-aws-proxy: