MDBF-709: KVM Bintar Builders Need More Resources

The core reason for this pull request is that the KVM Bintar queues on buildbot.mariadb.org have become quite large, with delays of approximately 2-3 weeks at the time of writing.

We have three KVM Bintar builders:

- x86-kvm-centos6-bintar
- x86-kvm-ubuntu-1210-bintar (this one is no longer triggered as it only runs for updates to MariaDB server 10.4.x, which is no longer maintained)
- x86-kvm-ubuntu-1604-bintar

Strategy:

    The maximum builds for the current worker (apexis-bbw3) have been increased to 2, allowing CentOS and Ubuntu builders to run in parallel.
    We have configured the bg-bbw1-x64 host with a maximum builds of 1 to enable an additional parallel build for either CentOS or Ubuntu.

Prerequisites:

    bg-bbw1 preparation:
        - Preparing buildbot-worker services for both DEV and PROD environments. The PROD service is currently disabled until this commit reaches production.
        - KVM QCOW2 image files were replicated on this host and QEMU-KVM was configured.

Technical Details:

    - The .env.dev file is sourced when docker-compose orchestrates container startup. I utilized the BINTARS_WORKER variable to consolidate all workers and their properties in one place.
    - The .env file is used only when PROD is upgraded to containers. PROD masters run as standalone services, which is why master.cfg DEFAULTS are defined for each os.getenv function call.
    - QCOW2-based VMs run on localhost. Since KVM operates on the host system, DEV workloads need to be isolated from PROD. Therefore, the port range for DEV starts from 20000, while PROD defaults to 10000.
    - The maste-private change is for illustration purposes only. The actual change was made manually.

docker-compose/.env

@@ -9,5 +9,16 @@ NGINX_ARTIFACTS_VHOST="ci.mariadb.org"
 ENVIRON="PROD"
 BRANCH="main"
 MASTER_NONLATENT_DOCKERLIBRARY_WORKER="bb-rhel8-docker"
-MASTER_NONLATENT_BINTARS_WORKER="ro-apexis-bbw03-x64"
+MASTER_NONLATENT_BINTARS_WORKERS = '
+    {
+    "ro-apexis-bbw03-x64": {
+        "max_builds": 2,
+        "jobs": 12
+        },
+    "bg-bbw1-x64": {
+        "max_builds": 1,
+        "jobs": 12
+        }
+    }'
+MASTER_NONLATENT_BINTARS_VM_PORT="10000"
 MASTER_NONLATENT_BINTARS_WORKER_PORT="10002"

docker-compose/.env.dev

@@ -10,5 +10,13 @@ NGINX_ARTIFACTS_VHOST="ci.dev.mariadb.org"
 ENVIRON="DEV"
 BRANCH="dev"
 MASTER_NONLATENT_DOCKERLIBRARY_WORKER="bb-rhel9-docker"
-MASTER_NONLATENT_BINTARS_WORKER="ro-apexis-bbw03-x64"
+# MASTER_NONLATENT_BINTARS_WORKERS='["bg-bbw1"]'
+MASTER_NONLATENT_BINTARS_WORKERS = '
+    {
+    "bg-bbw1-x64": {
+        "max_builds": 1,
+        "jobs": 12
+        }
+    }'
+MASTER_NONLATENT_BINTARS_VM_PORT="20000"
 MASTER_NONLATENT_BINTARS_WORKER_PORT="10007"

docker-compose/docker-compose.yaml

@@ -70,7 +70,8 @@ services:
       - BUILDMASTER_WG_IP
       - ENVIRON
       - GALERA_PACKAGES_DIR
-      - MASTER_NONLATENT_BINTARS_WORKER
+      - MASTER_NONLATENT_BINTARS_VM_PORT
+      - MASTER_NONLATENT_BINTARS_WORKERS
       - MASTER_NONLATENT_BINTARS_WORKER_PORT
       - MASTER_NONLATENT_DOCKERLIBRARY_WORKER
       - MASTER_PACKAGES_DIR
@@ -105,7 +106,8 @@ services:
       - BUILDMASTER_WG_IP
       - ENVIRON
       - GALERA_PACKAGES_DIR
-      - MASTER_NONLATENT_BINTARS_WORKER
+      - MASTER_NONLATENT_BINTARS_VM_PORT
+      - MASTER_NONLATENT_BINTARS_WORKERS
       - MASTER_NONLATENT_BINTARS_WORKER_PORT
       - MASTER_NONLATENT_DOCKERLIBRARY_WORKER
       - MASTER_PACKAGES_DIR
@@ -143,7 +145,8 @@ services:
       - BUILDMASTER_WG_IP
       - ENVIRON
       - GALERA_PACKAGES_DIR
-      - MASTER_NONLATENT_BINTARS_WORKER
+      - MASTER_NONLATENT_BINTARS_VM_PORT
+      - MASTER_NONLATENT_BINTARS_WORKERS
       - MASTER_NONLATENT_BINTARS_WORKER_PORT
       - MASTER_NONLATENT_DOCKERLIBRARY_WORKER
       - MASTER_PACKAGES_DIR
@@ -180,7 +183,8 @@ services:
       - BUILDMASTER_WG_IP
       - ENVIRON
       - GALERA_PACKAGES_DIR
-      - MASTER_NONLATENT_BINTARS_WORKER
+      - MASTER_NONLATENT_BINTARS_VM_PORT
+      - MASTER_NONLATENT_BINTARS_WORKERS
       - MASTER_NONLATENT_BINTARS_WORKER_PORT
       - MASTER_NONLATENT_DOCKERLIBRARY_WORKER
       - MASTER_PACKAGES_DIR
@@ -217,7 +221,8 @@ services:
       - BUILDMASTER_WG_IP
       - ENVIRON
       - GALERA_PACKAGES_DIR
-      - MASTER_NONLATENT_BINTARS_WORKER
+      - MASTER_NONLATENT_BINTARS_VM_PORT
+      - MASTER_NONLATENT_BINTARS_WORKERS
       - MASTER_NONLATENT_BINTARS_WORKER_PORT
       - MASTER_NONLATENT_DOCKERLIBRARY_WORKER
       - MASTER_PACKAGES_DIR
@@ -254,7 +259,8 @@ services:
       - BUILDMASTER_WG_IP
       - ENVIRON
       - GALERA_PACKAGES_DIR
-      - MASTER_NONLATENT_BINTARS_WORKER
+      - MASTER_NONLATENT_BINTARS_VM_PORT
+      - MASTER_NONLATENT_BINTARS_WORKERS
       - MASTER_NONLATENT_BINTARS_WORKER_PORT
       - MASTER_NONLATENT_DOCKERLIBRARY_WORKER
       - MASTER_PACKAGES_DIR
@@ -291,7 +297,8 @@ services:
       - BUILDMASTER_WG_IP
       - ENVIRON
       - GALERA_PACKAGES_DIR
-      - MASTER_NONLATENT_BINTARS_WORKER
+      - MASTER_NONLATENT_BINTARS_VM_PORT
+      - MASTER_NONLATENT_BINTARS_WORKERS
       - MASTER_NONLATENT_BINTARS_WORKER_PORT
       - MASTER_NONLATENT_DOCKERLIBRARY_WORKER
       - MASTER_PACKAGES_DIR
@@ -328,7 +335,8 @@ services:
       - BUILDMASTER_WG_IP
       - ENVIRON
       - GALERA_PACKAGES_DIR
-      - MASTER_NONLATENT_BINTARS_WORKER
+      - MASTER_NONLATENT_BINTARS_VM_PORT
+      - MASTER_NONLATENT_BINTARS_WORKERS
       - MASTER_NONLATENT_BINTARS_WORKER_PORT
       - MASTER_NONLATENT_DOCKERLIBRARY_WORKER
       - MASTER_PACKAGES_DIR
@@ -365,7 +373,8 @@ services:
       - BUILDMASTER_WG_IP
       - ENVIRON
       - GALERA_PACKAGES_DIR
-      - MASTER_NONLATENT_BINTARS_WORKER
+      - MASTER_NONLATENT_BINTARS_VM_PORT
+      - MASTER_NONLATENT_BINTARS_WORKERS
       - MASTER_NONLATENT_BINTARS_WORKER_PORT
       - MASTER_NONLATENT_DOCKERLIBRARY_WORKER
       - MASTER_PACKAGES_DIR
@@ -402,7 +411,8 @@ services:
       - BUILDMASTER_WG_IP
       - ENVIRON
       - GALERA_PACKAGES_DIR
-      - MASTER_NONLATENT_BINTARS_WORKER
+      - MASTER_NONLATENT_BINTARS_VM_PORT
+      - MASTER_NONLATENT_BINTARS_WORKERS
       - MASTER_NONLATENT_BINTARS_WORKER_PORT
       - MASTER_NONLATENT_DOCKERLIBRARY_WORKER
       - MASTER_PACKAGES_DIR
@@ -439,7 +449,8 @@ services:
       - BUILDMASTER_WG_IP
       - ENVIRON
       - GALERA_PACKAGES_DIR
-      - MASTER_NONLATENT_BINTARS_WORKER
+      - MASTER_NONLATENT_BINTARS_VM_PORT
+      - MASTER_NONLATENT_BINTARS_WORKERS
       - MASTER_NONLATENT_BINTARS_WORKER_PORT
       - MASTER_NONLATENT_DOCKERLIBRARY_WORKER
       - MASTER_PACKAGES_DIR
@@ -476,7 +487,8 @@ services:
       - BUILDMASTER_WG_IP
       - ENVIRON
       - GALERA_PACKAGES_DIR
-      - MASTER_NONLATENT_BINTARS_WORKER
+      - MASTER_NONLATENT_BINTARS_VM_PORT
+      - MASTER_NONLATENT_BINTARS_WORKERS
       - MASTER_NONLATENT_BINTARS_WORKER_PORT
       - MASTER_NONLATENT_DOCKERLIBRARY_WORKER
       - MASTER_PACKAGES_DIR
@@ -513,7 +525,8 @@ services:
       - BUILDMASTER_WG_IP
       - ENVIRON
       - GALERA_PACKAGES_DIR
-      - MASTER_NONLATENT_BINTARS_WORKER
+      - MASTER_NONLATENT_BINTARS_VM_PORT
+      - MASTER_NONLATENT_BINTARS_WORKERS
       - MASTER_NONLATENT_BINTARS_WORKER_PORT
       - MASTER_NONLATENT_DOCKERLIBRARY_WORKER
       - MASTER_PACKAGES_DIR
@@ -550,7 +563,8 @@ services:
       - BUILDMASTER_WG_IP
       - ENVIRON
       - GALERA_PACKAGES_DIR
-      - MASTER_NONLATENT_BINTARS_WORKER
+      - MASTER_NONLATENT_BINTARS_VM_PORT
+      - MASTER_NONLATENT_BINTARS_WORKERS
       - MASTER_NONLATENT_BINTARS_WORKER_PORT
       - MASTER_NONLATENT_DOCKERLIBRARY_WORKER
       - MASTER_PACKAGES_DIR

master-bintars/master.cfg

@@ -13,6 +13,7 @@ from twisted.internet import defer
 import docker
 import os
 import sys
+import json
 
 sys.path.insert(0, "/srv/buildbot/master")
 sys.setrecursionlimit(10000)
@@ -107,16 +108,33 @@ def mkWorker(name, **kwargs):
 # The 'workers' list defines the set of recognized workers. Each element is
 # a Worker object, specifying a unique worker name and password.  The same
 # worker name and password must be configured on the worker.
+
+# default_workers_bintar exists because master-bintar in production
+# is a standolone service, not a container sourcing .env
+default_workers_bintar = """
+{
+    "ro-apexis-bbw03-x64": {
+        "max_builds": 2,
+        "jobs": 12
+    },
+    "bg-bbw1-x64": {
+        "max_builds": 1,
+        "jobs": 12
+    }
+}
+"""
+workers_bintar = json.loads(os.getenv('MASTER_NONLATENT_BINTARS_WORKERS', default=default_workers_bintar))
+
 c["workers"] = []
 
-c["workers"].append(
-    mkWorker(
-        os.getenv("MASTER_NONLATENT_BINTARS_WORKER", default="ro-apexis-bbw03-x64"),
-        max_builds=1,
-        properties={"jobs": 12},
+for k, v in workers_bintar.items():
+    c["workers"].append(
+        mkWorker(
+            f"{k}",
+            max_builds=v['max_builds'],
+            properties={"jobs": v['jobs']},
+        )
     )
-)
-
 
 ####### FACTORY CODE
 def memory_def():
@@ -326,7 +344,8 @@ builder_definitions = {
     "centos-6": "centos6",
 }
 
-current_port = 10000
+current_port = os.getenv('MASTER_NONLATENT_BINTARS_VM_PORT', default=10000)
+
 for b in builder_definitions:
     for arch in ["amd64", "i386"]:
         if arch == "i386":
@@ -346,9 +365,7 @@ for b in builder_definitions:
     c["builders"].append(
         util.BuilderConfig(
             name=builder_name + "-kvm-" + b + "-bintar",
-            workernames=[
-                os.getenv("MASTER_NONLATENT_BINTARS_WORKER", default="ro-apexis-bbw03-x64")
-            ],
+            workernames=[*workers_bintar],
             factory=f,
         )
     )

master-private.cfg-sample

@@ -56,6 +56,7 @@ private["worker_pass"]= {
     "monty-bbw1-x64": "1234",
     "s390x-rhel8":"1234",
     "libvirt": "1234",
+    "bg-bbw1-x64": "1234", # non-latent worker for master-bintars
 }
 private["docker_workers"]= {
     "amd-bbw1-docker":"tcp://IP_address:port",