Incorporate docker based etcd approach into docker integration tests.

PR 5 for #4144 This PR makes the docker integration tests use containerized etcd. Previously, these relied on M3DB running an embbeded etcd server. There's no inherent need for this, and it opens us up to dependency issues as described in the linked github issue. Note: there are a handful that require multiple servers; these are currently "skipped" (commented). I intend to bring those back at a later date.. commit-id:e67a5172
m3db · Aug 29, 2022 · e0944c8 · e0944c8
1 parent 7b49976
commit e0944c8
Show file tree

Hide file tree

Showing 58 changed files with 386 additions and 189 deletions.
diff --git a/scripts/docker-integration-tests/aggregator/docker-compose.yml b/scripts/docker-integration-tests/aggregator/docker-compose.yml
@@ -1,17 +1,27 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+    networks:
+      - backend
   dbnode01:
     expose:
       - "9000-9004"
-      - "2379-2380"
       - "7201"
     ports:
       - "0.0.0.0:9000-9004:9000-9004"
-      - "0.0.0.0:2379-2380:2379-2380"
       - "0.0.0.0:7201:7201"
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    depends_on:
+      - etcd
   m3coordinator01:
     expose:
       - "7202"
@@ -26,6 +36,8 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml"
+    depends_on:
+      - etcd
   m3aggregator01:
     expose:
       - "6001"
@@ -38,6 +50,8 @@ services:
     image: "m3aggregator_integration:${REVISION}"
     volumes:
       - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml"
+    depends_on:
+      - etcd
   m3aggregator02:
     networks:
       - backend
@@ -46,5 +60,7 @@ services:
     image: "m3aggregator_integration:${REVISION}"
     volumes:
       - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/aggregator/m3aggregator.yml b/scripts/docker-integration-tests/aggregator/m3aggregator.yml
@@ -38,7 +38,7 @@ kvClient:
         autoSyncInterval: 10m
         dialTimeout: 1m
         endpoints:
-          - dbnode01:2379
+          - etcd:2379
 
 runtimeOptions:
   kvConfig:

diff --git a/scripts/docker-integration-tests/aggregator/m3coordinator.yml b/scripts/docker-integration-tests/aggregator/m3coordinator.yml
@@ -24,7 +24,7 @@ clusters:
               autoSyncInterval: 10m
               dialTimeout: 1m
               endpoints:
-                - dbnode01:2379
+                - etcd:2379
 
 downsample:
   rules:

diff --git a/scripts/docker-integration-tests/aggregator/test.sh b/scripts/docker-integration-tests/aggregator/test.sh
@@ -14,6 +14,8 @@ echo "Pull containers required for test"
 docker pull $PROMREMOTECLI_IMAGE
 docker pull $JQ_IMAGE
 
+docker-compose -f ${COMPOSE_FILE} up -d etcd
+
 echo "Run m3dbnode"
 docker-compose -f ${COMPOSE_FILE} up -d dbnode01
 

diff --git a/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml b/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml
@@ -1,17 +1,23 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+    networks:
+      - backend
   dbnode01:
     expose:
       - "9000-9004"
-      - "2379-2380"
       - "7201"
     ports:
       - "0.0.0.0:9000-9004:9000-9004"
-      - "0.0.0.0:2379-2380:2379-2380"
       - "0.0.0.0:7201:7201"
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    depends_on:
+      - etcd
   m3coordinator01:
     expose:
       - "7202"
@@ -26,6 +32,8 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml"
+    depends_on:
+      - etcd
   m3aggregator01:
     expose:
       - "6001"
@@ -38,6 +46,8 @@ services:
     image: "m3aggregator_integration:${REVISION}"
     volumes:
       - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml"
+    depends_on:
+      - etcd
   m3aggregator02:
     networks:
       - backend
@@ -46,5 +56,7 @@ services:
     image: "m3aggregator_integration:${REVISION}"
     volumes:
       - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml
@@ -57,7 +57,7 @@ kvClient:
     etcdClusters:
       - zone: embedded
         endpoints:
-          - dbnode01:2379
+          - etcd:2379
 
 runtimeOptions:
   kvConfig:

diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml
@@ -22,7 +22,7 @@ clusters:
           etcdClusters:
             - zone: embedded
               endpoints:
-                - dbnode01:2379
+                - etcd:2379
 
 downsample:
   remoteAggregator:

diff --git a/scripts/docker-integration-tests/aggregator_legacy/test.sh b/scripts/docker-integration-tests/aggregator_legacy/test.sh
@@ -7,6 +7,9 @@ REVISION=$(git rev-parse HEAD)
 COMPOSE_FILE="$M3_PATH"/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml
 export REVISION
 
+echo "Run etcd"
+docker-compose -f ${COMPOSE_FILE} up -d etcd
+
 echo "Run m3dbnode"
 docker-compose -f ${COMPOSE_FILE} up -d dbnode01
 

diff --git a/scripts/docker-integration-tests/carbon/docker-compose.yml b/scripts/docker-integration-tests/carbon/docker-compose.yml
@@ -1,15 +1,25 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+    networks:
+      - backend
   dbnode01:
     expose:
       - "9000-9004"
-      - "2379-2380"
     ports:
       - "0.0.0.0:9000-9004:9000-9004"
-      - "0.0.0.0:2379-2380:2379-2380"
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    depends_on:
+      - etcd
   coordinator01:
     expose:
       - "7201"
@@ -24,5 +34,7 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./:/etc/m3coordinator/"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/carbon/m3coordinator.yml b/scripts/docker-integration-tests/carbon/m3coordinator.yml
@@ -9,7 +9,7 @@ clusters:
           etcdClusters:
             - zone: embedded
               endpoints:
-                - dbnode01:2379
+                - etcd:2379
 
 carbon:
   findResultsIncludeBothExpandableAndLeaf: true

diff --git a/scripts/docker-integration-tests/carbon/test.sh b/scripts/docker-integration-tests/carbon/test.sh
@@ -10,8 +10,7 @@ EXPECTED_PATH=$SCRIPT_PATH/expected
 export REVISION
 
 echo "Run m3dbnode and m3coordinator containers"
-docker-compose -f ${COMPOSE_FILE} up -d dbnode01
-docker-compose -f ${COMPOSE_FILE} up -d coordinator01
+docker-compose -f ${COMPOSE_FILE} up -d
 
 # Think of this as a defer func() in golang
 METRIC_EMIT_PID="-1"
@@ -152,7 +151,7 @@ ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "wait_carbon_values_accum
 
 # Now test the max datapoints behavior using max of four datapoints (4x 5s resolution = 20s)
 end=$(date +%s)
-start=$(($end-20)) 
+start=$(($end-20))
 # 1. no max datapoints set, should not adjust number of datapoints coming back
 ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'stat.already-aggregated.foo' 42 $start $end"
 # 2. max datapoints with LTTB, should be an existing value (i.e. 42)

diff --git a/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml b/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml
@@ -1,15 +1,25 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+    networks:
+      - backend
   dbnode01:
     expose:
       - "9000-9004"
-      - "2379-2380"
     ports:
       - "0.0.0.0:9000-9004:9000-9004"
-      - "0.0.0.0:2379-2380:2379-2380"
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    depends_on:
+      - etcd
   coordinator01:
     expose:
       - "7201"
@@ -24,5 +34,7 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./:/etc/m3coordinator/"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml
@@ -13,4 +13,4 @@ clusters:
           etcdClusters:
             - zone: embedded
               endpoints:
-                - dbnode01:2379
+                - etcd:2379
diff --git a/scripts/docker-integration-tests/cold_writes_simple/test.sh b/scripts/docker-integration-tests/cold_writes_simple/test.sh
@@ -9,8 +9,9 @@ COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml
 export REVISION
 
 echo "Run m3dbnode and m3coordinator containers"
-docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode01
-docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01
+docker-compose -f "${COMPOSE_FILE}" up -d --renew-anon-volumes # etcd
+#docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode01
+#docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01
 
 # Think of this as a defer func() in golang
 function defer {

diff --git a/scripts/docker-integration-tests/common.sh b/scripts/docker-integration-tests/common.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
 # Retries a command a configurable number of times with backoff.
 #
 # The retry count is given by ATTEMPTS (default 3), the initial backoff

diff --git a/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml b/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml
@@ -1,15 +1,25 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+    networks:
+      - backend
   dbnode01:
     expose:
       - "9000-9004"
-      - "2379-2380"
     ports:
       - "0.0.0.0:9000-9004:9000-9004"
-      - "0.0.0.0:2379-2380:2379-2380"
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    depends_on:
+      - etcd
   coordinator01:
     expose:
       - "7201"
@@ -24,5 +34,7 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./:/etc/m3coordinator/"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml
@@ -9,7 +9,7 @@ clusters:
           etcdClusters:
             - zone: embedded
               endpoints:
-                - dbnode01:2379
+                - etcd:2379
 
 downsample:
   rules:

diff --git a/scripts/docker-integration-tests/coordinator_config_rules/test.sh b/scripts/docker-integration-tests/coordinator_config_rules/test.sh
@@ -16,8 +16,7 @@ docker pull $PROMREMOTECLI_IMAGE
 docker pull $JQ_IMAGE
 
 echo "Run m3dbnode and m3coordinator containers"
-docker-compose -f ${COMPOSE_FILE} up -d dbnode01
-docker-compose -f ${COMPOSE_FILE} up -d coordinator01
+docker-compose -f ${COMPOSE_FILE} up -d
 
 # Think of this as a defer func() in golang
 function defer {