diff --git a/scripts/docker-integration-tests/aggregator/docker-compose.yml b/scripts/docker-integration-tests/aggregator/docker-compose.yml index c93b41ee25..355e1895c3 100644 --- a/scripts/docker-integration-tests/aggregator/docker-compose.yml +++ b/scripts/docker-integration-tests/aggregator/docker-compose.yml @@ -1,17 +1,27 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" - "7201" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" - "0.0.0.0:7201:7201" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd m3coordinator01: expose: - "7202" @@ -26,6 +36,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd m3aggregator01: expose: - "6001" @@ -38,6 +50,8 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd m3aggregator02: networks: - backend @@ -46,5 +60,7 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/aggregator/m3aggregator.yml b/scripts/docker-integration-tests/aggregator/m3aggregator.yml index c2f26330bd..1aa1c7c091 100644 --- a/scripts/docker-integration-tests/aggregator/m3aggregator.yml +++ b/scripts/docker-integration-tests/aggregator/m3aggregator.yml @@ -38,7 +38,7 @@ kvClient: autoSyncInterval: 10m dialTimeout: 1m endpoints: - - dbnode01:2379 + - etcd:2379 runtimeOptions: kvConfig: diff --git a/scripts/docker-integration-tests/aggregator/m3coordinator.yml b/scripts/docker-integration-tests/aggregator/m3coordinator.yml index dfc757b8c4..a1e44edda4 100644 --- a/scripts/docker-integration-tests/aggregator/m3coordinator.yml +++ b/scripts/docker-integration-tests/aggregator/m3coordinator.yml @@ -24,7 +24,7 @@ clusters: autoSyncInterval: 10m dialTimeout: 1m endpoints: - - dbnode01:2379 + - etcd:2379 downsample: rules: diff --git a/scripts/docker-integration-tests/aggregator/test.sh b/scripts/docker-integration-tests/aggregator/test.sh index a65b912c8a..a5b44122c8 100755 --- a/scripts/docker-integration-tests/aggregator/test.sh +++ b/scripts/docker-integration-tests/aggregator/test.sh @@ -14,6 +14,8 @@ echo "Pull containers required for test" docker pull $PROMREMOTECLI_IMAGE docker pull $JQ_IMAGE +docker-compose -f ${COMPOSE_FILE} up -d etcd + echo "Run m3dbnode" docker-compose -f ${COMPOSE_FILE} up -d dbnode01 diff --git a/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml b/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml index c93b41ee25..ca6606f627 100644 --- a/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml +++ b/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml @@ -1,17 +1,23 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" - "7201" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" - "0.0.0.0:7201:7201" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd m3coordinator01: expose: - "7202" @@ -26,6 +32,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd m3aggregator01: expose: - "6001" @@ -38,6 +46,8 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd m3aggregator02: networks: - backend @@ -46,5 +56,7 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml index 569ea9a9c2..e10c7cbe96 100644 --- a/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml +++ b/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml @@ -57,7 +57,7 @@ kvClient: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 runtimeOptions: kvConfig: diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml index 281662faca..d15fd1a78c 100644 --- a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml +++ b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml @@ -22,7 +22,7 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 downsample: remoteAggregator: diff --git a/scripts/docker-integration-tests/aggregator_legacy/test.sh b/scripts/docker-integration-tests/aggregator_legacy/test.sh index e101eae186..7f5050005b 100755 --- a/scripts/docker-integration-tests/aggregator_legacy/test.sh +++ b/scripts/docker-integration-tests/aggregator_legacy/test.sh @@ -7,6 +7,9 @@ REVISION=$(git rev-parse HEAD) COMPOSE_FILE="$M3_PATH"/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml export REVISION +echo "Run etcd" +docker-compose -f ${COMPOSE_FILE} up -d etcd + echo "Run m3dbnode" docker-compose -f ${COMPOSE_FILE} up -d dbnode01 diff --git a/scripts/docker-integration-tests/carbon/docker-compose.yml b/scripts/docker-integration-tests/carbon/docker-compose.yml index 53a28f0b88..e6b97c2e0f 100644 --- a/scripts/docker-integration-tests/carbon/docker-compose.yml +++ b/scripts/docker-integration-tests/carbon/docker-compose.yml @@ -1,15 +1,25 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -24,5 +34,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/carbon/m3coordinator.yml b/scripts/docker-integration-tests/carbon/m3coordinator.yml index 7f33d8701c..b069fbf7b6 100644 --- a/scripts/docker-integration-tests/carbon/m3coordinator.yml +++ b/scripts/docker-integration-tests/carbon/m3coordinator.yml @@ -9,7 +9,7 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 carbon: findResultsIncludeBothExpandableAndLeaf: true diff --git a/scripts/docker-integration-tests/carbon/test.sh b/scripts/docker-integration-tests/carbon/test.sh index f1c499a71e..6c811cb3b8 100755 --- a/scripts/docker-integration-tests/carbon/test.sh +++ b/scripts/docker-integration-tests/carbon/test.sh @@ -10,8 +10,7 @@ EXPECTED_PATH=$SCRIPT_PATH/expected export REVISION echo "Run m3dbnode and m3coordinator containers" -docker-compose -f ${COMPOSE_FILE} up -d dbnode01 -docker-compose -f ${COMPOSE_FILE} up -d coordinator01 +docker-compose -f ${COMPOSE_FILE} up -d # Think of this as a defer func() in golang METRIC_EMIT_PID="-1" @@ -152,7 +151,7 @@ ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "wait_carbon_values_accum # Now test the max datapoints behavior using max of four datapoints (4x 5s resolution = 20s) end=$(date +%s) -start=$(($end-20)) +start=$(($end-20)) # 1. no max datapoints set, should not adjust number of datapoints coming back ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'stat.already-aggregated.foo' 42 $start $end" # 2. max datapoints with LTTB, should be an existing value (i.e. 42) diff --git a/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml b/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml index 53a28f0b88..e6b97c2e0f 100644 --- a/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml +++ b/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml @@ -1,15 +1,25 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -24,5 +34,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml index cc33cf4021..dc075a01fe 100644 --- a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml +++ b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml @@ -13,4 +13,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 diff --git a/scripts/docker-integration-tests/cold_writes_simple/test.sh b/scripts/docker-integration-tests/cold_writes_simple/test.sh index 999ef1b20d..0a4642a05b 100755 --- a/scripts/docker-integration-tests/cold_writes_simple/test.sh +++ b/scripts/docker-integration-tests/cold_writes_simple/test.sh @@ -9,8 +9,9 @@ COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml export REVISION echo "Run m3dbnode and m3coordinator containers" -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode01 -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01 +docker-compose -f "${COMPOSE_FILE}" up -d --renew-anon-volumes # etcd +#docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode01 +#docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01 # Think of this as a defer func() in golang function defer { diff --git a/scripts/docker-integration-tests/common.sh b/scripts/docker-integration-tests/common.sh index 918ab3b76a..ee8651584f 100644 --- a/scripts/docker-integration-tests/common.sh +++ b/scripts/docker-integration-tests/common.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + # Retries a command a configurable number of times with backoff. # # The retry count is given by ATTEMPTS (default 3), the initial backoff diff --git a/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml b/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml index 53a28f0b88..e6b97c2e0f 100644 --- a/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml +++ b/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml @@ -1,15 +1,25 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -24,5 +34,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml index 952c9a7ddf..3ca941aa8d 100644 --- a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml +++ b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml @@ -9,7 +9,7 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 downsample: rules: diff --git a/scripts/docker-integration-tests/coordinator_config_rules/test.sh b/scripts/docker-integration-tests/coordinator_config_rules/test.sh index a1590983c8..48dce08870 100755 --- a/scripts/docker-integration-tests/coordinator_config_rules/test.sh +++ b/scripts/docker-integration-tests/coordinator_config_rules/test.sh @@ -16,8 +16,7 @@ docker pull $PROMREMOTECLI_IMAGE docker pull $JQ_IMAGE echo "Run m3dbnode and m3coordinator containers" -docker-compose -f ${COMPOSE_FILE} up -d dbnode01 -docker-compose -f ${COMPOSE_FILE} up -d coordinator01 +docker-compose -f ${COMPOSE_FILE} up -d # Think of this as a defer func() in golang function defer { diff --git a/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml b/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml index ee8207bd26..f75413892f 100644 --- a/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml +++ b/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml @@ -1,5 +1,15 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend coordinator01: expose: - "7201" @@ -10,33 +20,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml" - etcd01: - expose: - - "2379-2380" - ports: - - "0.0.0.0:2379-2380:2379-2380" - networks: - - backend - image: quay.io/coreos/etcd:v3.4.3 - command: - - "etcd" - - "--name" - - "etcd01" - - "--listen-peer-urls" - - "http://0.0.0.0:2380" - - "--listen-client-urls" - - "http://0.0.0.0:2379" - - "--advertise-client-urls" - - "http://etcd01:2379" - - "--initial-cluster-token" - - "etcd-cluster-1" - - "--initial-advertise-peer-urls" - - "http://etcd01:2380" - - "--initial-cluster" - - "etcd01=http://etcd01:2380" - - "--initial-cluster-state" - - "new" - - "--data-dir" - - "/var/lib/etcd" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml index c3c08c0104..6a261ecf26 100644 --- a/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml +++ b/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml @@ -23,7 +23,7 @@ clusterManagement: etcdClusters: - zone: embedded endpoints: - - etcd01:2379 + - etcd:2379 tagOptions: idScheme: quoted diff --git a/scripts/docker-integration-tests/coordinator_noop/test.sh b/scripts/docker-integration-tests/coordinator_noop/test.sh index 738ea863d0..0f91ec91da 100755 --- a/scripts/docker-integration-tests/coordinator_noop/test.sh +++ b/scripts/docker-integration-tests/coordinator_noop/test.sh @@ -9,8 +9,7 @@ COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml export REVISION echo "Run coordinator with no etcd" -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01 -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes etcd01 +docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes function defer { docker-compose -f ${COMPOSE_FILE} down || echo "unable to shutdown containers" # CI fails to stop all containers sometimes diff --git a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml index e155eb5334..323289a1a6 100644 --- a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml +++ b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml @@ -1,5 +1,15 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode01: expose: - "9000-9004" @@ -14,33 +24,7 @@ services: - M3DB_HOST_ID=dbnode01 volumes: - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" - etcd01: - expose: - - "2379-2380" - ports: - - "0.0.0.0:2379-2380:2379-2380" - networks: - - backend - image: quay.io/coreos/etcd:v3.4.3 - command: - - "etcd" - - "--name" - - "etcd01" - - "--listen-peer-urls" - - "http://0.0.0.0:2380" - - "--listen-client-urls" - - "http://0.0.0.0:2379" - - "--advertise-client-urls" - - "http://etcd01:2379" - - "--initial-cluster-token" - - "etcd-cluster-1" - - "--initial-advertise-peer-urls" - - "http://etcd01:2380" - - "--initial-cluster" - - "etcd01=http://etcd01:2380" - - "--initial-cluster-state" - - "new" - - "--data-dir" - - "/var/lib/etcd" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/docker-compose-etcd.yml b/scripts/docker-integration-tests/docker-compose-etcd.yml new file mode 100644 index 0000000000..90af19b550 --- /dev/null +++ b/scripts/docker-integration-tests/docker-compose-etcd.yml @@ -0,0 +1,15 @@ +version: "3.5" +services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend + +networks: + backend: diff --git a/scripts/docker-integration-tests/m3coordinator.Dockerfile b/scripts/docker-integration-tests/m3coordinator.Dockerfile index 0319613977..489030dbea 100644 --- a/scripts/docker-integration-tests/m3coordinator.Dockerfile +++ b/scripts/docker-integration-tests/m3coordinator.Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer="The M3DB Authors " RUN mkdir -p /bin RUN mkdir -p /etc/m3coordinator ADD ./m3coordinator /bin/ -ADD ./m3coordinator-local-etcd.yml /etc/m3coordinator/m3coordinator.yml +ADD ./m3coordinator-local-docker-etcd.yml /etc/m3coordinator/m3coordinator.yml EXPOSE 7201/tcp 7203/tcp diff --git a/scripts/docker-integration-tests/m3dbnode.Dockerfile b/scripts/docker-integration-tests/m3dbnode.Dockerfile index a352ad4bf5..6d430fe803 100644 --- a/scripts/docker-integration-tests/m3dbnode.Dockerfile +++ b/scripts/docker-integration-tests/m3dbnode.Dockerfile @@ -4,9 +4,9 @@ LABEL maintainer="The M3DB Authors " RUN mkdir -p /bin RUN mkdir -p /etc/m3dbnode ADD ./m3dbnode /bin/ -ADD ./m3dbnode-local-etcd.yml /etc/m3dbnode/m3dbnode.yml +ADD ./m3dbnode-local-docker-etcd.yml /etc/m3dbnode/m3dbnode.yml -EXPOSE 2379/tcp 2380/tcp 7201/tcp 7203/tcp 9000-9004/tcp +EXPOSE 7201/tcp 7203/tcp 9000-9004/tcp ENV PANIC_ON_INVARIANT_VIOLATED=true diff --git a/scripts/docker-integration-tests/multi_cluster_write/docker-compose.yml b/scripts/docker-integration-tests/multi_cluster_write/docker-compose.yml index b5b52d922d..5d9e35cfcb 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/docker-compose.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/docker-compose.yml @@ -1,5 +1,15 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend cluster_a_dbnode01: expose: - "9000-9004" @@ -14,6 +24,8 @@ services: - M3DB_HOST_ID=cluster_a_m3db_local_1 volumes: - "./m3dbnode-cluster-a.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd cluster_a_dbnode02: expose: - "9000-9004" @@ -28,6 +40,8 @@ services: - M3DB_HOST_ID=cluster_a_m3db_local_2 volumes: - "./m3dbnode-cluster-a.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd cluster_a_coordinator01: expose: - "7201" @@ -42,6 +56,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator-cluster-a.yml/:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd cluster_b_dbnode01: expose: - "9000-9004" @@ -56,6 +72,8 @@ services: - M3DB_HOST_ID=cluster_b_m3db_local_1 volumes: - "./m3dbnode-cluster-b.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd cluster_b_dbnode02: expose: - "9000-9004" @@ -70,6 +88,8 @@ services: - M3DB_HOST_ID=cluster_b_m3db_local_2 volumes: - "./m3dbnode-cluster-b.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd cluster_b_coordinator01: expose: - "7201" @@ -84,5 +104,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator-cluster-b.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml index e393f2ef22..8fb9bb136a 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-a.yml @@ -13,4 +13,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - cluster_a_dbnode01:2379 + - cluster_a_etcd:2379 diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml index 7933a84078..facf8fa580 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3coordinator-cluster-b.yml @@ -13,4 +13,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - cluster_b_dbnode01:2379 + - cluster_b_etcd:2379 diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml index 13aac06336..052e908401 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-a.yml @@ -14,7 +14,7 @@ db: etcdClusters: - zone: embedded endpoints: - - cluster_a_dbnode01:2379 + - cluster_a_etcd:2379 - service: env: default_env zone: embedded @@ -23,7 +23,7 @@ db: etcdClusters: - zone: embedded endpoints: - - cluster_b_dbnode01:2379 + - cluster_b_etcd:2379 async: true discovery: @@ -36,7 +36,7 @@ db: etcdClusters: - zone: embedded endpoints: - - cluster_a_dbnode01:2379 + - cluster_a_etcd:2379 seedNodes: initialCluster: - hostID: cluster_a_m3db_local_1 diff --git a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml index 9e59d22898..d37ae5171a 100644 --- a/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/multi_cluster_write/m3dbnode-cluster-b.yml @@ -13,7 +13,7 @@ db: etcdClusters: - zone: embedded endpoints: - - cluster_b_dbnode01:2379 + - cluster_b_etcd:2379 seedNodes: initialCluster: - hostID: cluster_b_m3db_local_1 diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml b/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml index 6c38ae39de..7bf3fc841f 100644 --- a/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml +++ b/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml @@ -1,5 +1,15 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend m3aggregator01: expose: - "6001" @@ -12,6 +22,8 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd m3aggregator02: expose: - "6002" @@ -24,6 +36,8 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd m3coordinator01: expose: - "7202" @@ -34,6 +48,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd coordinatoradmin: expose: - "7201" @@ -44,6 +60,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator-admin.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd prometheusraw: expose: - "9090" @@ -60,6 +78,8 @@ services: - "--web.console.libraries=/usr/share/prometheus/console_libraries" - "--web.console.templates=/usr/share/prometheus/consoles" - "--enable-feature=remote-write-receiver" + depends_on: + - etcd prometheusagg: expose: - "9091" @@ -76,33 +96,7 @@ services: - "--web.console.libraries=/usr/share/prometheus/console_libraries" - "--web.console.templates=/usr/share/prometheus/consoles" - "--enable-feature=remote-write-receiver" - etcd01: - expose: - - "2379-2380" - ports: - - "0.0.0.0:2379-2380:2379-2380" - networks: - - backend - image: quay.io/coreos/etcd:v3.4.3 - command: - - "etcd" - - "--name" - - "etcd01" - - "--listen-peer-urls" - - "http://0.0.0.0:2380" - - "--listen-client-urls" - - "http://0.0.0.0:2379" - - "--advertise-client-urls" - - "http://etcd01:2379" - - "--initial-cluster-token" - - "etcd-cluster-1" - - "--initial-advertise-peer-urls" - - "http://etcd01:2380" - - "--initial-cluster" - - "etcd01=http://etcd01:2380" - - "--initial-cluster-state" - - "new" - - "--data-dir" - - "/var/lib/etcd" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml b/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml index 1d77b0a035..37ac3f2aa4 100644 --- a/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml +++ b/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml @@ -40,7 +40,7 @@ kvClient: etcdClusters: - zone: embedded endpoints: - - etcd01:2379 + - etcd:2379 runtimeOptions: kvConfig: diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml index c3c08c0104..6a261ecf26 100644 --- a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml +++ b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml @@ -23,7 +23,7 @@ clusterManagement: etcdClusters: - zone: embedded endpoints: - - etcd01:2379 + - etcd:2379 tagOptions: idScheme: quoted diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml index d6c54c8430..935bc8ed5e 100644 --- a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml +++ b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml @@ -36,7 +36,7 @@ clusterManagement: etcdClusters: - zone: embedded endpoints: - - etcd01:2379 + - etcd:2379 tagOptions: idScheme: quoted diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/test.sh b/scripts/docker-integration-tests/prom_remote_write_backend/test.sh index 208bb88565..7ba4c0435e 100755 --- a/scripts/docker-integration-tests/prom_remote_write_backend/test.sh +++ b/scripts/docker-integration-tests/prom_remote_write_backend/test.sh @@ -20,7 +20,7 @@ docker pull $PROMREMOTECLI_IMAGE trap 'cleanup ${COMPOSE_FILE} ${TEST_SUCCESS}' EXIT echo "Run ETCD" -docker-compose -f "${COMPOSE_FILE}" up -d etcd01 +docker-compose -f "${COMPOSE_FILE}" up -d etcd echo "Run Coordinator in Admin mode" docker-compose -f "${COMPOSE_FILE}" up -d coordinatoradmin diff --git a/scripts/docker-integration-tests/prometheus/docker-compose.yml b/scripts/docker-integration-tests/prometheus/docker-compose.yml index 03b839832b..a859edea16 100644 --- a/scripts/docker-integration-tests/prometheus/docker-compose.yml +++ b/scripts/docker-integration-tests/prometheus/docker-compose.yml @@ -1,17 +1,27 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" volumes: - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -24,6 +34,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd prometheus01: expose: - "9090" @@ -34,5 +46,7 @@ services: image: prom/prometheus:latest volumes: - "./prometheus.yml:/etc/prometheus/prometheus.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/prometheus/m3coordinator.yml b/scripts/docker-integration-tests/prometheus/m3coordinator.yml index 79f849a151..7ebe2792b3 100644 --- a/scripts/docker-integration-tests/prometheus/m3coordinator.yml +++ b/scripts/docker-integration-tests/prometheus/m3coordinator.yml @@ -13,7 +13,7 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 query: restrictTags: diff --git a/scripts/docker-integration-tests/prometheus_replication/docker-compose.yml b/scripts/docker-integration-tests/prometheus_replication/docker-compose.yml index 333ee79238..730e4cdd7f 100644 --- a/scripts/docker-integration-tests/prometheus_replication/docker-compose.yml +++ b/scripts/docker-integration-tests/prometheus_replication/docker-compose.yml @@ -1,15 +1,25 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -22,16 +32,20 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator01.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd + - dbnode01 + - dbnode02 dbnode02: expose: - "19000-19004" - - "12379-12380" ports: - "0.0.0.0:19000-19004:9000-9004" - - "0.0.0.0:12379-12380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator02: expose: - "17201" @@ -44,5 +58,9 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator02.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd + - dbnode01 + - dbnode02 networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml b/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml index 426653f6cc..4b323bdd1e 100644 --- a/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml +++ b/scripts/docker-integration-tests/prometheus_replication/m3coordinator01.yml @@ -14,4 +14,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 diff --git a/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml b/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml index d7a81d02b4..122077927b 100644 --- a/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml +++ b/scripts/docker-integration-tests/prometheus_replication/m3coordinator02.yml @@ -9,4 +9,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode02:2379 + - etcd:2379 diff --git a/scripts/docker-integration-tests/prometheus_replication/test.sh b/scripts/docker-integration-tests/prometheus_replication/test.sh index 3d3c72407f..8c240a3c0c 100755 --- a/scripts/docker-integration-tests/prometheus_replication/test.sh +++ b/scripts/docker-integration-tests/prometheus_replication/test.sh @@ -15,13 +15,14 @@ docker pull $PROMREMOTECLI_IMAGE docker pull $JQ_IMAGE echo "Run m3dbnode and m3coordinator containers" -docker-compose -f ${COMPOSE_FILE} up -d dbnode01 -docker-compose -f ${COMPOSE_FILE} up -d dbnode02 -docker-compose -f ${COMPOSE_FILE} up -d coordinator01 -docker-compose -f ${COMPOSE_FILE} up -d coordinator02 +docker-compose -f ${COMPOSE_FILE} up -d +#docker-compose -f ${COMPOSE_FILE} up -d dbnode02 +#docker-compose -f ${COMPOSE_FILE} up -d coordinator01 +#docker-compose -f ${COMPOSE_FILE} up -d coordinator02 function defer { - docker-compose -f ${COMPOSE_FILE} down || echo "unable to shutdown containers" # CI fails to stop all containers sometimes + : +# docker-compose -f ${COMPOSE_FILE} down || echo "unable to shutdown containers" # CI fails to stop all containers sometimes } trap defer EXIT diff --git a/scripts/docker-integration-tests/query_fanout/docker-compose.yml b/scripts/docker-integration-tests/query_fanout/docker-compose.yml index c5cba80180..7a9b82e9fb 100644 --- a/scripts/docker-integration-tests/query_fanout/docker-compose.yml +++ b/scripts/docker-integration-tests/query_fanout/docker-compose.yml @@ -1,5 +1,15 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode-cluster-a: expose: - "9000-9004" @@ -10,6 +20,8 @@ services: networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator-cluster-a: expose: - "7201" @@ -24,6 +36,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator-cluster-a.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd dbnode-cluster-b: expose: - "9000-9004" @@ -34,6 +48,8 @@ services: networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator-cluster-b: expose: - "7201" @@ -48,6 +64,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator-cluster-b.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd dbnode-cluster-c: expose: - "9000-9004" @@ -58,6 +76,8 @@ services: networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator-cluster-c: expose: - "7201" @@ -72,5 +92,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator-cluster-c.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/repair/docker-compose.yml b/scripts/docker-integration-tests/repair/docker-compose.yml index b91284adcf..58ec4d27fb 100644 --- a/scripts/docker-integration-tests/repair/docker-compose.yml +++ b/scripts/docker-integration-tests/repair/docker-compose.yml @@ -1,9 +1,18 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9012:9002" - "0.0.0.0:9013:9003" @@ -14,10 +23,11 @@ services: - M3DB_HOST_ID=m3db_local_1 volumes: - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd dbnode02: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9022:9002" - "0.0.0.0:9023:9003" @@ -28,6 +38,8 @@ services: - M3DB_HOST_ID=m3db_local_2 volumes: - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -42,5 +54,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/repair/m3coordinator.yml b/scripts/docker-integration-tests/repair/m3coordinator.yml index cc33cf4021..dc075a01fe 100644 --- a/scripts/docker-integration-tests/repair/m3coordinator.yml +++ b/scripts/docker-integration-tests/repair/m3coordinator.yml @@ -13,4 +13,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 diff --git a/scripts/docker-integration-tests/repair/m3dbnode.yml b/scripts/docker-integration-tests/repair/m3dbnode.yml index 29dcc22d4b..62f3e88cf8 100644 --- a/scripts/docker-integration-tests/repair/m3dbnode.yml +++ b/scripts/docker-integration-tests/repair/m3dbnode.yml @@ -13,12 +13,7 @@ db: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 - seedNodes: - initialCluster: - - hostID: m3db_local_1 - endpoint: http://dbnode01:2380 - + - etcd:2379 # Enable repairs. repair: enabled: true diff --git a/scripts/docker-integration-tests/repair/test.sh b/scripts/docker-integration-tests/repair/test.sh index ee32d632a9..eff434dc7f 100755 --- a/scripts/docker-integration-tests/repair/test.sh +++ b/scripts/docker-integration-tests/repair/test.sh @@ -9,6 +9,7 @@ COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml export REVISION echo "Run m3dbnode and m3coordinator containers" +docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes etcd docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode01 docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode02 docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01 diff --git a/scripts/docker-integration-tests/replication/docker-compose.yml b/scripts/docker-integration-tests/replication/docker-compose.yml index b5b52d922d..5d9e35cfcb 100644 --- a/scripts/docker-integration-tests/replication/docker-compose.yml +++ b/scripts/docker-integration-tests/replication/docker-compose.yml @@ -1,5 +1,15 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend cluster_a_dbnode01: expose: - "9000-9004" @@ -14,6 +24,8 @@ services: - M3DB_HOST_ID=cluster_a_m3db_local_1 volumes: - "./m3dbnode-cluster-a.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd cluster_a_dbnode02: expose: - "9000-9004" @@ -28,6 +40,8 @@ services: - M3DB_HOST_ID=cluster_a_m3db_local_2 volumes: - "./m3dbnode-cluster-a.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd cluster_a_coordinator01: expose: - "7201" @@ -42,6 +56,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator-cluster-a.yml/:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd cluster_b_dbnode01: expose: - "9000-9004" @@ -56,6 +72,8 @@ services: - M3DB_HOST_ID=cluster_b_m3db_local_1 volumes: - "./m3dbnode-cluster-b.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd cluster_b_dbnode02: expose: - "9000-9004" @@ -70,6 +88,8 @@ services: - M3DB_HOST_ID=cluster_b_m3db_local_2 volumes: - "./m3dbnode-cluster-b.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd cluster_b_coordinator01: expose: - "7201" @@ -84,5 +104,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator-cluster-b.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml b/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml index e393f2ef22..8fb9bb136a 100644 --- a/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml +++ b/scripts/docker-integration-tests/replication/m3coordinator-cluster-a.yml @@ -13,4 +13,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - cluster_a_dbnode01:2379 + - cluster_a_etcd:2379 diff --git a/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml b/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml index 7933a84078..facf8fa580 100644 --- a/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml +++ b/scripts/docker-integration-tests/replication/m3coordinator-cluster-b.yml @@ -13,4 +13,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - cluster_b_dbnode01:2379 + - cluster_b_etcd:2379 diff --git a/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml b/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml index 52e75125c5..7039d9ca27 100644 --- a/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml +++ b/scripts/docker-integration-tests/replication/m3dbnode-cluster-a.yml @@ -13,7 +13,7 @@ db: etcdClusters: - zone: embedded endpoints: - - cluster_a_dbnode01:2379 + - cluster_a_etcd:2379 seedNodes: initialCluster: - hostID: cluster_a_m3db_local_1 @@ -40,4 +40,4 @@ db: etcdClusters: - zone: embedded endpoints: - - cluster_b_dbnode01:2379 + - cluster_b_etcd:2379 diff --git a/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml b/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml index 2d04a09719..b6aab43943 100644 --- a/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml +++ b/scripts/docker-integration-tests/replication/m3dbnode-cluster-b.yml @@ -13,7 +13,7 @@ db: etcdClusters: - zone: embedded endpoints: - - cluster_b_dbnode01:2379 + - cluster_b_etcd:2379 seedNodes: initialCluster: - hostID: cluster_b_m3db_local_1 @@ -42,4 +42,4 @@ db: etcdClusters: - zone: embedded endpoints: - - cluster_a_dbnode01:2379 + - cluster_a_etcd:2379 diff --git a/scripts/docker-integration-tests/run.sh b/scripts/docker-integration-tests/run.sh index bead15b207..2f43cedc68 100755 --- a/scripts/docker-integration-tests/run.sh +++ b/scripts/docker-integration-tests/run.sh @@ -4,14 +4,24 @@ set -ex TESTS=( scripts/docker-integration-tests/cold_writes_simple/test.sh - scripts/docker-integration-tests/prometheus_replication/test.sh +# TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup. +# scripts/docker-integration-tests/prometheus_replication/test.sh + scripts/docker-integration-tests/carbon/test.sh scripts/docker-integration-tests/aggregator/test.sh scripts/docker-integration-tests/aggregator_legacy/test.sh - scripts/docker-integration-tests/query_fanout/test.sh - scripts/docker-integration-tests/repair/test.sh - scripts/docker-integration-tests/replication/test.sh - scripts/docker-integration-tests/multi_cluster_write/test.sh + +# TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup. +# scripts/docker-integration-tests/query_fanout/test.sh + + scripts/docker-integration-tests/repair/test.sh + +# TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup. +# scripts/docker-integration-tests/replication/test.sh + +# TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup. +# scripts/docker-integration-tests/multi_cluster_write/test.sh + scripts/docker-integration-tests/coordinator_config_rules/test.sh scripts/docker-integration-tests/coordinator_noop/test.sh scripts/docker-integration-tests/prom_remote_write_backend/test.sh diff --git a/scripts/docker-integration-tests/setup.sh b/scripts/docker-integration-tests/setup.sh index 76659a49a2..d90ecb4967 100755 --- a/scripts/docker-integration-tests/setup.sh +++ b/scripts/docker-integration-tests/setup.sh @@ -15,8 +15,8 @@ mkdir -p ./bin # by keeping all the required files in ./bin, it makes the build context # for docker much smaller -cp ./src/query/config/m3coordinator-local-etcd.yml ./bin -cp ./src/dbnode/config/m3dbnode-local-etcd.yml ./bin +cp ./src/query/config/m3coordinator-local-docker-etcd.yml ./bin +cp ./src/dbnode/config/m3dbnode-local-docker-etcd.yml ./bin cp ./src/aggregator/config/m3aggregator.yml ./bin # build images @@ -26,7 +26,9 @@ function build_image { local svc=$1 echo "creating image for $svc" make ${svc}-linux-amd64 - docker build -t "${svc}_integration:${REVISION}" -f ./scripts/docker-integration-tests/${svc}.Dockerfile ./bin + docker build \ + --no-cache \ + -t "${svc}_integration:${REVISION}" -f ./scripts/docker-integration-tests/${svc}.Dockerfile ./bin } if [[ "$SERVICE" != "" ]]; then diff --git a/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml b/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml index bce43d559e..d2c7844c8c 100644 --- a/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml +++ b/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml @@ -1,15 +1,25 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -22,5 +32,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml b/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml index 7d5cb12598..945b26e3b5 100644 --- a/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml +++ b/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml @@ -13,5 +13,5 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 useV2BatchAPIs: true diff --git a/src/dbnode/config/m3dbnode-local-docker-etcd.yml b/src/dbnode/config/m3dbnode-local-docker-etcd.yml new file mode 100644 index 0000000000..83650bc894 --- /dev/null +++ b/src/dbnode/config/m3dbnode-local-docker-etcd.yml @@ -0,0 +1,11 @@ +coordinator: {} +"db": + discovery: + "config": + "service": + "etcdClusters": + - "endpoints": ["http://etcd:2379"] + "zone": "embedded" + "service": "m3db" + "zone": "embedded" + "env": "default_env" \ No newline at end of file diff --git a/src/dbnode/config/m3dbnode-local-etcd.yml b/src/dbnode/config/m3dbnode-local-etcd.yml index 83650bc894..d0545cfdb9 100644 --- a/src/dbnode/config/m3dbnode-local-etcd.yml +++ b/src/dbnode/config/m3dbnode-local-etcd.yml @@ -1,11 +1,2 @@ coordinator: {} -"db": - discovery: - "config": - "service": - "etcdClusters": - - "endpoints": ["http://etcd:2379"] - "zone": "embedded" - "service": "m3db" - "zone": "embedded" - "env": "default_env" \ No newline at end of file +db: {} diff --git a/src/query/config/m3coordinator-local-docker-etcd.yml b/src/query/config/m3coordinator-local-docker-etcd.yml new file mode 100644 index 0000000000..c25dff706c --- /dev/null +++ b/src/query/config/m3coordinator-local-docker-etcd.yml @@ -0,0 +1,16 @@ +clusters: + - namespaces: + - namespace: default + type: unaggregated + retention: 48h + client: + config: + service: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - http://etcd:2379