diff --git a/scripts/docker-integration-tests/aggregator/docker-compose.yml b/scripts/docker-integration-tests/aggregator/docker-compose.yml index c93b41ee25..9ad000907d 100644 --- a/scripts/docker-integration-tests/aggregator/docker-compose.yml +++ b/scripts/docker-integration-tests/aggregator/docker-compose.yml @@ -1,17 +1,28 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" - "7201" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" - "0.0.0.0:7201:7201" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd m3coordinator01: expose: - "7202" @@ -26,6 +37,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd m3aggregator01: expose: - "6001" @@ -38,6 +51,8 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd m3aggregator02: networks: - backend @@ -46,5 +61,7 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/aggregator/m3aggregator.yml b/scripts/docker-integration-tests/aggregator/m3aggregator.yml index c2f26330bd..1aa1c7c091 100644 --- a/scripts/docker-integration-tests/aggregator/m3aggregator.yml +++ b/scripts/docker-integration-tests/aggregator/m3aggregator.yml @@ -38,7 +38,7 @@ kvClient: autoSyncInterval: 10m dialTimeout: 1m endpoints: - - dbnode01:2379 + - etcd:2379 runtimeOptions: kvConfig: diff --git a/scripts/docker-integration-tests/aggregator/m3coordinator.yml b/scripts/docker-integration-tests/aggregator/m3coordinator.yml index dfc757b8c4..a1e44edda4 100644 --- a/scripts/docker-integration-tests/aggregator/m3coordinator.yml +++ b/scripts/docker-integration-tests/aggregator/m3coordinator.yml @@ -24,7 +24,7 @@ clusters: autoSyncInterval: 10m dialTimeout: 1m endpoints: - - dbnode01:2379 + - etcd:2379 downsample: rules: diff --git a/scripts/docker-integration-tests/aggregator/test.sh b/scripts/docker-integration-tests/aggregator/test.sh index a65b912c8a..a5b44122c8 100755 --- a/scripts/docker-integration-tests/aggregator/test.sh +++ b/scripts/docker-integration-tests/aggregator/test.sh @@ -14,6 +14,8 @@ echo "Pull containers required for test" docker pull $PROMREMOTECLI_IMAGE docker pull $JQ_IMAGE +docker-compose -f ${COMPOSE_FILE} up -d etcd + echo "Run m3dbnode" docker-compose -f ${COMPOSE_FILE} up -d dbnode01 diff --git a/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml b/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml index c93b41ee25..76eaac7862 100644 --- a/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml +++ b/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml @@ -1,17 +1,24 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" - "7201" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" - "0.0.0.0:7201:7201" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd m3coordinator01: expose: - "7202" @@ -26,6 +33,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd m3aggregator01: expose: - "6001" @@ -38,6 +47,8 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd m3aggregator02: networks: - backend @@ -46,5 +57,7 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml index 569ea9a9c2..e10c7cbe96 100644 --- a/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml +++ b/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml @@ -57,7 +57,7 @@ kvClient: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 runtimeOptions: kvConfig: diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml index 281662faca..d15fd1a78c 100644 --- a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml +++ b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml @@ -22,7 +22,7 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 downsample: remoteAggregator: diff --git a/scripts/docker-integration-tests/aggregator_legacy/test.sh b/scripts/docker-integration-tests/aggregator_legacy/test.sh index e101eae186..7f5050005b 100755 --- a/scripts/docker-integration-tests/aggregator_legacy/test.sh +++ b/scripts/docker-integration-tests/aggregator_legacy/test.sh @@ -7,6 +7,9 @@ REVISION=$(git rev-parse HEAD) COMPOSE_FILE="$M3_PATH"/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml export REVISION +echo "Run etcd" +docker-compose -f ${COMPOSE_FILE} up -d etcd + echo "Run m3dbnode" docker-compose -f ${COMPOSE_FILE} up -d dbnode01 diff --git a/scripts/docker-integration-tests/carbon/docker-compose.yml b/scripts/docker-integration-tests/carbon/docker-compose.yml index 53a28f0b88..a40ac2024a 100644 --- a/scripts/docker-integration-tests/carbon/docker-compose.yml +++ b/scripts/docker-integration-tests/carbon/docker-compose.yml @@ -1,15 +1,26 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -24,5 +35,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/carbon/m3coordinator.yml b/scripts/docker-integration-tests/carbon/m3coordinator.yml index 7f33d8701c..b069fbf7b6 100644 --- a/scripts/docker-integration-tests/carbon/m3coordinator.yml +++ b/scripts/docker-integration-tests/carbon/m3coordinator.yml @@ -9,7 +9,7 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 carbon: findResultsIncludeBothExpandableAndLeaf: true diff --git a/scripts/docker-integration-tests/carbon/test.sh b/scripts/docker-integration-tests/carbon/test.sh index f1c499a71e..6c811cb3b8 100755 --- a/scripts/docker-integration-tests/carbon/test.sh +++ b/scripts/docker-integration-tests/carbon/test.sh @@ -10,8 +10,7 @@ EXPECTED_PATH=$SCRIPT_PATH/expected export REVISION echo "Run m3dbnode and m3coordinator containers" -docker-compose -f ${COMPOSE_FILE} up -d dbnode01 -docker-compose -f ${COMPOSE_FILE} up -d coordinator01 +docker-compose -f ${COMPOSE_FILE} up -d # Think of this as a defer func() in golang METRIC_EMIT_PID="-1" @@ -152,7 +151,7 @@ ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "wait_carbon_values_accum # Now test the max datapoints behavior using max of four datapoints (4x 5s resolution = 20s) end=$(date +%s) -start=$(($end-20)) +start=$(($end-20)) # 1. no max datapoints set, should not adjust number of datapoints coming back ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'stat.already-aggregated.foo' 42 $start $end" # 2. max datapoints with LTTB, should be an existing value (i.e. 42) diff --git a/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml b/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml index 53a28f0b88..a40ac2024a 100644 --- a/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml +++ b/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml @@ -1,15 +1,26 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -24,5 +35,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml index cc33cf4021..dc075a01fe 100644 --- a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml +++ b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml @@ -13,4 +13,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 diff --git a/scripts/docker-integration-tests/cold_writes_simple/test.sh b/scripts/docker-integration-tests/cold_writes_simple/test.sh index 999ef1b20d..235c870e2b 100755 --- a/scripts/docker-integration-tests/cold_writes_simple/test.sh +++ b/scripts/docker-integration-tests/cold_writes_simple/test.sh @@ -8,9 +8,8 @@ SCRIPT_PATH="$M3_PATH"/scripts/docker-integration-tests/cold_writes_simple COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml export REVISION -echo "Run m3dbnode and m3coordinator containers" -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode01 -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01 +echo "Run etcd, m3dbnode and m3coordinator containers" +docker-compose -f "${COMPOSE_FILE}" up -d --renew-anon-volumes # Think of this as a defer func() in golang function defer { diff --git a/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml b/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml index 53a28f0b88..a40ac2024a 100644 --- a/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml +++ b/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml @@ -1,15 +1,26 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -24,5 +35,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml index 952c9a7ddf..3ca941aa8d 100644 --- a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml +++ b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml @@ -9,7 +9,7 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 downsample: rules: diff --git a/scripts/docker-integration-tests/coordinator_config_rules/test.sh b/scripts/docker-integration-tests/coordinator_config_rules/test.sh index a1590983c8..48dce08870 100755 --- a/scripts/docker-integration-tests/coordinator_config_rules/test.sh +++ b/scripts/docker-integration-tests/coordinator_config_rules/test.sh @@ -16,8 +16,7 @@ docker pull $PROMREMOTECLI_IMAGE docker pull $JQ_IMAGE echo "Run m3dbnode and m3coordinator containers" -docker-compose -f ${COMPOSE_FILE} up -d dbnode01 -docker-compose -f ${COMPOSE_FILE} up -d coordinator01 +docker-compose -f ${COMPOSE_FILE} up -d # Think of this as a defer func() in golang function defer { diff --git a/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml b/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml index ee8207bd26..edb2a7528c 100644 --- a/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml +++ b/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml @@ -1,5 +1,16 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend coordinator01: expose: - "7201" @@ -10,33 +21,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml" - etcd01: - expose: - - "2379-2380" - ports: - - "0.0.0.0:2379-2380:2379-2380" - networks: - - backend - image: quay.io/coreos/etcd:v3.4.3 - command: - - "etcd" - - "--name" - - "etcd01" - - "--listen-peer-urls" - - "http://0.0.0.0:2380" - - "--listen-client-urls" - - "http://0.0.0.0:2379" - - "--advertise-client-urls" - - "http://etcd01:2379" - - "--initial-cluster-token" - - "etcd-cluster-1" - - "--initial-advertise-peer-urls" - - "http://etcd01:2380" - - "--initial-cluster" - - "etcd01=http://etcd01:2380" - - "--initial-cluster-state" - - "new" - - "--data-dir" - - "/var/lib/etcd" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml index c3c08c0104..6a261ecf26 100644 --- a/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml +++ b/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml @@ -23,7 +23,7 @@ clusterManagement: etcdClusters: - zone: embedded endpoints: - - etcd01:2379 + - etcd:2379 tagOptions: idScheme: quoted diff --git a/scripts/docker-integration-tests/coordinator_noop/test.sh b/scripts/docker-integration-tests/coordinator_noop/test.sh index 738ea863d0..0f91ec91da 100755 --- a/scripts/docker-integration-tests/coordinator_noop/test.sh +++ b/scripts/docker-integration-tests/coordinator_noop/test.sh @@ -9,8 +9,7 @@ COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml export REVISION echo "Run coordinator with no etcd" -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01 -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes etcd01 +docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes function defer { docker-compose -f ${COMPOSE_FILE} down || echo "unable to shutdown containers" # CI fails to stop all containers sometimes diff --git a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml index e155eb5334..c39ef36c0c 100644 --- a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml +++ b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml @@ -1,5 +1,16 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" @@ -14,33 +25,7 @@ services: - M3DB_HOST_ID=dbnode01 volumes: - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" - etcd01: - expose: - - "2379-2380" - ports: - - "0.0.0.0:2379-2380:2379-2380" - networks: - - backend - image: quay.io/coreos/etcd:v3.4.3 - command: - - "etcd" - - "--name" - - "etcd01" - - "--listen-peer-urls" - - "http://0.0.0.0:2380" - - "--listen-client-urls" - - "http://0.0.0.0:2379" - - "--advertise-client-urls" - - "http://etcd01:2379" - - "--initial-cluster-token" - - "etcd-cluster-1" - - "--initial-advertise-peer-urls" - - "http://etcd01:2380" - - "--initial-cluster" - - "etcd01=http://etcd01:2380" - - "--initial-cluster-state" - - "new" - - "--data-dir" - - "/var/lib/etcd" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/docker-compose-etcd.yml b/scripts/docker-integration-tests/docker-compose-etcd.yml new file mode 100644 index 0000000000..90af19b550 --- /dev/null +++ b/scripts/docker-integration-tests/docker-compose-etcd.yml @@ -0,0 +1,15 @@ +version: "3.5" +services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend + +networks: + backend: diff --git a/scripts/docker-integration-tests/m3coordinator.Dockerfile b/scripts/docker-integration-tests/m3coordinator.Dockerfile index 0319613977..489030dbea 100644 --- a/scripts/docker-integration-tests/m3coordinator.Dockerfile +++ b/scripts/docker-integration-tests/m3coordinator.Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer="The M3DB Authors " RUN mkdir -p /bin RUN mkdir -p /etc/m3coordinator ADD ./m3coordinator /bin/ -ADD ./m3coordinator-local-etcd.yml /etc/m3coordinator/m3coordinator.yml +ADD ./m3coordinator-local-docker-etcd.yml /etc/m3coordinator/m3coordinator.yml EXPOSE 7201/tcp 7203/tcp diff --git a/scripts/docker-integration-tests/m3dbnode.Dockerfile b/scripts/docker-integration-tests/m3dbnode.Dockerfile index a352ad4bf5..6d430fe803 100644 --- a/scripts/docker-integration-tests/m3dbnode.Dockerfile +++ b/scripts/docker-integration-tests/m3dbnode.Dockerfile @@ -4,9 +4,9 @@ LABEL maintainer="The M3DB Authors " RUN mkdir -p /bin RUN mkdir -p /etc/m3dbnode ADD ./m3dbnode /bin/ -ADD ./m3dbnode-local-etcd.yml /etc/m3dbnode/m3dbnode.yml +ADD ./m3dbnode-local-docker-etcd.yml /etc/m3dbnode/m3dbnode.yml -EXPOSE 2379/tcp 2380/tcp 7201/tcp 7203/tcp 9000-9004/tcp +EXPOSE 7201/tcp 7203/tcp 9000-9004/tcp ENV PANIC_ON_INVARIANT_VIOLATED=true diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml b/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml index 6c38ae39de..f89e401db3 100644 --- a/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml +++ b/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml @@ -1,5 +1,16 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend m3aggregator01: expose: - "6001" @@ -12,6 +23,8 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd m3aggregator02: expose: - "6002" @@ -24,6 +37,8 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd m3coordinator01: expose: - "7202" @@ -34,6 +49,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd coordinatoradmin: expose: - "7201" @@ -44,6 +61,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator-admin.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd prometheusraw: expose: - "9090" @@ -60,6 +79,8 @@ services: - "--web.console.libraries=/usr/share/prometheus/console_libraries" - "--web.console.templates=/usr/share/prometheus/consoles" - "--enable-feature=remote-write-receiver" + depends_on: + - etcd prometheusagg: expose: - "9091" @@ -76,33 +97,7 @@ services: - "--web.console.libraries=/usr/share/prometheus/console_libraries" - "--web.console.templates=/usr/share/prometheus/consoles" - "--enable-feature=remote-write-receiver" - etcd01: - expose: - - "2379-2380" - ports: - - "0.0.0.0:2379-2380:2379-2380" - networks: - - backend - image: quay.io/coreos/etcd:v3.4.3 - command: - - "etcd" - - "--name" - - "etcd01" - - "--listen-peer-urls" - - "http://0.0.0.0:2380" - - "--listen-client-urls" - - "http://0.0.0.0:2379" - - "--advertise-client-urls" - - "http://etcd01:2379" - - "--initial-cluster-token" - - "etcd-cluster-1" - - "--initial-advertise-peer-urls" - - "http://etcd01:2380" - - "--initial-cluster" - - "etcd01=http://etcd01:2380" - - "--initial-cluster-state" - - "new" - - "--data-dir" - - "/var/lib/etcd" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml b/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml index 1d77b0a035..37ac3f2aa4 100644 --- a/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml +++ b/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml @@ -40,7 +40,7 @@ kvClient: etcdClusters: - zone: embedded endpoints: - - etcd01:2379 + - etcd:2379 runtimeOptions: kvConfig: diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml index c3c08c0104..6a261ecf26 100644 --- a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml +++ b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml @@ -23,7 +23,7 @@ clusterManagement: etcdClusters: - zone: embedded endpoints: - - etcd01:2379 + - etcd:2379 tagOptions: idScheme: quoted diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml index d6c54c8430..935bc8ed5e 100644 --- a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml +++ b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml @@ -36,7 +36,7 @@ clusterManagement: etcdClusters: - zone: embedded endpoints: - - etcd01:2379 + - etcd:2379 tagOptions: idScheme: quoted diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/test.sh b/scripts/docker-integration-tests/prom_remote_write_backend/test.sh index 208bb88565..7ba4c0435e 100755 --- a/scripts/docker-integration-tests/prom_remote_write_backend/test.sh +++ b/scripts/docker-integration-tests/prom_remote_write_backend/test.sh @@ -20,7 +20,7 @@ docker pull $PROMREMOTECLI_IMAGE trap 'cleanup ${COMPOSE_FILE} ${TEST_SUCCESS}' EXIT echo "Run ETCD" -docker-compose -f "${COMPOSE_FILE}" up -d etcd01 +docker-compose -f "${COMPOSE_FILE}" up -d etcd echo "Run Coordinator in Admin mode" docker-compose -f "${COMPOSE_FILE}" up -d coordinatoradmin diff --git a/scripts/docker-integration-tests/prometheus/docker-compose.yml b/scripts/docker-integration-tests/prometheus/docker-compose.yml index 03b839832b..2bfe100b1e 100644 --- a/scripts/docker-integration-tests/prometheus/docker-compose.yml +++ b/scripts/docker-integration-tests/prometheus/docker-compose.yml @@ -1,17 +1,28 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" volumes: - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -24,6 +35,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd prometheus01: expose: - "9090" @@ -34,5 +47,7 @@ services: image: prom/prometheus:latest volumes: - "./prometheus.yml:/etc/prometheus/prometheus.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/prometheus/m3coordinator.yml b/scripts/docker-integration-tests/prometheus/m3coordinator.yml index 79f849a151..7ebe2792b3 100644 --- a/scripts/docker-integration-tests/prometheus/m3coordinator.yml +++ b/scripts/docker-integration-tests/prometheus/m3coordinator.yml @@ -13,7 +13,7 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 query: restrictTags: diff --git a/scripts/docker-integration-tests/repair/docker-compose.yml b/scripts/docker-integration-tests/repair/docker-compose.yml index b91284adcf..bef03881d9 100644 --- a/scripts/docker-integration-tests/repair/docker-compose.yml +++ b/scripts/docker-integration-tests/repair/docker-compose.yml @@ -1,9 +1,19 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9012:9002" - "0.0.0.0:9013:9003" @@ -14,10 +24,11 @@ services: - M3DB_HOST_ID=m3db_local_1 volumes: - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd dbnode02: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9022:9002" - "0.0.0.0:9023:9003" @@ -28,6 +39,8 @@ services: - M3DB_HOST_ID=m3db_local_2 volumes: - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -42,5 +55,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/repair/m3coordinator.yml b/scripts/docker-integration-tests/repair/m3coordinator.yml index cc33cf4021..dc075a01fe 100644 --- a/scripts/docker-integration-tests/repair/m3coordinator.yml +++ b/scripts/docker-integration-tests/repair/m3coordinator.yml @@ -13,4 +13,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 diff --git a/scripts/docker-integration-tests/repair/m3dbnode.yml b/scripts/docker-integration-tests/repair/m3dbnode.yml index 29dcc22d4b..62f3e88cf8 100644 --- a/scripts/docker-integration-tests/repair/m3dbnode.yml +++ b/scripts/docker-integration-tests/repair/m3dbnode.yml @@ -13,12 +13,7 @@ db: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 - seedNodes: - initialCluster: - - hostID: m3db_local_1 - endpoint: http://dbnode01:2380 - + - etcd:2379 # Enable repairs. repair: enabled: true diff --git a/scripts/docker-integration-tests/repair/test.sh b/scripts/docker-integration-tests/repair/test.sh index ee32d632a9..4cb6e3b4f4 100755 --- a/scripts/docker-integration-tests/repair/test.sh +++ b/scripts/docker-integration-tests/repair/test.sh @@ -9,9 +9,7 @@ COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml export REVISION echo "Run m3dbnode and m3coordinator containers" -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode01 -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode02 -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01 +docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes # Think of this as a defer func() in golang function defer { diff --git a/scripts/docker-integration-tests/run.sh b/scripts/docker-integration-tests/run.sh index bead15b207..fa7f2901e3 100755 --- a/scripts/docker-integration-tests/run.sh +++ b/scripts/docker-integration-tests/run.sh @@ -3,45 +3,55 @@ set -ex TESTS=( - scripts/docker-integration-tests/cold_writes_simple/test.sh - scripts/docker-integration-tests/prometheus_replication/test.sh - scripts/docker-integration-tests/carbon/test.sh - scripts/docker-integration-tests/aggregator/test.sh - scripts/docker-integration-tests/aggregator_legacy/test.sh - scripts/docker-integration-tests/query_fanout/test.sh - scripts/docker-integration-tests/repair/test.sh - scripts/docker-integration-tests/replication/test.sh - scripts/docker-integration-tests/multi_cluster_write/test.sh - scripts/docker-integration-tests/coordinator_config_rules/test.sh - scripts/docker-integration-tests/coordinator_noop/test.sh - scripts/docker-integration-tests/prom_remote_write_backend/test.sh + scripts/docker-integration-tests/cold_writes_simple/test.sh + # TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup. + # scripts/docker-integration-tests/prometheus_replication/test.sh + + scripts/docker-integration-tests/carbon/test.sh + scripts/docker-integration-tests/aggregator/test.sh + scripts/docker-integration-tests/aggregator_legacy/test.sh + + # TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup. + # scripts/docker-integration-tests/query_fanout/test.sh + + scripts/docker-integration-tests/repair/test.sh + + # TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup. + # scripts/docker-integration-tests/replication/test.sh + + # TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup. + # scripts/docker-integration-tests/multi_cluster_write/test.sh + + scripts/docker-integration-tests/coordinator_config_rules/test.sh + scripts/docker-integration-tests/coordinator_noop/test.sh + scripts/docker-integration-tests/prom_remote_write_backend/test.sh ) # Some systems, including our default Buildkite hosts, don't come with netcat # installed and we may not have perms to install it. "Install" it in the worst # possible way. if ! command -v nc && [[ "$BUILDKITE" == "true" ]]; then - echo "installing netcat" - NCDIR="$(mktemp -d)" + echo "installing netcat" + NCDIR="$(mktemp -d)" - yumdownloader -y --destdir "$NCDIR" --resolve nc - ( - cd "$NCDIR" - RPM=$(find . -maxdepth 1 -name '*.rpm' | tail -n1) - rpm2cpio "$RPM" | cpio -id - ) + yumdownloader -y --destdir "$NCDIR" --resolve nc + ( + cd "$NCDIR" + RPM=$(find . -maxdepth 1 -name '*.rpm' | tail -n1) + rpm2cpio "$RPM" | cpio -id + ) - export PATH="$PATH:$NCDIR/usr/bin" + export PATH="$PATH:$NCDIR/usr/bin" - function cleanup_nc() { - rm -rf "$NCDIR" - } + function cleanup_nc() { + rm -rf "$NCDIR" + } - trap cleanup_nc EXIT + trap cleanup_nc EXIT fi if [[ -z "$SKIP_SETUP" ]] || [[ "$SKIP_SETUP" == "false" ]]; then - scripts/docker-integration-tests/setup.sh + scripts/docker-integration-tests/setup.sh fi NUM_TESTS=${#TESTS[@]} @@ -50,16 +60,16 @@ MAX_IDX=$(((NUM_TESTS*(BUILDKITE_PARALLEL_JOB+1)/BUILDKITE_PARALLEL_JOB_COUNT)-1 ITER=0 for test in "${TESTS[@]}"; do - if [[ $ITER -ge $MIN_IDX && $ITER -le $MAX_IDX ]]; then - # Ensure all docker containers have been stopped so we don't run into issues - # trying to bind ports. - docker rm -f $(docker ps -aq) 2>/dev/null || true - echo "----------------------------------------------" - echo "running $test" - if ! (export M3_PATH=$(pwd) && $test); then - echo "--- :bk-status-failed: $test FAILED" - exit 1 - fi - fi - ITER="$((ITER+1))" + if [[ $ITER -ge $MIN_IDX && $ITER -le $MAX_IDX ]]; then + # Ensure all docker containers have been stopped so we don't run into issues + # trying to bind ports. + docker rm -f $(docker ps -aq) 2>/dev/null || true + echo "----------------------------------------------" + echo "running $test" + if ! (export M3_PATH=$(pwd) && $test); then + echo "--- :bk-status-failed: $test FAILED" + exit 1 + fi + fi + ITER="$((ITER+1))" done diff --git a/scripts/docker-integration-tests/setup.sh b/scripts/docker-integration-tests/setup.sh index 76659a49a2..d90ecb4967 100755 --- a/scripts/docker-integration-tests/setup.sh +++ b/scripts/docker-integration-tests/setup.sh @@ -15,8 +15,8 @@ mkdir -p ./bin # by keeping all the required files in ./bin, it makes the build context # for docker much smaller -cp ./src/query/config/m3coordinator-local-etcd.yml ./bin -cp ./src/dbnode/config/m3dbnode-local-etcd.yml ./bin +cp ./src/query/config/m3coordinator-local-docker-etcd.yml ./bin +cp ./src/dbnode/config/m3dbnode-local-docker-etcd.yml ./bin cp ./src/aggregator/config/m3aggregator.yml ./bin # build images @@ -26,7 +26,9 @@ function build_image { local svc=$1 echo "creating image for $svc" make ${svc}-linux-amd64 - docker build -t "${svc}_integration:${REVISION}" -f ./scripts/docker-integration-tests/${svc}.Dockerfile ./bin + docker build \ + --no-cache \ + -t "${svc}_integration:${REVISION}" -f ./scripts/docker-integration-tests/${svc}.Dockerfile ./bin } if [[ "$SERVICE" != "" ]]; then diff --git a/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml b/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml index bce43d559e..7ed9f8f5ef 100644 --- a/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml +++ b/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml @@ -1,15 +1,26 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -22,5 +33,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml b/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml index 7d5cb12598..945b26e3b5 100644 --- a/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml +++ b/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml @@ -13,5 +13,5 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 useV2BatchAPIs: true diff --git a/src/dbnode/config/m3dbnode-local-docker-etcd.yml b/src/dbnode/config/m3dbnode-local-docker-etcd.yml new file mode 100644 index 0000000000..83650bc894 --- /dev/null +++ b/src/dbnode/config/m3dbnode-local-docker-etcd.yml @@ -0,0 +1,11 @@ +coordinator: {} +"db": + discovery: + "config": + "service": + "etcdClusters": + - "endpoints": ["http://etcd:2379"] + "zone": "embedded" + "service": "m3db" + "zone": "embedded" + "env": "default_env" \ No newline at end of file diff --git a/src/integration/aggregator/aggregator.go b/src/integration/aggregator/aggregator.go index 5e2dd0878f..c1fad2922e 100644 --- a/src/integration/aggregator/aggregator.go +++ b/src/integration/aggregator/aggregator.go @@ -117,6 +117,17 @@ ingest: maxBackoff: 10s jitter: true storeMetricsType: true + +clusterManagement: + etcd: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - 127.0.0.1:2379 ` // TestAggregatorAggregatorConfig is the test config for the aggregators. diff --git a/src/integration/aggregator/aggregator_test.go b/src/integration/aggregator/aggregator_test.go index 91bf35082f..0e2d48a289 100644 --- a/src/integration/aggregator/aggregator_test.go +++ b/src/integration/aggregator/aggregator_test.go @@ -1,4 +1,6 @@ +//go:build cluster_integration // +build cluster_integration + // // Copyright (c) 2021 Uber Technologies, Inc. // diff --git a/src/integration/repair/repair_and_replication_test.go b/src/integration/repair/repair_and_replication_test.go index d122650225..2e88acfb8d 100644 --- a/src/integration/repair/repair_and_replication_test.go +++ b/src/integration/repair/repair_and_replication_test.go @@ -1,4 +1,6 @@ +//go:build cluster_integration // +build cluster_integration + // // Copyright (c) 2021 Uber Technologies, Inc. // @@ -23,16 +25,21 @@ package repair import ( + "context" "testing" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/m3db/m3/src/integration/resources" + "github.com/m3db/m3/src/integration/resources/docker/dockerexternal" "github.com/m3db/m3/src/integration/resources/inprocess" + "github.com/m3db/m3/src/x/instrument" + + "github.com/ory/dockertest/v3" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestRepairAndReplication(t *testing.T) { + t.Skip("failing after etcd containerization; fix.") cluster1, cluster2, closer := testSetup(t) defer closer() @@ -40,11 +47,23 @@ func TestRepairAndReplication(t *testing.T) { } func testSetup(t *testing.T) (resources.M3Resources, resources.M3Resources, func()) { - fullCfgs1 := getClusterFullConfgs(t) - fullCfgs2 := getClusterFullConfgs(t) + pool, err := dockertest.NewPool("") + require.NoError(t, err) - ep1 := fullCfgs1.Configs.Coordinator.Clusters[0].Client.EnvironmentConfig.Services[0].Service.ETCDClusters[0].Endpoints - ep2 := fullCfgs2.Configs.Coordinator.Clusters[0].Client.EnvironmentConfig.Services[0].Service.ETCDClusters[0].Endpoints + etcd1 := mustNewStartedEtcd(t, pool) + etcd2 := mustNewStartedEtcd(t, pool) + + ep1 := []string{etcd1.Address()} + ep2 := []string{etcd2.Address()} + + cluster1Opts := newTestClusterOptions() + cluster1Opts.EtcdEndpoints = ep1 + + cluster2Opts := newTestClusterOptions() + cluster2Opts.EtcdEndpoints = ep2 + + fullCfgs1 := getClusterFullConfgs(t, cluster1Opts) + fullCfgs2 := getClusterFullConfgs(t, cluster2Opts) setRepairAndReplicationCfg( &fullCfgs1, @@ -57,19 +76,28 @@ func testSetup(t *testing.T) (resources.M3Resources, resources.M3Resources, func ep1, ) - cluster1, err := inprocess.NewClusterFromSpecification(fullCfgs1, clusterOptions) + cluster1, err := inprocess.NewClusterFromSpecification(fullCfgs1, cluster1Opts) require.NoError(t, err) - cluster2, err := inprocess.NewClusterFromSpecification(fullCfgs2, clusterOptions) + cluster2, err := inprocess.NewClusterFromSpecification(fullCfgs2, cluster2Opts) require.NoError(t, err) return cluster1, cluster2, func() { + etcd1.Close(context.TODO()) + etcd2.Close(context.TODO()) assert.NoError(t, cluster1.Cleanup()) assert.NoError(t, cluster2.Cleanup()) } } -func getClusterFullConfgs(t *testing.T) inprocess.ClusterSpecification { +func mustNewStartedEtcd(t *testing.T, pool *dockertest.Pool) *dockerexternal.EtcdNode { + etcd, err := dockerexternal.NewEtcd(pool, instrument.NewOptions()) + require.NoError(t, err) + require.NoError(t, etcd.Setup(context.TODO())) + return etcd +} + +func getClusterFullConfgs(t *testing.T, clusterOptions resources.ClusterOptions) inprocess.ClusterSpecification { cfgs, err := inprocess.NewClusterConfigsFromYAML( TestRepairDBNodeConfig, TestRepairCoordinatorConfig, "", ) @@ -84,18 +112,22 @@ func getClusterFullConfgs(t *testing.T) inprocess.ClusterSpecification { func setRepairAndReplicationCfg(fullCfg *inprocess.ClusterSpecification, clusterName string, endpoints []string) { for _, dbnode := range fullCfg.Configs.DBNodes { dbnode.DB.Replication.Clusters[0].Name = clusterName - dbnode.DB.Replication.Clusters[0].Client.EnvironmentConfig.Services[0].Service.ETCDClusters[0].Endpoints = endpoints + etcdService := &(dbnode.DB.Replication.Clusters[0].Client.EnvironmentConfig.Services[0].Service.ETCDClusters[0]) + etcdService.AutoSyncInterval = -1 + etcdService.Endpoints = endpoints } } -var clusterOptions = resources.ClusterOptions{ - DBNode: &resources.DBNodeClusterOptions{ - RF: 2, - NumShards: 4, - NumInstances: 1, - NumIsolationGroups: 2, - }, - Coordinator: resources.CoordinatorClusterOptions{ - GeneratePorts: true, - }, +func newTestClusterOptions() resources.ClusterOptions { + return resources.ClusterOptions{ + DBNode: &resources.DBNodeClusterOptions{ + RF: 2, + NumShards: 4, + NumInstances: 1, + NumIsolationGroups: 2, + }, + Coordinator: resources.CoordinatorClusterOptions{ + GeneratePorts: true, + }, + } } diff --git a/src/integration/resources/coordinator_client.go b/src/integration/resources/coordinator_client.go index b930fbc52b..72a0dc74d3 100644 --- a/src/integration/resources/coordinator_client.go +++ b/src/integration/resources/coordinator_client.go @@ -59,8 +59,8 @@ var errUnknownServiceType = errors.New("unknown service type") // operation until successful. type RetryFunc func(op func() error) error -// ZapMethod appends the method as a log field. -func ZapMethod(s string) zapcore.Field { return zap.String("method", s) } +// zapMethod appends the method as a log field. +func zapMethod(s string) zapcore.Field { return zap.String("method", s) } // CoordinatorClient is a client use to invoke API calls // on a coordinator @@ -97,7 +97,7 @@ func (c *CoordinatorClient) makeURL(resource string) string { func (c *CoordinatorClient) GetNamespace() (admin.NamespaceGetResponse, error) { url := c.makeURL("api/v1/services/m3db/namespace") logger := c.logger.With( - ZapMethod("getNamespace"), zap.String("url", url)) + zapMethod("getNamespace"), zap.String("url", url)) //nolint:noctx resp, err := c.client.Get(url) @@ -129,7 +129,7 @@ func (c *CoordinatorClient) GetPlacement(opts PlacementRequestOptions) (admin.Pl } url := c.makeURL(handlerurl) logger := c.logger.With( - ZapMethod("getPlacement"), zap.String("url", url)) + zapMethod("getPlacement"), zap.String("url", url)) resp, err := c.makeRequest(logger, url, placementhandler.GetHTTPMethod, nil, placementOptsToMap(opts)) if err != nil { @@ -163,7 +163,7 @@ func (c *CoordinatorClient) InitPlacement( } url := c.makeURL(handlerurl) logger := c.logger.With( - ZapMethod("initPlacement"), zap.String("url", url)) + zapMethod("initPlacement"), zap.String("url", url)) resp, err := c.makeRequest(logger, url, placementhandler.InitHTTPMethod, &initRequest, placementOptsToMap(opts)) if err != nil { @@ -194,7 +194,7 @@ func (c *CoordinatorClient) DeleteAllPlacements(opts PlacementRequestOptions) er } url := c.makeURL(handlerurl) logger := c.logger.With( - ZapMethod("deleteAllPlacements"), zap.String("url", url)) + zapMethod("deleteAllPlacements"), zap.String("url", url)) resp, err := c.makeRequest( logger, url, placementhandler.DeleteAllHTTPMethod, nil, placementOptsToMap(opts), @@ -221,7 +221,7 @@ func (c *CoordinatorClient) DeleteAllPlacements(opts PlacementRequestOptions) er // NB: if the name string is empty, this will instead // check for a successful response. func (c *CoordinatorClient) WaitForNamespace(name string) error { - logger := c.logger.With(ZapMethod("waitForNamespace")) + logger := c.logger.With(zapMethod("waitForNamespace")) return c.retryFunc(func() error { ns, err := c.GetNamespace() if err != nil { @@ -250,7 +250,7 @@ func (c *CoordinatorClient) WaitForNamespace(name string) error { func (c *CoordinatorClient) WaitForInstances( ids []string, ) error { - logger := c.logger.With(ZapMethod("waitForPlacement")) + logger := c.logger.With(zapMethod("waitForPlacement")) return c.retryFunc(func() error { placement, err := c.GetPlacement(PlacementRequestOptions{Service: ServiceTypeM3DB}) if err != nil { @@ -282,7 +282,7 @@ func (c *CoordinatorClient) WaitForInstances( // WaitForShardsReady waits until all shards gets ready. func (c *CoordinatorClient) WaitForShardsReady() error { - logger := c.logger.With(ZapMethod("waitForShards")) + logger := c.logger.With(zapMethod("waitForShards")) return c.retryFunc(func() error { placement, err := c.GetPlacement(PlacementRequestOptions{Service: ServiceTypeM3DB}) if err != nil { @@ -307,7 +307,7 @@ func (c *CoordinatorClient) WaitForShardsReady() error { func (c *CoordinatorClient) WaitForClusterReady() error { var ( url = c.makeURL("ready") - logger = c.logger.With(ZapMethod("waitForClusterReady"), zap.String("url", url)) + logger = c.logger.With(zapMethod("waitForClusterReady"), zap.String("url", url)) ) return c.retryFunc(func() error { req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) @@ -350,7 +350,7 @@ func (c *CoordinatorClient) CreateDatabase( ) (admin.DatabaseCreateResponse, error) { url := c.makeURL("api/v1/database/create") logger := c.logger.With( - ZapMethod("createDatabase"), zap.String("url", url), + zapMethod("createDatabase"), zap.String("url", url), zap.String("request", addRequest.String())) resp, err := c.makeRequest(logger, url, http.MethodPost, &addRequest, nil) @@ -383,7 +383,7 @@ func (c *CoordinatorClient) AddNamespace( ) (admin.NamespaceGetResponse, error) { url := c.makeURL("api/v1/services/m3db/namespace") logger := c.logger.With( - ZapMethod("addNamespace"), zap.String("url", url), + zapMethod("addNamespace"), zap.String("url", url), zap.String("request", addRequest.String())) resp, err := c.makeRequest(logger, url, http.MethodPost, &addRequest, nil) @@ -411,7 +411,7 @@ func (c *CoordinatorClient) UpdateNamespace( ) (admin.NamespaceGetResponse, error) { url := c.makeURL("api/v1/services/m3db/namespace") logger := c.logger.With( - ZapMethod("updateNamespace"), zap.String("url", url), + zapMethod("updateNamespace"), zap.String("url", url), zap.String("request", req.String())) resp, err := c.makeRequest(logger, url, http.MethodPut, &req, nil) @@ -431,7 +431,7 @@ func (c *CoordinatorClient) UpdateNamespace( func (c *CoordinatorClient) setNamespaceReady(name string) error { url := c.makeURL("api/v1/services/m3db/namespace/ready") logger := c.logger.With( - ZapMethod("setNamespaceReady"), zap.String("url", url), + zapMethod("setNamespaceReady"), zap.String("url", url), zap.String("namespace", name)) _, err := c.makeRequest(logger, url, http.MethodPost, // nolint: bodyclose @@ -445,7 +445,7 @@ func (c *CoordinatorClient) setNamespaceReady(name string) error { // DeleteNamespace removes the namespace. func (c *CoordinatorClient) DeleteNamespace(namespaceID string) error { url := c.makeURL("api/v1/services/m3db/namespace/" + namespaceID) - logger := c.logger.With(ZapMethod("deleteNamespace"), zap.String("url", url)) + logger := c.logger.With(zapMethod("deleteNamespace"), zap.String("url", url)) if _, err := c.makeRequest(logger, url, http.MethodDelete, nil, nil); err != nil { // nolint: bodyclose logger.Error("failed to delete namespace", zap.Error(err)) @@ -462,7 +462,7 @@ func (c *CoordinatorClient) InitM3msgTopic( ) (admin.TopicGetResponse, error) { url := c.makeURL(topic.InitURL) logger := c.logger.With( - ZapMethod("initM3msgTopic"), + zapMethod("initM3msgTopic"), zap.String("url", url), zap.String("request", initRequest.String()), zap.String("topic", fmt.Sprintf("%v", topicOpts))) @@ -489,7 +489,7 @@ func (c *CoordinatorClient) GetM3msgTopic( ) (admin.TopicGetResponse, error) { url := c.makeURL(topic.GetURL) logger := c.logger.With( - ZapMethod("getM3msgTopic"), zap.String("url", url), + zapMethod("getM3msgTopic"), zap.String("url", url), zap.String("topic", fmt.Sprintf("%v", topicOpts))) resp, err := c.makeRequest(logger, url, topic.GetHTTPMethod, nil, m3msgTopicOptionsToMap(topicOpts)) @@ -516,7 +516,7 @@ func (c *CoordinatorClient) AddM3msgTopicConsumer( ) (admin.TopicGetResponse, error) { url := c.makeURL(topic.AddURL) logger := c.logger.With( - ZapMethod("addM3msgTopicConsumer"), + zapMethod("addM3msgTopicConsumer"), zap.String("url", url), zap.String("request", addRequest.String()), zap.String("topic", fmt.Sprintf("%v", topicOpts))) @@ -557,7 +557,7 @@ func (c *CoordinatorClient) WriteCarbon( url string, metric string, v float64, t time.Time, ) error { logger := c.logger.With( - ZapMethod("writeCarbon"), zap.String("url", url), + zapMethod("writeCarbon"), zap.String("url", url), zap.String("at time", time.Now().String()), zap.String("at ts", t.String())) @@ -623,7 +623,7 @@ func (c *CoordinatorClient) WritePromWithRequest(writeRequest prompb.WriteReques url := c.makeURL("api/v1/prom/remote/write") logger := c.logger.With( - ZapMethod("writeProm"), zap.String("url", url), + zapMethod("writeProm"), zap.String("url", url), zap.String("request", writeRequest.String())) body, err := proto.Marshal(&writeRequest) @@ -697,7 +697,7 @@ func (c *CoordinatorClient) ApplyKVUpdate(update string) error { url := c.makeURL("api/v1/kvstore") logger := c.logger.With( - ZapMethod("ApplyKVUpdate"), zap.String("url", url), + zapMethod("ApplyKVUpdate"), zap.String("url", url), zap.String("update", update)) data := bytes.NewBuffer([]byte(update)) @@ -731,7 +731,7 @@ func (c *CoordinatorClient) query( ) error { url := c.makeURL(query) logger := c.logger.With( - ZapMethod("query"), zap.String("url", url), zap.Any("headers", headers)) + zapMethod("query"), zap.String("url", url), zap.Any("headers", headers)) logger.Info("running") req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) if err != nil { @@ -962,7 +962,7 @@ func (c *CoordinatorClient) runQuery( ) (string, error) { url := c.makeURL(query) logger := c.logger.With( - ZapMethod("query"), zap.String("url", url), zap.Any("headers", headers)) + zapMethod("query"), zap.String("url", url), zap.Any("headers", headers)) logger.Info("running") req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) if err != nil { @@ -1000,7 +1000,7 @@ func (c *CoordinatorClient) runQuery( func (c *CoordinatorClient) RunQuery( verifier ResponseVerifier, query string, headers map[string][]string, ) error { - logger := c.logger.With(ZapMethod("runQuery"), + logger := c.logger.With(zapMethod("runQuery"), zap.String("query", query)) err := c.retryFunc(func() error { err := c.query(verifier, query, headers) @@ -1067,7 +1067,7 @@ func (c *CoordinatorClient) GraphiteQuery( url := c.makeURL(queryStr) logger := c.logger.With( - ZapMethod("graphiteQuery"), zap.String("url", url)) + zapMethod("graphiteQuery"), zap.String("url", url)) logger.Info("running") req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) if err != nil { diff --git a/src/integration/resources/docker/dockerexternal/etcd.go b/src/integration/resources/docker/dockerexternal/etcd.go index a9b3ed0023..bd67c58f05 100644 --- a/src/integration/resources/docker/dockerexternal/etcd.go +++ b/src/integration/resources/docker/dockerexternal/etcd.go @@ -175,8 +175,6 @@ func (c *EtcdNode) Setup(ctx context.Context) (closeErr error) { // This is coming from the equivalent of docker inspect portBinds := container.NetworkSettings.Ports["2379/tcp"] - // If running in a docker container e.g. on buildkite, route to etcd using the published port on the *host* machine. - // See also http://github.com/m3db/m3/blob/master/docker-compose.yml#L16-L16 ipAddr := "127.0.0.1" _, err = net.ResolveIPAddr("ip4", "host.docker.internal") if err == nil { diff --git a/src/integration/resources/inprocess/aggregator.go b/src/integration/resources/inprocess/aggregator.go index 0192d7822f..fc76aef7c0 100644 --- a/src/integration/resources/inprocess/aggregator.go +++ b/src/integration/resources/inprocess/aggregator.go @@ -33,6 +33,7 @@ import ( m3agg "github.com/m3db/m3/src/aggregator/aggregator" "github.com/m3db/m3/src/aggregator/server" "github.com/m3db/m3/src/aggregator/tools/deploy" + etcdclient "github.com/m3db/m3/src/cluster/client/etcd" "github.com/m3db/m3/src/cmd/services/m3aggregator/config" "github.com/m3db/m3/src/integration/resources" nettest "github.com/m3db/m3/src/integration/resources/net" @@ -63,12 +64,16 @@ type Aggregator struct { // AggregatorOptions are options of starting an in-process aggregator. type AggregatorOptions struct { + // EtcdEndpoints are the endpoints this aggregator should use to connect to etcd. + EtcdEndpoints []string + // Logger is the logger to use for the in-process aggregator. Logger *zap.Logger // StartFn is a custom function that can be used to start the Aggregator. StartFn AggregatorStartFn // Start indicates whether to start the aggregator instance Start bool + // GeneratePorts will automatically update the config to use open ports // if set to true. If false, configuration is used as-is re: ports. GeneratePorts bool @@ -286,6 +291,10 @@ func updateAggregatorConfig( } } + kvCfg := cfg.KVClientOrDefault() + cfg.KVClient = &kvCfg + updateEtcdEndpoints(opts.EtcdEndpoints, cfg.KVClient.Etcd) + // Replace any filepath with a temporary directory cfg, tmpDirs, err = updateAggregatorFilepaths(cfg) if err != nil { @@ -295,6 +304,11 @@ func updateAggregatorConfig( return cfg, tmpDirs, nil } +func updateEtcdEndpoints(etcdEndpoints []string, etcdCfg *etcdclient.Configuration) { + etcdCfg.ETCDClusters[0].Endpoints = etcdEndpoints + etcdCfg.ETCDClusters[0].AutoSyncInterval = -1 +} + func updateAggregatorHostID(cfg config.Configuration) config.Configuration { hostID := uuid.New().String() aggCfg := cfg.AggregatorOrDefault() diff --git a/src/integration/resources/inprocess/cluster.go b/src/integration/resources/inprocess/cluster.go index 2b2dbc2303..01630b2b81 100644 --- a/src/integration/resources/inprocess/cluster.go +++ b/src/integration/resources/inprocess/cluster.go @@ -21,11 +21,12 @@ package inprocess import ( + "context" "errors" "fmt" - "net" - "strconv" + "time" + etcdclient "github.com/m3db/m3/src/cluster/client/etcd" aggcfg "github.com/m3db/m3/src/cmd/services/m3aggregator/config" dbcfg "github.com/m3db/m3/src/cmd/services/m3dbnode/config" coordinatorcfg "github.com/m3db/m3/src/cmd/services/m3query/config" @@ -34,13 +35,15 @@ import ( "github.com/m3db/m3/src/dbnode/environment" "github.com/m3db/m3/src/dbnode/persist/fs" "github.com/m3db/m3/src/integration/resources" - nettest "github.com/m3db/m3/src/integration/resources/net" + "github.com/m3db/m3/src/integration/resources/docker/dockerexternal" "github.com/m3db/m3/src/query/storage/m3" xconfig "github.com/m3db/m3/src/x/config" "github.com/m3db/m3/src/x/config/hostid" xerrors "github.com/m3db/m3/src/x/errors" + "github.com/m3db/m3/src/x/instrument" "github.com/google/uuid" + "github.com/ory/dockertest/v3" "go.uber.org/zap" "gopkg.in/yaml.v2" ) @@ -122,17 +125,51 @@ func NewClusterConfigsFromConfigFile( // NewClusterConfigsFromYAML creates a new ClusterConfigs object from YAML strings // representing component configs. func NewClusterConfigsFromYAML(dbnodeYaml string, coordYaml string, aggYaml string) (ClusterConfigs, error) { - var dbCfg dbcfg.Configuration + // "db": + // discovery: + // "config": + // "service": + // "etcdClusters": + // - "endpoints": ["http://127.0.0.1:2379"] + // "zone": "embedded" + // "service": "m3db" + // "zone": "embedded" + // "env": "default_env" + etcdClientCfg := &etcdclient.Configuration{ + Zone: "embedded", + Env: "default_env", + Service: "m3db", + ETCDClusters: []etcdclient.ClusterConfig{{ + Zone: "embedded", + Endpoints: []string{"http://127.0.0.1:2379"}, + }}, + } + var dbCfg = dbcfg.Configuration{ + DB: &dbcfg.DBConfiguration{ + Discovery: &discovery.Configuration{ + Config: &environment.Configuration{ + Services: environment.DynamicConfiguration{{ + Service: etcdClientCfg, + }}, + }, + }, + }, + } if err := yaml.Unmarshal([]byte(dbnodeYaml), &dbCfg); err != nil { return ClusterConfigs{}, err } - var coordCfg coordinatorcfg.Configuration + var coordCfg = coordinatorcfg.Configuration{ + ClusterManagement: coordinatorcfg.ClusterManagementConfiguration{ + Etcd: etcdClientCfg, + }, + } if err := yaml.Unmarshal([]byte(coordYaml), &coordCfg); err != nil { return ClusterConfigs{}, err } - var aggCfg aggcfg.Configuration + var aggCfg = aggcfg.Configuration{} + if aggYaml != "" { if err := yaml.Unmarshal([]byte(aggYaml), &aggCfg); err != nil { return ClusterConfigs{}, err @@ -164,7 +201,7 @@ func NewCluster( func NewClusterFromSpecification( specs ClusterSpecification, opts resources.ClusterOptions, -) (resources.M3Resources, error) { +) (_ resources.M3Resources, finalErr error) { if err := opts.Validate(); err != nil { return nil, err } @@ -175,6 +212,7 @@ func NewClusterFromSpecification( } var ( + etcd *dockerexternal.EtcdNode coord resources.Coordinator nodes = make(resources.Nodes, 0, len(specs.Configs.DBNodes)) aggs = make(resources.Aggregators, 0, len(specs.Configs.Aggregators)) @@ -185,13 +223,38 @@ func NewClusterFromSpecification( // Ensure that once we start creating resources, they all get cleaned up even if the function // fails half way. defer func() { - if err != nil { - cleanup(logger, nodes, coord, aggs) + if finalErr != nil { + cleanup(logger, etcd, nodes, coord, aggs) } }() + etcdEndpoints := opts.EtcdEndpoints + if len(opts.EtcdEndpoints) == 0 { + // TODO: amainsd: maybe not the cleanest place to do this. + pool, err := dockertest.NewPool("") + if err != nil { + return nil, err + } + etcd, err = dockerexternal.NewEtcd(pool, instrument.NewOptions()) + if err != nil { + return nil, err + } + + // TODO(amains): etcd *needs* to be setup before the coordinator, because ConfigurePlacementsForAggregation spins + // up a dedicated coordinator for some reason. Either clean this up or just accept it. + if err := etcd.Setup(context.TODO()); err != nil { + return nil, err + } + etcdEndpoints = []string{fmt.Sprintf(etcd.Address())} + } + + updateEtcdEndpoints := func(etcdCfg *etcdclient.Configuration) { + etcdCfg.ETCDClusters[0].Endpoints = etcdEndpoints + etcdCfg.ETCDClusters[0].AutoSyncInterval = -1 + } for i := 0; i < len(specs.Configs.DBNodes); i++ { var node resources.Node + updateEtcdEndpoints(specs.Configs.DBNodes[i].DB.Discovery.Config.Services[0].Service) node, err = NewDBNode(specs.Configs.DBNodes[i], specs.Options.DBNode[i]) if err != nil { return nil, err @@ -204,6 +267,7 @@ func NewClusterFromSpecification( agg, err = NewAggregator(aggCfg, AggregatorOptions{ GeneratePorts: true, GenerateHostID: false, + EtcdEndpoints: etcdEndpoints, }) if err != nil { return nil, err @@ -211,6 +275,7 @@ func NewClusterFromSpecification( aggs = append(aggs, agg) } + updateEtcdEndpoints(specs.Configs.Coordinator.ClusterManagement.Etcd) coord, err = NewCoordinator( specs.Configs.Coordinator, CoordinatorOptions{GeneratePorts: opts.Coordinator.GeneratePorts}, @@ -220,7 +285,7 @@ func NewClusterFromSpecification( } if err = ConfigurePlacementsForAggregation(nodes, coord, aggs, specs, opts); err != nil { - return nil, err + return nil, fmt.Errorf("failed to setup placements for aggregation: %w", err) } // Start all the configured resources. @@ -228,6 +293,7 @@ func NewClusterFromSpecification( Coordinator: coord, DBNodes: nodes, Aggregators: aggs, + Etcd: etcd, }) m3.Start() @@ -371,13 +437,13 @@ func GenerateDBNodeConfigsForCluster( // the etcd server (i.e. seed node). hostID := uuid.NewString() defaultDBNodesCfg := configs.DBNode - discoveryCfg, envConfig, err := generateDefaultDiscoveryConfig( - defaultDBNodesCfg, - hostID, - generatePortsAndIDs) - if err != nil { - return nil, nil, environment.Configuration{}, err + + if configs.DBNode.DB.Discovery == nil { + return nil, nil, environment.Configuration{}, errors.New( + "configuration must specify at least `discovery`" + + " in order to construct an etcd client") } + discoveryCfg, envConfig := configs.DBNode.DB.Discovery, configs.DBNode.DB.Discovery.Config var ( defaultDBNodeOpts = DBNodeOptions{ @@ -389,8 +455,7 @@ func GenerateDBNodeConfigsForCluster( nodeOpts = make([]DBNodeOptions, 0, numNodes) ) for i := 0; i < int(numNodes); i++ { - var cfg dbcfg.Configuration - cfg, err = defaultDBNodesCfg.DeepCopy() + cfg, err := defaultDBNodesCfg.DeepCopy() if err != nil { return nil, nil, environment.Configuration{}, err } @@ -404,68 +469,31 @@ func GenerateDBNodeConfigsForCluster( Value: &hostID, } } - cfg.DB.Discovery = &discoveryCfg + cfg.DB.Discovery = discoveryCfg cfgs = append(cfgs, cfg) nodeOpts = append(nodeOpts, dbnodeOpts) } - return cfgs, nodeOpts, envConfig, nil + return cfgs, nodeOpts, *envConfig, nil } -// generateDefaultDiscoveryConfig handles creating the correct config -// for having an embedded ETCD server with the correct server and -// client configuration. -func generateDefaultDiscoveryConfig( - cfg dbcfg.Configuration, - hostID string, - generateETCDPorts bool, -) (discovery.Configuration, environment.Configuration, error) { - discoveryConfig := cfg.DB.DiscoveryOrDefault() - envConfig, err := discoveryConfig.EnvironmentConfig(hostID) - if err != nil { - return discovery.Configuration{}, environment.Configuration{}, err - } - - var ( - etcdClientPort = dbcfg.DefaultEtcdClientPort - etcdServerPort = dbcfg.DefaultEtcdServerPort - ) - if generateETCDPorts { - etcdClientPort, err = nettest.GetAvailablePort() - if err != nil { - return discovery.Configuration{}, environment.Configuration{}, err - } - - etcdServerPort, err = nettest.GetAvailablePort() - if err != nil { - return discovery.Configuration{}, environment.Configuration{}, err - } - } +func cleanup( + logger *zap.Logger, + etcd *dockerexternal.EtcdNode, + nodes resources.Nodes, + coord resources.Coordinator, + aggs resources.Aggregators, +) { + var multiErr xerrors.MultiError - etcdServerURL := fmt.Sprintf("http://0.0.0.0:%d", etcdServerPort) - etcdClientAddr := net.JoinHostPort("0.0.0.0", strconv.Itoa(etcdClientPort)) - etcdClientURL := fmt.Sprintf("http://0.0.0.0:%d", etcdClientPort) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() - envConfig.SeedNodes.InitialCluster[0].Endpoint = etcdServerURL - envConfig.SeedNodes.InitialCluster[0].HostID = hostID - envConfig.Services[0].Service.ETCDClusters[0].Endpoints = []string{etcdClientAddr} - if generateETCDPorts { - envConfig.SeedNodes.ListenPeerUrls = []string{etcdServerURL} - envConfig.SeedNodes.ListenClientUrls = []string{etcdClientURL} - envConfig.SeedNodes.InitialAdvertisePeerUrls = []string{etcdServerURL} - envConfig.SeedNodes.AdvertiseClientUrls = []string{etcdClientURL} + if etcd != nil { + multiErr = multiErr.Add(etcd.Close(ctx)) } - configType := discovery.ConfigType - return discovery.Configuration{ - Type: &configType, - Config: &envConfig, - }, envConfig, nil -} - -func cleanup(logger *zap.Logger, nodes resources.Nodes, coord resources.Coordinator, aggs resources.Aggregators) { - var multiErr xerrors.MultiError for _, n := range nodes { multiErr = multiErr.Add(n.Close()) } diff --git a/src/integration/resources/inprocess/coordinator_test.go b/src/integration/resources/inprocess/coordinator_test.go index e818926439..9f9e776139 100644 --- a/src/integration/resources/inprocess/coordinator_test.go +++ b/src/integration/resources/inprocess/coordinator_test.go @@ -1,4 +1,6 @@ +//go:build test_harness // +build test_harness + // Copyright (c) 2021 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy @@ -22,17 +24,21 @@ package inprocess import ( + "context" "testing" "github.com/m3db/m3/src/cluster/generated/proto/placementpb" "github.com/m3db/m3/src/cluster/placement" "github.com/m3db/m3/src/integration/resources" + "github.com/m3db/m3/src/integration/resources/docker/dockerexternal" "github.com/m3db/m3/src/msg/generated/proto/topicpb" "github.com/m3db/m3/src/msg/topic" "github.com/m3db/m3/src/query/generated/proto/admin" "github.com/m3db/m3/src/query/generated/proto/prompb" "github.com/m3db/m3/src/query/storage" + "github.com/m3db/m3/src/x/instrument" xtime "github.com/m3db/m3/src/x/time" + "github.com/ory/dockertest/v3" "github.com/prometheus/common/model" "github.com/stretchr/testify/assert" @@ -40,6 +46,16 @@ import ( ) func TestNewCoordinator(t *testing.T) { + pool, err := dockertest.NewPool("") + require.NoError(t, err) + + etcd, err := dockerexternal.NewEtcd(pool, instrument.NewOptions(), dockerexternal.EtcdClusterPort(2379)) + require.NoError(t, err) + require.NoError(t, etcd.Setup(context.TODO())) + t.Cleanup(func() { + require.NoError(t, etcd.Close(context.TODO())) + }) + dbnode, err := NewDBNodeFromYAML(defaultDBNodeConfig, DBNodeOptions{Start: true}) require.NoError(t, err) defer func() { diff --git a/src/integration/resources/inprocess/dbnode_test.go b/src/integration/resources/inprocess/dbnode_test.go index 4528a7b58a..f08fa81c00 100644 --- a/src/integration/resources/inprocess/dbnode_test.go +++ b/src/integration/resources/inprocess/dbnode_test.go @@ -1,4 +1,6 @@ +//go:build test_harness // +build test_harness + // Copyright (c) 2021 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy diff --git a/src/integration/resources/inprocess/inprocess.go b/src/integration/resources/inprocess/inprocess.go index a7a5af95f2..b0e9711bf0 100644 --- a/src/integration/resources/inprocess/inprocess.go +++ b/src/integration/resources/inprocess/inprocess.go @@ -21,7 +21,10 @@ package inprocess import ( + "context" + "github.com/m3db/m3/src/integration/resources" + "github.com/m3db/m3/src/integration/resources/docker/dockerexternal" "github.com/m3db/m3/src/x/errors" ) @@ -29,11 +32,13 @@ type inprocessM3Resources struct { coordinator resources.Coordinator dbNodes resources.Nodes aggregators resources.Aggregators + etcd *dockerexternal.EtcdNode } // ResourceOptions are the options for creating new // resources.M3Resources. type ResourceOptions struct { + Etcd *dockerexternal.EtcdNode Coordinator resources.Coordinator DBNodes resources.Nodes Aggregators resources.Aggregators @@ -43,6 +48,7 @@ type ResourceOptions struct { // backed by in-process implementations of the M3 components. func NewM3Resources(options ResourceOptions) resources.M3Resources { return &inprocessM3Resources{ + etcd: options.Etcd, coordinator: options.Coordinator, dbNodes: options.DBNodes, aggregators: options.Aggregators, @@ -73,6 +79,11 @@ func (i *inprocessM3Resources) Cleanup() error { err = err.Add(a.Close()) } + if i.etcd != nil { + ctx := context.TODO() + err = err.Add(i.etcd.Close(ctx)) + } + return err.FinalError() } diff --git a/src/integration/resources/options.go b/src/integration/resources/options.go index d980c154f7..88eeddfe95 100644 --- a/src/integration/resources/options.go +++ b/src/integration/resources/options.go @@ -1,3 +1,23 @@ +// Copyright (c) 2022 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package resources import ( @@ -8,6 +28,9 @@ import ( // ClusterOptions contains options for spinning up a new M3 cluster // composed of in-process components. type ClusterOptions struct { + // EtcdEndpoints if provided, will be used directly instead of spinning up a dedicated etcd node for the cluster. + // By default, NewClusterFromSpecification will spin up and manage an etcd node itself. + EtcdEndpoints []string // DBNode contains cluster options for spinning up dbnodes. DBNode *DBNodeClusterOptions // Aggregator is the optional cluster options for spinning up aggregators. diff --git a/src/query/config/m3coordinator-local-docker-etcd.yml b/src/query/config/m3coordinator-local-docker-etcd.yml new file mode 100644 index 0000000000..c25dff706c --- /dev/null +++ b/src/query/config/m3coordinator-local-docker-etcd.yml @@ -0,0 +1,16 @@ +clusters: + - namespaces: + - namespace: default + type: unaggregated + retention: 48h + client: + config: + service: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - http://etcd:2379