From e40f0b2ff7e8c0c759345488e2e598364d39dd70 Mon Sep 17 00:00:00 2001 From: junjie Date: Tue, 20 Feb 2024 09:38:27 +0700 Subject: [PATCH] fixed quick-compose validation test --- ...er-compose-without-neo4j-m1.quickstart.yml | 228 +++++++++--------- ...ocker-compose-without-neo4j.quickstart.yml | 228 +++++++++--------- 2 files changed, 228 insertions(+), 228 deletions(-) diff --git a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml index fa810bdeee739..f6284edc83648 100644 --- a/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j-m1.quickstart.yml @@ -8,14 +8,14 @@ services: zookeeper: condition: service_healthy environment: - - KAFKA_BROKER_ID=1 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 - - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m - - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 + - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m + - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false healthcheck: interval: 1s retries: 5 @@ -25,27 +25,27 @@ services: hostname: broker image: confluentinc/cp-kafka:7.4.0 ports: - - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 + - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 volumes: - - broker:/var/lib/kafka/data/ + - broker:/var/lib/kafka/data/ datahub-actions: container_name: datahub-actions depends_on: datahub-gms: condition: service_healthy environment: - - ACTIONS_CONFIG=${ACTIONS_CONFIG:-} - - ACTIONS_EXTRA_PACKAGES=${ACTIONS_EXTRA_PACKAGES:-} - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - DATAHUB_GMS_PROTOCOL=http - - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT - - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 - - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 - - SCHEMA_REGISTRY_URL=http://schema-registry:8081 + - ACTIONS_CONFIG=${ACTIONS_CONFIG:-} + - ACTIONS_EXTRA_PACKAGES=${ACTIONS_EXTRA_PACKAGES:-} + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - DATAHUB_GMS_PROTOCOL=http + - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system + - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT + - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 + - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 + - SCHEMA_REGISTRY_URL=http://schema-registry:8081 hostname: actions image: ${DATAHUB_ACTIONS_IMAGE:-acryldata/datahub-actions}:${ACTIONS_VERSION:-head} datahub-frontend-react: @@ -54,52 +54,52 @@ services: datahub-gms: condition: service_healthy environment: - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - DATAHUB_SECRET=YouKnowNothing - - DATAHUB_APP_VERSION=1.0 - - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB - - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 - - ELASTIC_CLIENT_HOST=elasticsearch - - ELASTIC_CLIENT_PORT=9200 + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - DATAHUB_SECRET=YouKnowNothing + - DATAHUB_APP_VERSION=1.0 + - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB + - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 + - ELASTIC_CLIENT_HOST=elasticsearch + - ELASTIC_CLIENT_PORT=9200 hostname: datahub-frontend-react image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} ports: - - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 + - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 volumes: - - ${HOME}/.datahub/plugins:/etc/datahub/plugins + - ${HOME}/.datahub/plugins:/etc/datahub/plugins datahub-gms: container_name: datahub-gms depends_on: datahub-upgrade: condition: service_completed_successfully environment: - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - EBEAN_DATASOURCE_HOST=mysql:3306 - - EBEAN_DATASOURCE_PASSWORD=datahub - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - - EBEAN_DATASOURCE_USERNAME=datahub - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ELASTICSEARCH_PORT=9200 - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - ENTITY_SERVICE_ENABLE_RETENTION=true - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - GRAPH_SERVICE_IMPL=elasticsearch - - JAVA_OPTS=-Xms1g -Xmx1g - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - MAE_CONSUMER_ENABLED=true - - MCE_CONSUMER_ENABLED=true - - PE_CONSUMER_ENABLED=true - - UI_INGESTION_ENABLED=true + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - EBEAN_DATASOURCE_HOST=mysql:3306 + - EBEAN_DATASOURCE_PASSWORD=datahub + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 + - EBEAN_DATASOURCE_USERNAME=datahub + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - ELASTICSEARCH_PORT=9200 + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - ENTITY_SERVICE_ENABLE_RETENTION=true + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - GRAPH_SERVICE_IMPL=elasticsearch + - JAVA_OPTS=-Xms1g -Xmx1g + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - MAE_CONSUMER_ENABLED=true + - MCE_CONSUMER_ENABLED=true + - PE_CONSUMER_ENABLED=true + - UI_INGESTION_ENABLED=true healthcheck: interval: 1s retries: 3 @@ -109,13 +109,13 @@ services: hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: - - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 + - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 volumes: - - ${HOME}/.datahub/plugins:/etc/datahub/plugins + - ${HOME}/.datahub/plugins:/etc/datahub/plugins datahub-upgrade: command: - - -u - - SystemUpdate + - -u + - SystemUpdate container_name: datahub-upgrade depends_on: elasticsearch-setup: @@ -125,23 +125,23 @@ services: mysql-setup: condition: service_completed_successfully environment: - - EBEAN_DATASOURCE_USERNAME=datahub - - EBEAN_DATASOURCE_PASSWORD=datahub - - EBEAN_DATASOURCE_HOST=mysql:3306 - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false - - GRAPH_SERVICE_IMPL=elasticsearch - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - BACKFILL_BROWSE_PATHS_V2=true + - EBEAN_DATASOURCE_USERNAME=datahub + - EBEAN_DATASOURCE_PASSWORD=datahub + - EBEAN_DATASOURCE_HOST=mysql:3306 + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false + - GRAPH_SERVICE_IMPL=elasticsearch + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - BACKFILL_BROWSE_PATHS_V2=true hostname: datahub-upgrade image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head} labels: @@ -153,10 +153,10 @@ services: limits: memory: 1G environment: - - discovery.type=single-node - - ${XPACK_SECURITY_ENABLED:-xpack.security.enabled=false} - - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true - - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true + - discovery.type=single-node + - ${XPACK_SECURITY_ENABLED:-xpack.security.enabled=false} + - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true healthcheck: interval: 1s retries: 3 @@ -166,20 +166,20 @@ services: hostname: elasticsearch image: ${DATAHUB_SEARCH_IMAGE:-elasticsearch}:${DATAHUB_SEARCH_TAG:-7.10.1} ports: - - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 + - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 volumes: - - esdata:/usr/share/elasticsearch/data + - esdata:/usr/share/elasticsearch/data elasticsearch-setup: container_name: elasticsearch-setup depends_on: elasticsearch: condition: service_healthy environment: - - ELASTICSEARCH_USE_SSL=${ELASTICSEARCH_USE_SSL:-false} - - USE_AWS_ELASTICSEARCH=${USE_AWS_ELASTICSEARCH:-false} - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ELASTICSEARCH_PROTOCOL=http + - ELASTICSEARCH_USE_SSL=${ELASTICSEARCH_USE_SSL:-false} + - USE_AWS_ELASTICSEARCH=${USE_AWS_ELASTICSEARCH:-false} + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ELASTICSEARCH_PROTOCOL=http hostname: elasticsearch-setup image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} labels: @@ -192,10 +192,10 @@ services: schema-registry: condition: service_healthy environment: - - DATAHUB_PRECREATE_TOPICS=${DATAHUB_PRECREATE_TOPICS:-false} - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - USE_CONFLUENT_SCHEMA_REGISTRY=TRUE + - DATAHUB_PRECREATE_TOPICS=${DATAHUB_PRECREATE_TOPICS:-false} + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - USE_CONFLUENT_SCHEMA_REGISTRY=TRUE hostname: kafka-setup image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} labels: @@ -204,10 +204,10 @@ services: command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password container_name: mysql environment: - - MYSQL_DATABASE=datahub - - MYSQL_USER=datahub - - MYSQL_PASSWORD=datahub - - MYSQL_ROOT_PASSWORD=datahub + - MYSQL_DATABASE=datahub + - MYSQL_USER=datahub + - MYSQL_PASSWORD=datahub + - MYSQL_ROOT_PASSWORD=datahub healthcheck: interval: 1s retries: 5 @@ -217,22 +217,22 @@ services: hostname: mysql image: mariadb:10.5.8 ports: - - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 + - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 restart: on-failure volumes: - - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql - - mysqldata:/var/lib/mysql + - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql + - mysqldata:/var/lib/mysql mysql-setup: container_name: mysql-setup depends_on: mysql: condition: service_healthy environment: - - MYSQL_HOST=mysql - - MYSQL_PORT=3306 - - MYSQL_USERNAME=datahub - - MYSQL_PASSWORD=datahub - - DATAHUB_DB_NAME=datahub + - MYSQL_HOST=mysql + - MYSQL_PORT=3306 + - MYSQL_USERNAME=datahub + - MYSQL_PASSWORD=datahub + - DATAHUB_DB_NAME=datahub hostname: mysql-setup image: ${DATAHUB_MYSQL_SETUP_IMAGE:-acryldata/datahub-mysql-setup}:${DATAHUB_VERSION:-head} labels: @@ -243,9 +243,9 @@ services: broker: condition: service_healthy environment: - - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - - SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT - - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 + - SCHEMA_REGISTRY_HOST_NAME=schemaregistry + - SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT + - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 healthcheck: interval: 1s retries: 3 @@ -255,12 +255,12 @@ services: hostname: schema-registry image: confluentinc/cp-schema-registry:7.4.0 ports: - - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 + - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: container_name: zookeeper environment: - - ZOOKEEPER_CLIENT_PORT=2181 - - ZOOKEEPER_TICK_TIME=2000 + - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_TICK_TIME=2000 healthcheck: interval: 5s retries: 3 @@ -270,9 +270,9 @@ services: hostname: zookeeper image: confluentinc/cp-zookeeper:7.4.0 ports: - - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 + - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: - - zkdata:/var/lib/zookeeper + - zkdata:/var/lib/zookeeper version: '3.9' volumes: broker: null diff --git a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml index c3e0802d8831a..4e3503e35c0db 100644 --- a/docker/quickstart/docker-compose-without-neo4j.quickstart.yml +++ b/docker/quickstart/docker-compose-without-neo4j.quickstart.yml @@ -8,14 +8,14 @@ services: zookeeper: condition: service_healthy environment: - - KAFKA_BROKER_ID=1 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT - - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 - - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 - - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 - - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m - - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false + - KAFKA_BROKER_ID=1 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092 + - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 + - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0 + - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m + - KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE=false healthcheck: interval: 1s retries: 5 @@ -25,27 +25,27 @@ services: hostname: broker image: confluentinc/cp-kafka:7.4.0 ports: - - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 + - ${DATAHUB_MAPPED_KAFKA_BROKER_PORT:-9092}:9092 volumes: - - broker:/var/lib/kafka/data/ + - broker:/var/lib/kafka/data/ datahub-actions: container_name: datahub-actions depends_on: datahub-gms: condition: service_healthy environment: - - ACTIONS_CONFIG=${ACTIONS_CONFIG:-} - - ACTIONS_EXTRA_PACKAGES=${ACTIONS_EXTRA_PACKAGES:-} - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - DATAHUB_GMS_PROTOCOL=http - - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system - - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT - - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 - - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 - - SCHEMA_REGISTRY_URL=http://schema-registry:8081 + - ACTIONS_CONFIG=${ACTIONS_CONFIG:-} + - ACTIONS_EXTRA_PACKAGES=${ACTIONS_EXTRA_PACKAGES:-} + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - DATAHUB_GMS_PROTOCOL=http + - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system + - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT + - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4 + - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1 + - SCHEMA_REGISTRY_URL=http://schema-registry:8081 hostname: actions image: ${DATAHUB_ACTIONS_IMAGE:-acryldata/datahub-actions}:${ACTIONS_VERSION:-head} datahub-frontend-react: @@ -54,52 +54,52 @@ services: datahub-gms: condition: service_healthy environment: - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - DATAHUB_SECRET=YouKnowNothing - - DATAHUB_APP_VERSION=1.0 - - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB - - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 - - ELASTIC_CLIENT_HOST=elasticsearch - - ELASTIC_CLIENT_PORT=9200 + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - DATAHUB_SECRET=YouKnowNothing + - DATAHUB_APP_VERSION=1.0 + - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB + - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1 + - ELASTIC_CLIENT_HOST=elasticsearch + - ELASTIC_CLIENT_PORT=9200 hostname: datahub-frontend-react image: ${DATAHUB_FRONTEND_IMAGE:-linkedin/datahub-frontend-react}:${DATAHUB_VERSION:-head} ports: - - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 + - ${DATAHUB_MAPPED_FRONTEND_PORT:-9002}:9002 volumes: - - ${HOME}/.datahub/plugins:/etc/datahub/plugins + - ${HOME}/.datahub/plugins:/etc/datahub/plugins datahub-gms: container_name: datahub-gms depends_on: datahub-upgrade: condition: service_completed_successfully environment: - - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} - - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} - - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - EBEAN_DATASOURCE_HOST=mysql:3306 - - EBEAN_DATASOURCE_PASSWORD=datahub - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - - EBEAN_DATASOURCE_USERNAME=datahub - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ELASTICSEARCH_PORT=9200 - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - ENTITY_SERVICE_ENABLE_RETENTION=true - - ES_BULK_REFRESH_POLICY=WAIT_UNTIL - - GRAPH_SERVICE_DIFF_MODE_ENABLED=true - - GRAPH_SERVICE_IMPL=elasticsearch - - JAVA_OPTS=-Xms1g -Xmx1g - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - MAE_CONSUMER_ENABLED=true - - MCE_CONSUMER_ENABLED=true - - PE_CONSUMER_ENABLED=true - - UI_INGESTION_ENABLED=true + - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart} + - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true} + - DATAHUB_UPGRADE_HISTORY_KAFKA_CONSUMER_GROUP_ID=generic-duhe-consumer-job-client-gms + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - EBEAN_DATASOURCE_HOST=mysql:3306 + - EBEAN_DATASOURCE_PASSWORD=datahub + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 + - EBEAN_DATASOURCE_USERNAME=datahub + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - ELASTICSEARCH_PORT=9200 + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - ENTITY_SERVICE_ENABLE_RETENTION=true + - ES_BULK_REFRESH_POLICY=WAIT_UNTIL + - GRAPH_SERVICE_DIFF_MODE_ENABLED=true + - GRAPH_SERVICE_IMPL=elasticsearch + - JAVA_OPTS=-Xms1g -Xmx1g + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - MAE_CONSUMER_ENABLED=true + - MCE_CONSUMER_ENABLED=true + - PE_CONSUMER_ENABLED=true + - UI_INGESTION_ENABLED=true healthcheck: interval: 1s retries: 3 @@ -109,13 +109,13 @@ services: hostname: datahub-gms image: ${DATAHUB_GMS_IMAGE:-linkedin/datahub-gms}:${DATAHUB_VERSION:-head} ports: - - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 + - ${DATAHUB_MAPPED_GMS_PORT:-8080}:8080 volumes: - - ${HOME}/.datahub/plugins:/etc/datahub/plugins + - ${HOME}/.datahub/plugins:/etc/datahub/plugins datahub-upgrade: command: - - -u - - SystemUpdate + - -u + - SystemUpdate container_name: datahub-upgrade depends_on: elasticsearch-setup: @@ -125,23 +125,23 @@ services: mysql-setup: condition: service_completed_successfully environment: - - EBEAN_DATASOURCE_USERNAME=datahub - - EBEAN_DATASOURCE_PASSWORD=datahub - - EBEAN_DATASOURCE_HOST=mysql:3306 - - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 - - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true - - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true - - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false - - GRAPH_SERVICE_IMPL=elasticsearch - - DATAHUB_GMS_HOST=datahub-gms - - DATAHUB_GMS_PORT=8080 - - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml - - BACKFILL_BROWSE_PATHS_V2=true + - EBEAN_DATASOURCE_USERNAME=datahub + - EBEAN_DATASOURCE_PASSWORD=datahub + - EBEAN_DATASOURCE_HOST=mysql:3306 + - EBEAN_DATASOURCE_URL=jdbc:mysql://mysql:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8 + - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081 + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ELASTICSEARCH_INDEX_BUILDER_MAPPINGS_REINDEX=true + - ELASTICSEARCH_INDEX_BUILDER_SETTINGS_REINDEX=true + - ELASTICSEARCH_BUILD_INDICES_CLONE_INDICES=false + - GRAPH_SERVICE_IMPL=elasticsearch + - DATAHUB_GMS_HOST=datahub-gms + - DATAHUB_GMS_PORT=8080 + - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml + - BACKFILL_BROWSE_PATHS_V2=true hostname: datahub-upgrade image: ${DATAHUB_UPGRADE_IMAGE:-acryldata/datahub-upgrade}:${DATAHUB_VERSION:-head} labels: @@ -153,10 +153,10 @@ services: limits: memory: 1G environment: - - discovery.type=single-node - - ${XPACK_SECURITY_ENABLED:-xpack.security.enabled=false} - - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true - - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true + - discovery.type=single-node + - ${XPACK_SECURITY_ENABLED:-xpack.security.enabled=false} + - ES_JAVA_OPTS=-Xms256m -Xmx512m -Dlog4j2.formatMsgNoLookups=true + - OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true healthcheck: interval: 1s retries: 3 @@ -166,20 +166,20 @@ services: hostname: elasticsearch image: ${DATAHUB_SEARCH_IMAGE:-elasticsearch}:${DATAHUB_SEARCH_TAG:-7.10.1} ports: - - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 + - ${DATAHUB_MAPPED_ELASTIC_PORT:-9200}:9200 volumes: - - esdata:/usr/share/elasticsearch/data + - esdata:/usr/share/elasticsearch/data elasticsearch-setup: container_name: elasticsearch-setup depends_on: elasticsearch: condition: service_healthy environment: - - ELASTICSEARCH_USE_SSL=${ELASTICSEARCH_USE_SSL:-false} - - USE_AWS_ELASTICSEARCH=${USE_AWS_ELASTICSEARCH:-false} - - ELASTICSEARCH_HOST=elasticsearch - - ELASTICSEARCH_PORT=9200 - - ELASTICSEARCH_PROTOCOL=http + - ELASTICSEARCH_USE_SSL=${ELASTICSEARCH_USE_SSL:-false} + - USE_AWS_ELASTICSEARCH=${USE_AWS_ELASTICSEARCH:-false} + - ELASTICSEARCH_HOST=elasticsearch + - ELASTICSEARCH_PORT=9200 + - ELASTICSEARCH_PROTOCOL=http hostname: elasticsearch-setup image: ${DATAHUB_ELASTIC_SETUP_IMAGE:-linkedin/datahub-elasticsearch-setup}:${DATAHUB_VERSION:-head} labels: @@ -192,10 +192,10 @@ services: schema-registry: condition: service_healthy environment: - - DATAHUB_PRECREATE_TOPICS=${DATAHUB_PRECREATE_TOPICS:-false} - - KAFKA_BOOTSTRAP_SERVER=broker:29092 - - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 - - USE_CONFLUENT_SCHEMA_REGISTRY=TRUE + - DATAHUB_PRECREATE_TOPICS=${DATAHUB_PRECREATE_TOPICS:-false} + - KAFKA_BOOTSTRAP_SERVER=broker:29092 + - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181 + - USE_CONFLUENT_SCHEMA_REGISTRY=TRUE hostname: kafka-setup image: ${DATAHUB_KAFKA_SETUP_IMAGE:-linkedin/datahub-kafka-setup}:${DATAHUB_VERSION:-head} labels: @@ -204,10 +204,10 @@ services: command: --character-set-server=utf8mb4 --collation-server=utf8mb4_bin --default-authentication-plugin=mysql_native_password container_name: mysql environment: - - MYSQL_DATABASE=datahub - - MYSQL_USER=datahub - - MYSQL_PASSWORD=datahub - - MYSQL_ROOT_PASSWORD=datahub + - MYSQL_DATABASE=datahub + - MYSQL_USER=datahub + - MYSQL_PASSWORD=datahub + - MYSQL_ROOT_PASSWORD=datahub healthcheck: interval: 1s retries: 5 @@ -217,22 +217,22 @@ services: hostname: mysql image: mysql:5.7 ports: - - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 + - ${DATAHUB_MAPPED_MYSQL_PORT:-3306}:3306 restart: on-failure volumes: - - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql - - mysqldata:/var/lib/mysql + - ../mysql/init.sql:/docker-entrypoint-initdb.d/init.sql + - mysqldata:/var/lib/mysql mysql-setup: container_name: mysql-setup depends_on: mysql: condition: service_healthy environment: - - MYSQL_HOST=mysql - - MYSQL_PORT=3306 - - MYSQL_USERNAME=datahub - - MYSQL_PASSWORD=datahub - - DATAHUB_DB_NAME=datahub + - MYSQL_HOST=mysql + - MYSQL_PORT=3306 + - MYSQL_USERNAME=datahub + - MYSQL_PASSWORD=datahub + - DATAHUB_DB_NAME=datahub hostname: mysql-setup image: ${DATAHUB_MYSQL_SETUP_IMAGE:-acryldata/datahub-mysql-setup}:${DATAHUB_VERSION:-head} labels: @@ -243,9 +243,9 @@ services: broker: condition: service_healthy environment: - - SCHEMA_REGISTRY_HOST_NAME=schemaregistry - - SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT - - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 + - SCHEMA_REGISTRY_HOST_NAME=schemaregistry + - SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL=PLAINTEXT + - SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=broker:29092 healthcheck: interval: 1s retries: 3 @@ -255,12 +255,12 @@ services: hostname: schema-registry image: confluentinc/cp-schema-registry:7.4.0 ports: - - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 + - ${DATAHUB_MAPPED_SCHEMA_REGISTRY_PORT:-8081}:8081 zookeeper: container_name: zookeeper environment: - - ZOOKEEPER_CLIENT_PORT=2181 - - ZOOKEEPER_TICK_TIME=2000 + - ZOOKEEPER_CLIENT_PORT=2181 + - ZOOKEEPER_TICK_TIME=2000 healthcheck: interval: 5s retries: 3 @@ -270,9 +270,9 @@ services: hostname: zookeeper image: confluentinc/cp-zookeeper:7.4.0 ports: - - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 + - ${DATAHUB_MAPPED_ZK_PORT:-2181}:2181 volumes: - - zkdata:/var/lib/zookeeper + - zkdata:/var/lib/zookeeper version: '3.9' volumes: broker: null