Release 4.0.0

CluedIn-io · Feb 7, 2024 · d09c87e · d09c87e
1 parent afb1393
commit d09c87e
Show file tree

Hide file tree

Showing 19 changed files with 7,089 additions and 85 deletions.
diff --git a/docker/compose/docker-compose.db.yml b/docker/compose/docker-compose.db.yml
@@ -1,7 +1,7 @@
 version: '3.7'
 services:
     sqlserver:
-        image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cluedin/sqlserver:${CLUEDIN_SQLSERVER_TAG}
+        image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cluedin/sqlserver-home:${CLUEDIN_SQLSERVER_TAG}
         entrypoint: ["sh","-c","mkdir -p /mssql-data/data/ && /init/init.sh"]
         ports:
             - "${CLUEDIN_SQLSERVER_LOCALPORT:-1433}:1433"

diff --git a/docker/compose/docker-compose.micro.yml b/docker/compose/docker-compose.micro.yml
@@ -15,55 +15,57 @@ services:
     - DbUserName=${CLUEDIN_SQLSERVER_USER}
     - DbName=${CLUEDIN_CLEAN_DBNAME:-DataStore.Db.MicroServices}
     - ForceTableCreation=${CLUEDIN_CLEAN_FORCETABLECREATION:-false}
-    - AnnotationURL=http://${CLUEDIN_ANNOTATION_HOST:-annotation}:${CLUEDIN_ANNOTATION_PORT:-8888}
-    - DataSourceURL=http://${CLUEDIN_DATASOURCE_HOST:-datasource}:${CLUEDIN_DATASOURCE_PORT:-8888}
+    - AnnotationURL=http://${CLUEDIN_DATASOURCE_HOST:-datasource}:${CLUEDIN_DATASOURCE_PORT:-9011}
+    - DataSourceURL=http://${CLUEDIN_DATASOURCE_HOST:-datasource}:${CLUEDIN_DATASOURCE_PORT:-9011}
     - NODE_ENV=${CLUEDIN_ENVIRONMENT}
+    - MAX_JSON_PAYLOAD_REST=${CLUEDIN_DATASOURCE_MAX_JSON_PAYLOAD_REST:-100000kb}
     ports:
       - "${CLUEDIN_CLEAN_LOCALPORT:-9009}:8888"
     depends_on:
     - elasticsearch
     - rabbitmq
     - sqlserver
     - seq
-    - annotation
     - datasource
     deploy:
       restart_policy:
         condition: ${CLUEDIN_COMMON_RESTART_POLICY:-unless-stopped}
         delay: 5s
 
-  annotation:
+  datasource:
     user: root
-    image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cluedin/cluedin-micro-annotation:${CLUEDIN_ANNOTATION_TAG}
+    image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cluedin/data-source:${CLUEDIN_DATASOURCE_TAG}
     environment:
-    - WEB_API=http://${CLUEDIN_SERVER_HOST:-server}:9000/
-    - ElasticURL=http://${CLUEDIN_ELASTIC_HOST:-elasticsearch}:9200
     - RabbitMQ_CS=amqp://${CLUEDIN_RABBITMQ_USERNAME:-cluedin}:${CLUEDIN_RABBITMQ_PASSWORD:-cluedin}@${CLUEDIN_RABBITMQ_HOST:-rabbitmq}:5672
+    - ElasticURL=http://${CLUEDIN_ELASTIC_HOST:-elasticsearch}:9200
+    - DbName=${CLUEDIN_DATASOURCE_DBNAME:-DataStore.Db.MicroServices}
     - DbHost=${CLUEDIN_SQLSERVER_HOST}
     - DbPassword=${CLUEDIN_SQLSERVER_PASS}
     - DbUserName=${CLUEDIN_SQLSERVER_USER}
-    - DbName=${CLUEDIN_ANNOTATION_DBNAME:-DataStore.Db.MicroServices}
     - SeqURL=http://${CLUEDIN_SEQ_HOST:-seq}:5341
     - SeqLoggingLevel=debug
-    - ForceTableCreation=${CLUEDIN_ANNOTATION_FORCETABLECREATION:-false}
-    - NODE_ENV=${CLUEDIN_ENVIRONMENT}
-    - MICRO_DATASOURCE_URL=http://${CLUEDIN_DATASOURCE_HOST:-datasource}:${CLUEDIN_DATASOURCE_PORT:-8888}
+    - ConsoleLoggingLevel=info
+    - WEB_API=http://${CLUEDIN_SERVER_HOST:-server}:9000
+    - DataSourceHost=${CLUEDIN_DATASOURCE_HOST:-datasource}
+    - DataSourcePort=${CLUEDIN_DATASOURCE_PORT:-9011}
+    - TMP_FILES_PATH=/shared
     ports:
-      - "${CLUEDIN_ANNOTATION_LOCALPORT:-9010}:8888"
+      - "${CLUEDIN_DATASOURCE_LOCALPORT:-9011}:9011"
     depends_on:
     - rabbitmq
     - sqlserver
     - seq
-    - datasource
+    - elasticsearch
     deploy:
       restart_policy:
         condition: ${CLUEDIN_COMMON_RESTART_POLICY:-unless-stopped}
         delay: 5s
+    volumes:
+      - datasource:/shared
 
-
-  datasource:
+  datasource-processing:
     user: root
-    image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cluedin/cluedin-micro-datasource:${CLUEDIN_DATASOURCE_TAG}
+    image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cluedin/data-source-processing:${CLUEDIN_DATASOURCEPROCESSING_TAG}
     environment:
     - RabbitMQ_CS=amqp://${CLUEDIN_RABBITMQ_USERNAME:-cluedin}:${CLUEDIN_RABBITMQ_PASSWORD:-cluedin}@${CLUEDIN_RABBITMQ_HOST:-rabbitmq}:5672
     - ElasticURL=http://${CLUEDIN_ELASTIC_HOST:-elasticsearch}:9200
@@ -73,16 +75,16 @@ services:
     - DbUserName=${CLUEDIN_SQLSERVER_USER}
     - SeqURL=http://${CLUEDIN_SEQ_HOST:-seq}:5341
     - SeqLoggingLevel=debug
-    - AnnotationURL=http://${CLUEDIN_ANNOTATION_HOST:-annotation}:${CLUEDIN_ANNOTATION_PORT:-8888}/
     - ConsoleLoggingLevel=info
-    - ForceTableCreation=${CLUEDIN_DATASOURCE_FORCETABLECREATION:-false}
-    - NODE_ENV=${CLUEDIN_ENVIRONMENT}
-    - WEB_API=http://${CLUEDIN_SERVER_HOST:-server}:9000/
+    - DataSourceProcessingHost=${CLUEDIN_DATASOURCEPROCESSING_HOST:-datasource-processing}
+    - DataSourceProcessingPort=${CLUEDIN_DATASOURCEPROCESSING_LOCALPORT:-5757}
+    - DataSourceProcessingNumOfDataLoadingWorkers=1
+    - DataSourceProcessingNumOfIngestionWorkers=1
+    - DataSourceProcessingNumOfMappingWorkers=1
+    - DataSourceProcessingNumOfFileRecoveryWorkers=1
+    - TMP_FILES_PATH=/shared
     ports:
-      - "${CLUEDIN_DATASOURCE_LOCALPORT:-9011}:8888"
-      - "${CLUEDIN_DATASOURCE_PROMETHEUS_ENDPOINT_METRICS_LOCALPORT:-9014}:9014"
-      - "${CLUEDIN_DATASOURCE_PROMETHEUS_FILE_METRICS_LOCALPORT:-9015}:9015"
-      - "${CLUEDIN_DATASOURCE_PROMETHEUS_SQL_METRICS_LOCALPORT:-9016}:9016"
+      - "${CLUEDIN_DATASOURCEPROCESSING_LOCALPORT:-5757}:8888"
     depends_on:
     - elasticsearch
     - rabbitmq
@@ -92,35 +94,38 @@ services:
       restart_policy:
         condition: ${CLUEDIN_COMMON_RESTART_POLICY:-unless-stopped}
         delay: 5s
+    volumes:
+      - datasource:/shared
 
-  submitter:
-    image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cluedin/cluedin-micro-submitter-node:${CLUEDIN_SUBMITTER_TAG}
+  datasource-submitter:
+    user: root
+    image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cluedin/data-source-submitter:${CLUEDIN_DATASOURCESUBMITTER_TAG}
     environment:
-      CluedIn_API_URL: ${CLUEDIN_SUBMITTER_CLUE_API_URL:-/api/v2/clue?save=true}
-      RabbitMQ_CS: "amqp://${CLUEDIN_RABBITMQ_USERNAME:-cluedin}:${CLUEDIN_RABBITMQ_PASSWORD:-cluedin}@${CLUEDIN_RABBITMQ_HOST:-rabbitmq}:5672"
-      Serilog__WriteTo__1__Name: Seq
-      Serilog__WriteTo__1__Args__serverUrl: http://${CLUEDIN_SEQ_HOST:-seq}:5341
-      Serilog__WriteTo__1__Args__controlLevelSwitch: "$$controlSwitch"
-      DbName: ${CLUEDIN_DATASOURCE_DBNAME:-DataStore.Db.MicroServices}
-      DbHost: ${CLUEDIN_SQLSERVER_HOST}
-      DbPassword: ${CLUEDIN_SQLSERVER_PASS}
-      DbUserName: ${CLUEDIN_SQLSERVER_USER}
-      MICRO_DATASOURCE_URL: http://${CLUEDIN_DATASOURCE_HOST:-datasource}:${CLUEDIN_DATASOURCE_PORT:-8888}
-      APP_API_URL: http://${CLUEDIN_SERVER_HOST:-server}:9007
-      WORKERS_NUMBER: 4
-      DISABLE_SUBMISSION_REPORT: 'false'
+    - RabbitMQ_CS=amqp://${CLUEDIN_RABBITMQ_USERNAME:-cluedin}:${CLUEDIN_RABBITMQ_PASSWORD:-cluedin}@${CLUEDIN_RABBITMQ_HOST:-rabbitmq}:5672
+    - ElasticURL=http://${CLUEDIN_ELASTIC_HOST:-elasticsearch}:9200
+    - DbName=${CLUEDIN_DATASOURCE_DBNAME:-DataStore.Db.MicroServices}
+    - DbHost=${CLUEDIN_SQLSERVER_HOST}
+    - DbPassword=${CLUEDIN_SQLSERVER_PASS}
+    - DbUserName=${CLUEDIN_SQLSERVER_USER}
+    - SeqURL=http://${CLUEDIN_SEQ_HOST:-seq}:5341
+    - SeqLoggingLevel=debug
+    - ConsoleLoggingLevel=info
+    - DataSourceSubmitterPort=${CLUEDIN_DATASOURCESUBMITTER_LOCALPORT:-9012}
+    - PUBLIC_WEB_API=http://${CLUEDIN_SERVER_HOST:-server}:9007
+    - CluedIn_API_URL=${CLUEDIN_DATASOURCESUBMITTER_CLUE_API_URL:-/api/v2/clue?save=true}
+    - DataSourceSubmitterHost=${CLUEDIN_DATASOURCESUBMITTER_HOST:-datasource-submitter}
+    - DataSourceSubmitterWORKERS_NUMBER=4
     ports:
-      - "${CLUEDIN_SUBMITTER_LOCALPORT:-9012}:8888"
+      - "${CLUEDIN_DATASOURCESUBMITTER_LOCALPORT:-9012}:8888"
     depends_on:
+    - elasticsearch
     - rabbitmq
+    - sqlserver
     - seq
-    - annotation
     deploy:
       restart_policy:
         condition: ${CLUEDIN_COMMON_RESTART_POLICY:-unless-stopped}
         delay: 5s
 
-  documentation:
-    image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cluedin/cluedin-micro-documentation:${CLUEDIN_DOCUMENTATION_TAG}
-    ports:
-      - "${CLUEDIN_DOCUMENTATION_LOCALPORT:-9021}:8888"
+volumes:
+  datasource:
diff --git a/docker/compose/docker-compose.server.yml b/docker/compose/docker-compose.server.yml
@@ -35,19 +35,21 @@ services:
       - CLUEDIN_CONNECTIONSTRINGS__SIGNALRSCALEOUT=amqp://${CLUEDIN_RABBITMQ_USERNAME:-cluedin}:${CLUEDIN_RABBITMQ_PASSWORD:-cluedin}@${CLUEDIN_RABBITMQ_HOST:-rabbitmq}:5672
 
       - CLUEDIN_CONNECTIONSTRINGS__AUTHENTICATIONSTORE=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.Authentication;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True
-      - CLUEDIN_CONNECTIONSTRINGS__AUDITLOG=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.AuditLog;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True
-      - CLUEDIN_CONNECTIONSTRINGS__BLOBSTORAGE=Data source=${CLUEDIN_SQLSERVER_HOST};Initial catalog=DataStore.Db.BlobStorage;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True
-      - CLUEDIN_CONNECTIONSTRINGS__CONFIGURATIONSTORE=Data source=${CLUEDIN_SQLSERVER_HOST};Initial catalog=DataStore.Db.Configuration;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True
-      - CLUEDIN_CONNECTIONSTRINGS__CLUEDINENTITIES=Data source=${CLUEDIN_SQLSERVER_HOST};Initial catalog=DataStore.Db.OpenCommunication;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True
-      - CLUEDIN_CONNECTIONSTRINGS__TOKENSTORE=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.TokenStore;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True
-      - CLUEDIN_CONNECTIONSTRINGS__TRAINING=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.Training;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True
-      - CLUEDIN_CONNECTIONSTRINGS__EXTERNALSEARCH=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.ExternalSearch;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True
-      - CLUEDIN_CONNECTIONSTRINGS__METRICS=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.Metrics;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True
-      - CLUEDIN_CONNECTIONSTRINGS__ML_LOGGING=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.ML-Logging;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True
-      - CLUEDIN_CONNECTIONSTRINGS__WEBAPPSTORE=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.WebApp;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True
-      - CLUEDIN_CONNECTIONSTRINGS__MICROSERVICES=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.MicroServices;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True
-      - CLUEDIN_CONNECTIONSTRINGS__CLEANCACHE=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.CleanCache;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True
-      - CLUEDIN_CONNECTIONSTRINGS__INVITATIONSTORE=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.WebApp;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True
+      - CLUEDIN_CONNECTIONSTRINGS__AUDITLOG=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.AuditLog;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True
+      - CLUEDIN_CONNECTIONSTRINGS__BLOBSTORAGE=Data source=${CLUEDIN_SQLSERVER_HOST};Initial catalog=DataStore.Db.BlobStorage;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True
+      - CLUEDIN_CONNECTIONSTRINGS__CONFIGURATIONSTORE=Data source=${CLUEDIN_SQLSERVER_HOST};Initial catalog=DataStore.Db.Configuration;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True
+      - CLUEDIN_CONNECTIONSTRINGS__CLUEDINENTITIES=Data source=${CLUEDIN_SQLSERVER_HOST};Initial catalog=DataStore.Db.OpenCommunication;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True
+      - CLUEDIN_CONNECTIONSTRINGS__TOKENSTORE=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.TokenStore;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True
+      - CLUEDIN_CONNECTIONSTRINGS__TRAINING=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.Training;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True
+      - CLUEDIN_CONNECTIONSTRINGS__EXTERNALSEARCH=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.ExternalSearch;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True
+      - CLUEDIN_CONNECTIONSTRINGS__METRICS=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.Metrics;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True
+      - CLUEDIN_CONNECTIONSTRINGS__LOCKING=Data source=${CLUEDIN_SQLSERVER_HOST};Initial catalog=DataStore.Db.OpenCommunication;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True
+      - CLUEDIN_CONNECTIONSTRINGS__ML_LOGGING=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.ML-Logging;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True
+      - CLUEDIN_CONNECTIONSTRINGS__WEBAPPSTORE=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.WebApp;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True
+      - CLUEDIN_CONNECTIONSTRINGS__MICROSERVICES=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.MicroServices;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True
+      - CLUEDIN_CONNECTIONSTRINGS__CLEANCACHE=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.CleanCache;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True
+      - CLUEDIN_CONNECTIONSTRINGS__INVITATIONSTORE=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.WebApp;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True
+      - CLUEDIN_CONNECTIONSTRINGS__STREAMLOG=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.StreamLog;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True
       - CLUEDIN_APPSETTINGS__AGENT_ENABLED=${CLUEDIN_AGENT_ENABLED}
       - CLUEDIN_APPSETTINGS__MODELS_ENABLED=${CLUEDIN_MODELS_ENABLED:-false}
       - DOTNET_ENVIRONMENT=${CLUEDIN_ENVIRONMENT}

diff --git a/docker/compose/docker-compose.ui.yml b/docker/compose/docker-compose.ui.yml
@@ -21,6 +21,7 @@ services:
     - REACT_APP_UI_PROTOCOL=http://
     - REACT_APP_UI_UPLOAD_LIMIT=${CLUEDIN_UI_UPLOAD_LIMIT:-1gb}
     - REACT_APP_UI_CLUEDIN_PUBLICAPI_URL=http://localhost:9007
+    - REACT_APP_UI_CLUEDIN_COPILOTHUB_URL=http://localhost:9000/hub/copilot
     depends_on:
     - server
     deploy:
@@ -34,9 +35,8 @@ services:
         - "AUTH_API=http://${CLUEDIN_SERVER_HOST:-server}:9001/"
         - "WEB_API=http://${CLUEDIN_SERVER_HOST:-server}:9000/"
         - "MICRO_CLEAN_URL=http://${CLUEDIN_CLEAN_HOST:-clean}:${CLUEDIN_CLEAN_PORT:-8888}"
-        - "MICRO_ANNOTATION_URL=http://${CLUEDIN_ANNOTATION_HOST:-annotation}:${CLUEDIN_ANNOTATION_PORT:-8888}"
-        - "MICRO_DATASOURCE_URL=http://${CLUEDIN_DATASOURCE_HOST:-datasource}:${CLUEDIN_DATASOURCE_PORT:-8888}"
-        - "MICRO_SUBMITTER_URL=http://${CLUEDIN_SUBMITTER_HOST:-submitter}:${CLUEDIN_SUBMITTER_PORT:-9012}"
+        - "MICRO_DATASOURCE_URL=http://${CLUEDIN_DATASOURCE_HOST:-datasource}:${CLUEDIN_DATASOURCE_PORT:-9011}"
+        - "MICRO_SUBMITTER_URL=http://${CLUEDIN_DATASOURCESUBMITTER_HOST:-datasource-submitter}:${CLUEDIN_DATASOURCESUBMITTER_PORT:-9012}"
         - "MICRO_DOCUMENTATION_URL=http://${CLUEDIN_DOCUMENTATION_HOST:-documentation}:${CLUEDIN_DOCUMENTATION_PORT:-8888}"
         - "PUBLIC_API=http://${CLUEDIN_SERVER_HOST:-server}:9007/"
         - "WEBHOOKS_URL=http://${CLUEDIN_SERVER_HOST:-server}:9006/"
@@ -57,12 +57,14 @@ services:
         - "ENGINE_ROOM_STREAM_DASHBOARD_ID=h0W9h3knk"
         - "ENGINE_ROOM_STREAM_DASHBOARD_NAME=streams"
         - "UPLOAD_LIMIT=${CLUEDIN_UI_UPLOAD_LIMIT:-1gb}"
+        - "ENABLE_DEVELOPMENT_FEATURES=true"
     ports:
     - "${CLUEDIN_GQL_LOCALPORT:-8888}:8888"
     depends_on:
-    - annotation
     - clean
     - datasource
+    - datasource-processing
+    - datasource-submitter
     - sqlserver
     deploy:
       restart_policy: