From d09c87e47197d6100fec39c562df4942bba1d11d Mon Sep 17 00:00:00 2001 From: CluedIn CI Date: Wed, 7 Feb 2024 16:17:20 +0000 Subject: [PATCH] Release 4.0.0 --- docker/compose/docker-compose.db.yml | 2 +- docker/compose/docker-compose.micro.yml | 95 +- docker/compose/docker-compose.server.yml | 28 +- docker/compose/docker-compose.ui.yml | 10 +- docker/compose/docker-compose.yml | 34 +- env/default/.env | 20 +- env/default/grafana/dashboards.yml | 8 +- env/default/grafana/datasource.yml | 26 +- .../profiling/boolean-vocabulary-key.json | 503 ++++ .../profiling/datetime-vocabulary-key.json | 588 +++++ .../profiling/float-vocabulary-key.json | 764 ++++++ .../grafana/profiling/general-stats.json | 2119 +++++++++++++++++ .../profiling/money-vocabulary-key.json | 767 ++++++ .../profiling/processing-over-time.json | 714 ++++++ .../profiling/text-vocabulary-key.json | 519 ++++ .../profiling/unmapped-vocabulary-key.json | 475 ++++ .../profiling/untyped-vocabulary-key.json | 497 ++++ env/default/prometheus/plugins.conf | 2 +- env/default/prometheus/rabbitmq.conf | 3 +- 19 files changed, 7089 insertions(+), 85 deletions(-) create mode 100644 env/default/grafana/profiling/boolean-vocabulary-key.json create mode 100644 env/default/grafana/profiling/datetime-vocabulary-key.json create mode 100644 env/default/grafana/profiling/float-vocabulary-key.json create mode 100644 env/default/grafana/profiling/general-stats.json create mode 100644 env/default/grafana/profiling/money-vocabulary-key.json create mode 100644 env/default/grafana/profiling/processing-over-time.json create mode 100644 env/default/grafana/profiling/text-vocabulary-key.json create mode 100644 env/default/grafana/profiling/unmapped-vocabulary-key.json create mode 100644 env/default/grafana/profiling/untyped-vocabulary-key.json diff --git a/docker/compose/docker-compose.db.yml b/docker/compose/docker-compose.db.yml index 746503b..9ebcf74 100644 --- a/docker/compose/docker-compose.db.yml +++ b/docker/compose/docker-compose.db.yml @@ -1,7 +1,7 @@ version: '3.7' services: sqlserver: - image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cluedin/sqlserver:${CLUEDIN_SQLSERVER_TAG} + image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cluedin/sqlserver-home:${CLUEDIN_SQLSERVER_TAG} entrypoint: ["sh","-c","mkdir -p /mssql-data/data/ && /init/init.sh"] ports: - "${CLUEDIN_SQLSERVER_LOCALPORT:-1433}:1433" diff --git a/docker/compose/docker-compose.micro.yml b/docker/compose/docker-compose.micro.yml index a9833cd..2f88a24 100644 --- a/docker/compose/docker-compose.micro.yml +++ b/docker/compose/docker-compose.micro.yml @@ -15,9 +15,10 @@ services: - DbUserName=${CLUEDIN_SQLSERVER_USER} - DbName=${CLUEDIN_CLEAN_DBNAME:-DataStore.Db.MicroServices} - ForceTableCreation=${CLUEDIN_CLEAN_FORCETABLECREATION:-false} - - AnnotationURL=http://${CLUEDIN_ANNOTATION_HOST:-annotation}:${CLUEDIN_ANNOTATION_PORT:-8888} - - DataSourceURL=http://${CLUEDIN_DATASOURCE_HOST:-datasource}:${CLUEDIN_DATASOURCE_PORT:-8888} + - AnnotationURL=http://${CLUEDIN_DATASOURCE_HOST:-datasource}:${CLUEDIN_DATASOURCE_PORT:-9011} + - DataSourceURL=http://${CLUEDIN_DATASOURCE_HOST:-datasource}:${CLUEDIN_DATASOURCE_PORT:-9011} - NODE_ENV=${CLUEDIN_ENVIRONMENT} + - MAX_JSON_PAYLOAD_REST=${CLUEDIN_DATASOURCE_MAX_JSON_PAYLOAD_REST:-100000kb} ports: - "${CLUEDIN_CLEAN_LOCALPORT:-9009}:8888" depends_on: @@ -25,45 +26,46 @@ services: - rabbitmq - sqlserver - seq - - annotation - datasource deploy: restart_policy: condition: ${CLUEDIN_COMMON_RESTART_POLICY:-unless-stopped} delay: 5s - annotation: + datasource: user: root - image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cluedin/cluedin-micro-annotation:${CLUEDIN_ANNOTATION_TAG} + image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cluedin/data-source:${CLUEDIN_DATASOURCE_TAG} environment: - - WEB_API=http://${CLUEDIN_SERVER_HOST:-server}:9000/ - - ElasticURL=http://${CLUEDIN_ELASTIC_HOST:-elasticsearch}:9200 - RabbitMQ_CS=amqp://${CLUEDIN_RABBITMQ_USERNAME:-cluedin}:${CLUEDIN_RABBITMQ_PASSWORD:-cluedin}@${CLUEDIN_RABBITMQ_HOST:-rabbitmq}:5672 + - ElasticURL=http://${CLUEDIN_ELASTIC_HOST:-elasticsearch}:9200 + - DbName=${CLUEDIN_DATASOURCE_DBNAME:-DataStore.Db.MicroServices} - DbHost=${CLUEDIN_SQLSERVER_HOST} - DbPassword=${CLUEDIN_SQLSERVER_PASS} - DbUserName=${CLUEDIN_SQLSERVER_USER} - - DbName=${CLUEDIN_ANNOTATION_DBNAME:-DataStore.Db.MicroServices} - SeqURL=http://${CLUEDIN_SEQ_HOST:-seq}:5341 - SeqLoggingLevel=debug - - ForceTableCreation=${CLUEDIN_ANNOTATION_FORCETABLECREATION:-false} - - NODE_ENV=${CLUEDIN_ENVIRONMENT} - - MICRO_DATASOURCE_URL=http://${CLUEDIN_DATASOURCE_HOST:-datasource}:${CLUEDIN_DATASOURCE_PORT:-8888} + - ConsoleLoggingLevel=info + - WEB_API=http://${CLUEDIN_SERVER_HOST:-server}:9000 + - DataSourceHost=${CLUEDIN_DATASOURCE_HOST:-datasource} + - DataSourcePort=${CLUEDIN_DATASOURCE_PORT:-9011} + - TMP_FILES_PATH=/shared ports: - - "${CLUEDIN_ANNOTATION_LOCALPORT:-9010}:8888" + - "${CLUEDIN_DATASOURCE_LOCALPORT:-9011}:9011" depends_on: - rabbitmq - sqlserver - seq - - datasource + - elasticsearch deploy: restart_policy: condition: ${CLUEDIN_COMMON_RESTART_POLICY:-unless-stopped} delay: 5s + volumes: + - datasource:/shared - - datasource: + datasource-processing: user: root - image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cluedin/cluedin-micro-datasource:${CLUEDIN_DATASOURCE_TAG} + image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cluedin/data-source-processing:${CLUEDIN_DATASOURCEPROCESSING_TAG} environment: - RabbitMQ_CS=amqp://${CLUEDIN_RABBITMQ_USERNAME:-cluedin}:${CLUEDIN_RABBITMQ_PASSWORD:-cluedin}@${CLUEDIN_RABBITMQ_HOST:-rabbitmq}:5672 - ElasticURL=http://${CLUEDIN_ELASTIC_HOST:-elasticsearch}:9200 @@ -73,16 +75,16 @@ services: - DbUserName=${CLUEDIN_SQLSERVER_USER} - SeqURL=http://${CLUEDIN_SEQ_HOST:-seq}:5341 - SeqLoggingLevel=debug - - AnnotationURL=http://${CLUEDIN_ANNOTATION_HOST:-annotation}:${CLUEDIN_ANNOTATION_PORT:-8888}/ - ConsoleLoggingLevel=info - - ForceTableCreation=${CLUEDIN_DATASOURCE_FORCETABLECREATION:-false} - - NODE_ENV=${CLUEDIN_ENVIRONMENT} - - WEB_API=http://${CLUEDIN_SERVER_HOST:-server}:9000/ + - DataSourceProcessingHost=${CLUEDIN_DATASOURCEPROCESSING_HOST:-datasource-processing} + - DataSourceProcessingPort=${CLUEDIN_DATASOURCEPROCESSING_LOCALPORT:-5757} + - DataSourceProcessingNumOfDataLoadingWorkers=1 + - DataSourceProcessingNumOfIngestionWorkers=1 + - DataSourceProcessingNumOfMappingWorkers=1 + - DataSourceProcessingNumOfFileRecoveryWorkers=1 + - TMP_FILES_PATH=/shared ports: - - "${CLUEDIN_DATASOURCE_LOCALPORT:-9011}:8888" - - "${CLUEDIN_DATASOURCE_PROMETHEUS_ENDPOINT_METRICS_LOCALPORT:-9014}:9014" - - "${CLUEDIN_DATASOURCE_PROMETHEUS_FILE_METRICS_LOCALPORT:-9015}:9015" - - "${CLUEDIN_DATASOURCE_PROMETHEUS_SQL_METRICS_LOCALPORT:-9016}:9016" + - "${CLUEDIN_DATASOURCEPROCESSING_LOCALPORT:-5757}:8888" depends_on: - elasticsearch - rabbitmq @@ -92,35 +94,38 @@ services: restart_policy: condition: ${CLUEDIN_COMMON_RESTART_POLICY:-unless-stopped} delay: 5s + volumes: + - datasource:/shared - submitter: - image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cluedin/cluedin-micro-submitter-node:${CLUEDIN_SUBMITTER_TAG} + datasource-submitter: + user: root + image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cluedin/data-source-submitter:${CLUEDIN_DATASOURCESUBMITTER_TAG} environment: - CluedIn_API_URL: ${CLUEDIN_SUBMITTER_CLUE_API_URL:-/api/v2/clue?save=true} - RabbitMQ_CS: "amqp://${CLUEDIN_RABBITMQ_USERNAME:-cluedin}:${CLUEDIN_RABBITMQ_PASSWORD:-cluedin}@${CLUEDIN_RABBITMQ_HOST:-rabbitmq}:5672" - Serilog__WriteTo__1__Name: Seq - Serilog__WriteTo__1__Args__serverUrl: http://${CLUEDIN_SEQ_HOST:-seq}:5341 - Serilog__WriteTo__1__Args__controlLevelSwitch: "$$controlSwitch" - DbName: ${CLUEDIN_DATASOURCE_DBNAME:-DataStore.Db.MicroServices} - DbHost: ${CLUEDIN_SQLSERVER_HOST} - DbPassword: ${CLUEDIN_SQLSERVER_PASS} - DbUserName: ${CLUEDIN_SQLSERVER_USER} - MICRO_DATASOURCE_URL: http://${CLUEDIN_DATASOURCE_HOST:-datasource}:${CLUEDIN_DATASOURCE_PORT:-8888} - APP_API_URL: http://${CLUEDIN_SERVER_HOST:-server}:9007 - WORKERS_NUMBER: 4 - DISABLE_SUBMISSION_REPORT: 'false' + - RabbitMQ_CS=amqp://${CLUEDIN_RABBITMQ_USERNAME:-cluedin}:${CLUEDIN_RABBITMQ_PASSWORD:-cluedin}@${CLUEDIN_RABBITMQ_HOST:-rabbitmq}:5672 + - ElasticURL=http://${CLUEDIN_ELASTIC_HOST:-elasticsearch}:9200 + - DbName=${CLUEDIN_DATASOURCE_DBNAME:-DataStore.Db.MicroServices} + - DbHost=${CLUEDIN_SQLSERVER_HOST} + - DbPassword=${CLUEDIN_SQLSERVER_PASS} + - DbUserName=${CLUEDIN_SQLSERVER_USER} + - SeqURL=http://${CLUEDIN_SEQ_HOST:-seq}:5341 + - SeqLoggingLevel=debug + - ConsoleLoggingLevel=info + - DataSourceSubmitterPort=${CLUEDIN_DATASOURCESUBMITTER_LOCALPORT:-9012} + - PUBLIC_WEB_API=http://${CLUEDIN_SERVER_HOST:-server}:9007 + - CluedIn_API_URL=${CLUEDIN_DATASOURCESUBMITTER_CLUE_API_URL:-/api/v2/clue?save=true} + - DataSourceSubmitterHost=${CLUEDIN_DATASOURCESUBMITTER_HOST:-datasource-submitter} + - DataSourceSubmitterWORKERS_NUMBER=4 ports: - - "${CLUEDIN_SUBMITTER_LOCALPORT:-9012}:8888" + - "${CLUEDIN_DATASOURCESUBMITTER_LOCALPORT:-9012}:8888" depends_on: + - elasticsearch - rabbitmq + - sqlserver - seq - - annotation deploy: restart_policy: condition: ${CLUEDIN_COMMON_RESTART_POLICY:-unless-stopped} delay: 5s - documentation: - image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cluedin/cluedin-micro-documentation:${CLUEDIN_DOCUMENTATION_TAG} - ports: - - "${CLUEDIN_DOCUMENTATION_LOCALPORT:-9021}:8888" +volumes: + datasource: diff --git a/docker/compose/docker-compose.server.yml b/docker/compose/docker-compose.server.yml index 4c76d09..9e93b75 100644 --- a/docker/compose/docker-compose.server.yml +++ b/docker/compose/docker-compose.server.yml @@ -35,19 +35,21 @@ services: - CLUEDIN_CONNECTIONSTRINGS__SIGNALRSCALEOUT=amqp://${CLUEDIN_RABBITMQ_USERNAME:-cluedin}:${CLUEDIN_RABBITMQ_PASSWORD:-cluedin}@${CLUEDIN_RABBITMQ_HOST:-rabbitmq}:5672 - CLUEDIN_CONNECTIONSTRINGS__AUTHENTICATIONSTORE=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.Authentication;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True - - CLUEDIN_CONNECTIONSTRINGS__AUDITLOG=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.AuditLog;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True - - CLUEDIN_CONNECTIONSTRINGS__BLOBSTORAGE=Data source=${CLUEDIN_SQLSERVER_HOST};Initial catalog=DataStore.Db.BlobStorage;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True - - CLUEDIN_CONNECTIONSTRINGS__CONFIGURATIONSTORE=Data source=${CLUEDIN_SQLSERVER_HOST};Initial catalog=DataStore.Db.Configuration;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True - - CLUEDIN_CONNECTIONSTRINGS__CLUEDINENTITIES=Data source=${CLUEDIN_SQLSERVER_HOST};Initial catalog=DataStore.Db.OpenCommunication;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True - - CLUEDIN_CONNECTIONSTRINGS__TOKENSTORE=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.TokenStore;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True - - CLUEDIN_CONNECTIONSTRINGS__TRAINING=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.Training;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True - - CLUEDIN_CONNECTIONSTRINGS__EXTERNALSEARCH=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.ExternalSearch;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True - - CLUEDIN_CONNECTIONSTRINGS__METRICS=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.Metrics;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True - - CLUEDIN_CONNECTIONSTRINGS__ML_LOGGING=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.ML-Logging;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True - - CLUEDIN_CONNECTIONSTRINGS__WEBAPPSTORE=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.WebApp;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True - - CLUEDIN_CONNECTIONSTRINGS__MICROSERVICES=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.MicroServices;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True - - CLUEDIN_CONNECTIONSTRINGS__CLEANCACHE=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.CleanCache;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True - - CLUEDIN_CONNECTIONSTRINGS__INVITATIONSTORE=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.WebApp;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};MultipleActiveResultSets=True;connection timeout=0;Max Pool Size=200;Pooling=True + - CLUEDIN_CONNECTIONSTRINGS__AUDITLOG=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.AuditLog;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True + - CLUEDIN_CONNECTIONSTRINGS__BLOBSTORAGE=Data source=${CLUEDIN_SQLSERVER_HOST};Initial catalog=DataStore.Db.BlobStorage;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True + - CLUEDIN_CONNECTIONSTRINGS__CONFIGURATIONSTORE=Data source=${CLUEDIN_SQLSERVER_HOST};Initial catalog=DataStore.Db.Configuration;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True + - CLUEDIN_CONNECTIONSTRINGS__CLUEDINENTITIES=Data source=${CLUEDIN_SQLSERVER_HOST};Initial catalog=DataStore.Db.OpenCommunication;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True + - CLUEDIN_CONNECTIONSTRINGS__TOKENSTORE=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.TokenStore;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True + - CLUEDIN_CONNECTIONSTRINGS__TRAINING=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.Training;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True + - CLUEDIN_CONNECTIONSTRINGS__EXTERNALSEARCH=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.ExternalSearch;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True + - CLUEDIN_CONNECTIONSTRINGS__METRICS=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.Metrics;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True + - CLUEDIN_CONNECTIONSTRINGS__LOCKING=Data source=${CLUEDIN_SQLSERVER_HOST};Initial catalog=DataStore.Db.OpenCommunication;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True + - CLUEDIN_CONNECTIONSTRINGS__ML_LOGGING=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.ML-Logging;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True + - CLUEDIN_CONNECTIONSTRINGS__WEBAPPSTORE=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.WebApp;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True + - CLUEDIN_CONNECTIONSTRINGS__MICROSERVICES=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.MicroServices;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True + - CLUEDIN_CONNECTIONSTRINGS__CLEANCACHE=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.CleanCache;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True + - CLUEDIN_CONNECTIONSTRINGS__INVITATIONSTORE=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.WebApp;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True + - CLUEDIN_CONNECTIONSTRINGS__STREAMLOG=Data Source=${CLUEDIN_SQLSERVER_HOST};Initial Catalog=DataStore.Db.StreamLog;User Id=${CLUEDIN_SQLSERVER_USER};Password=${CLUEDIN_SQLSERVER_PASS};connection timeout=0;Max Pool Size=200;Pooling=True - CLUEDIN_APPSETTINGS__AGENT_ENABLED=${CLUEDIN_AGENT_ENABLED} - CLUEDIN_APPSETTINGS__MODELS_ENABLED=${CLUEDIN_MODELS_ENABLED:-false} - DOTNET_ENVIRONMENT=${CLUEDIN_ENVIRONMENT} diff --git a/docker/compose/docker-compose.ui.yml b/docker/compose/docker-compose.ui.yml index b6feadc..3262d68 100644 --- a/docker/compose/docker-compose.ui.yml +++ b/docker/compose/docker-compose.ui.yml @@ -21,6 +21,7 @@ services: - REACT_APP_UI_PROTOCOL=http:// - REACT_APP_UI_UPLOAD_LIMIT=${CLUEDIN_UI_UPLOAD_LIMIT:-1gb} - REACT_APP_UI_CLUEDIN_PUBLICAPI_URL=http://localhost:9007 + - REACT_APP_UI_CLUEDIN_COPILOTHUB_URL=http://localhost:9000/hub/copilot depends_on: - server deploy: @@ -34,9 +35,8 @@ services: - "AUTH_API=http://${CLUEDIN_SERVER_HOST:-server}:9001/" - "WEB_API=http://${CLUEDIN_SERVER_HOST:-server}:9000/" - "MICRO_CLEAN_URL=http://${CLUEDIN_CLEAN_HOST:-clean}:${CLUEDIN_CLEAN_PORT:-8888}" - - "MICRO_ANNOTATION_URL=http://${CLUEDIN_ANNOTATION_HOST:-annotation}:${CLUEDIN_ANNOTATION_PORT:-8888}" - - "MICRO_DATASOURCE_URL=http://${CLUEDIN_DATASOURCE_HOST:-datasource}:${CLUEDIN_DATASOURCE_PORT:-8888}" - - "MICRO_SUBMITTER_URL=http://${CLUEDIN_SUBMITTER_HOST:-submitter}:${CLUEDIN_SUBMITTER_PORT:-9012}" + - "MICRO_DATASOURCE_URL=http://${CLUEDIN_DATASOURCE_HOST:-datasource}:${CLUEDIN_DATASOURCE_PORT:-9011}" + - "MICRO_SUBMITTER_URL=http://${CLUEDIN_DATASOURCESUBMITTER_HOST:-datasource-submitter}:${CLUEDIN_DATASOURCESUBMITTER_PORT:-9012}" - "MICRO_DOCUMENTATION_URL=http://${CLUEDIN_DOCUMENTATION_HOST:-documentation}:${CLUEDIN_DOCUMENTATION_PORT:-8888}" - "PUBLIC_API=http://${CLUEDIN_SERVER_HOST:-server}:9007/" - "WEBHOOKS_URL=http://${CLUEDIN_SERVER_HOST:-server}:9006/" @@ -57,12 +57,14 @@ services: - "ENGINE_ROOM_STREAM_DASHBOARD_ID=h0W9h3knk" - "ENGINE_ROOM_STREAM_DASHBOARD_NAME=streams" - "UPLOAD_LIMIT=${CLUEDIN_UI_UPLOAD_LIMIT:-1gb}" + - "ENABLE_DEVELOPMENT_FEATURES=true" ports: - "${CLUEDIN_GQL_LOCALPORT:-8888}:8888" depends_on: - - annotation - clean - datasource + - datasource-processing + - datasource-submitter - sqlserver deploy: restart_policy: diff --git a/docker/compose/docker-compose.yml b/docker/compose/docker-compose.yml index 06eb7c1..90c3ec6 100644 --- a/docker/compose/docker-compose.yml +++ b/docker/compose/docker-compose.yml @@ -6,10 +6,11 @@ services: - "${CLUEDIN_SMTP_HTTP_LOCALPORT:-25258}:80" - "${CLUEDIN_SMTP_EMAIL_LOCALPORT:-2525}:25" neo4j: - image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cluedin/neo4j:${CLUEDIN_NEO4J_TAG} + image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cluedin/neo4j-home:${CLUEDIN_NEO4J_TAG} environment: - NEO4J_AUTH=none - NEO4J_ACCEPT_LICENSE_AGREEMENT=yes + - NEO4J_initial_dbms_default__database=graph.db ports: - "${CLUEDIN_NEO4J_HTTP_LOCALPORT:-7474}:7474" - "${CLUEDIN_NEO4J_BOLT_LOCALPORT:-7687}:7687" @@ -23,7 +24,7 @@ services: - "${CLUEDIN_ELASTIC_DATA_LOCALPORT:-9300}:9300" rabbitmq: - image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/bitnami/rabbitmq:3.10.8-debian-11-r4 + image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/bitnami/rabbitmq:3.12.2-debian-11-r25 ports: - "${CLUEDIN_RABBITMQ_DATA_LOCALPORT:-5672}:5672" - "${CLUEDIN_RABBITMQ_HTTP_LOCALPORT:-15672}:15672" @@ -68,7 +69,7 @@ services: target: /etc/prometheus/prometheus.yml command: --web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml grafana: - image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/grafana/grafana:9.1.4 + image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/grafana/grafana:8.4.7 environment: - GF_AUTH_ANONYMOUS_ENABLED=true - GF_SECURITY_ALLOW_EMBEDDING=true @@ -100,6 +101,33 @@ services: - type: bind source: ./grafana/cadvisor-global.json target: /var/lib/grafana/dashboards/cadvisor-global.json + - type: bind + source: ./grafana/profiling/boolean-vocabulary-key.json + target: /var/lib/grafana/dashboards/profiling/boolean-vocabulary-key.json + - type: bind + source: ./grafana/profiling/datetime-vocabulary-key.json + target: /var/lib/grafana/dashboards/profiling/datetime-vocabulary-key.json + - type: bind + source: ./grafana/profiling/untyped-vocabulary-key.json + target: /var/lib/grafana/dashboards/profiling/untyped-vocabulary-key.json + - type: bind + source: ./grafana/profiling/float-vocabulary-key.json + target: /var/lib/grafana/dashboards/profiling/float-vocabulary-key.json + - type: bind + source: ./grafana/profiling/money-vocabulary-key.json + target: /var/lib/grafana/dashboards/profiling/money-vocabulary-key.json + - type: bind + source: ./grafana/profiling/text-vocabulary-key.json + target: /var/lib/grafana/dashboards/profiling/text-vocabulary-key.json + - type: bind + source: ./grafana/profiling/processing-over-time.json + target: /var/lib/grafana/dashboards/profiling/processing-over-time.json + - type: bind + source: ./grafana/profiling/general-stats.json + target: /var/lib/grafana/dashboards/profiling/general-stats.json + - type: bind + source: ./grafana/profiling/unmapped-vocabulary-key.json + target: /var/lib/grafana/dashboards/profiling/unmapped-vocabulary-key.json cadvisor: image: ${CLUEDIN_CONTAINER_REGISTRY:-cluedinprod.azurecr.io}/cadvisor/cadvisor:latest ports: diff --git a/env/default/.env b/env/default/.env index 87a6ec7..fc5f6e6 100644 --- a/env/default/.env +++ b/env/default/.env @@ -1,10 +1,4 @@ CLUEDIN_AGENT_ENABLED=true -CLUEDIN_ANNOTATION_DBNAME= -CLUEDIN_ANNOTATION_FORCETABLECREATION= -CLUEDIN_ANNOTATION_HOST= -CLUEDIN_ANNOTATION_LOCALPORT=9010 -CLUEDIN_ANNOTATION_PORT= -CLUEDIN_ANNOTATION_TAG=latest CLUEDIN_CLEAN_DBNAME= CLUEDIN_CLEAN_FORCETABLECREATION= CLUEDIN_CLEAN_HOST= @@ -14,14 +8,15 @@ CLUEDIN_CLEAN_TAG=latest CLUEDIN_CONTAINER_REGISTRY= CLUEDIN_COMMON_RESTART_POLICY= CLUEDIN_DATASOURCE_DBNAME= -CLUEDIN_DATASOURCE_FORCETABLECREATION= CLUEDIN_DATASOURCE_HOST= CLUEDIN_DATASOURCE_LOCALPORT= -CLUEDIN_DATASOURCE_PROMETHEUS_ENDPOINT_METRICS_LOCALPORT=9014 -CLUEDIN_DATASOURCE_PROMETHEUS_FILE_METRICS_LOCALPORT=9015 -CLUEDIN_DATASOURCE_PROMETHEUS_SQL_METRICS_LOCALPORT=9016 CLUEDIN_DATASOURCE_PORT= CLUEDIN_DATASOURCE_TAG=latest +CLUEDIN_DATASOURCEPROCESSING_LOCALPORT= +CLUEDIN_DATASOURCEPROCESSING_TAG=latest +CLUEDIN_DATASOURCESUBMITTER_CLUE_API_UR=/api/v2/clue?save=true +CLUEDIN_DATASOURCESUBMITTER_LOCALPORT= +CLUEDIN_DATASOURCESUBMITTER_TAG=latest CLUEDIN_DOCUMENTATION_HOST= CLUEDIN_DOCUMENTATION_LOCALPORT= CLUEDIN_DOCUMENTATION_PORT= @@ -90,11 +85,6 @@ CLUEDIN_SQLSERVER_LOCALPORT= CLUEDIN_SQLSERVER_PASS=yourStrong(!)Password CLUEDIN_SQLSERVER_TAG=latest CLUEDIN_SQLSERVER_USER=sa -CLUEDIN_SUBMITTER_HOST= -CLUEDIN_SUBMITTER_LOCALPORT= -CLUEDIN_SUBMITTER_PORT= -CLUEDIN_SUBMITTER_TAG=latest -CLUEDIN_SUBMITTER_CLUE_API_URL=/api/v2/clue?save=true CLUEDIN_UI_ALT_UPLOAD= CLUEDIN_UI_FEATURES_ANNOTATION=annotation CLUEDIN_UI_FEATURES_CLEAN=clean diff --git a/env/default/grafana/dashboards.yml b/env/default/grafana/dashboards.yml index 4664fbf..d466627 100644 --- a/env/default/grafana/dashboards.yml +++ b/env/default/grafana/dashboards.yml @@ -5,4 +5,10 @@ providers: folder: '' type: 'file' options: - folder: '/var/lib/grafana/dashboards' \ No newline at end of file + folder: '/var/lib/grafana/dashboards' +- name: 'cluedin-strong-typed' + org_id: 1 + folder: 'Profiling' + type: 'file' + options: + folder: '/var/lib/grafana/dashboards/profiling' \ No newline at end of file diff --git a/env/default/grafana/datasource.yml b/env/default/grafana/datasource.yml index 86356f8..f2cba4c 100644 --- a/env/default/grafana/datasource.yml +++ b/env/default/grafana/datasource.yml @@ -5,4 +5,28 @@ datasources: url: http://prometheus:9090 isDefault: true access: proxy - editable: true \ No newline at end of file + editable: true +- name: elasticsearch + type: elasticsearch + access: proxy + url: http://elasticsearch:9200 + uid: "28kyjMzIk" + jsonData: + esVersion: '7.7.0' + timeField: 'discoverydate' +- name: elasticsearch-unmapped + type: elasticsearch + access: proxy + url: http://elasticsearch:9200 + uid: "28kyjMzJk" + jsonData: + esVersion: '7.7.0' + timeField: 'cluedincreated' +- name: mssql + type: mssql + url: host.docker.internal + database: DataStore.Db.OpenCommunication + user: sa + uid: JmNKFSkSz + secureJsonData: + password: yourStrong(!)Password \ No newline at end of file diff --git a/env/default/grafana/profiling/boolean-vocabulary-key.json b/env/default/grafana/profiling/boolean-vocabulary-key.json new file mode 100644 index 0000000..0cf1d1f --- /dev/null +++ b/env/default/grafana/profiling/boolean-vocabulary-key.json @@ -0,0 +1,503 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 9, + "iteration": 1699535634250, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "/^Count$/", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "1", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "+typedproperties.${key}_${dataType}:* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Total Values for '${key}'", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "#476ca2", + "value": null + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "allValues" + ], + "fields": "/^Count$/", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "typedproperties.${key}_${dataType}", + "id": "5", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "organizationId:${organizationId} AND typedproperties.${key}_${dataType}=true", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "'${key}' is True", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 70 + }, + { + "color": "#64cfba", + "value": 85 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "allValues" + ], + "fields": "/^Count$/", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "typedproperties.${key}_${dataType}", + "id": "5", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "organizationId:${organizationId} AND typedproperties.${key}_${dataType}=false", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "'${key}' is False", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 2, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#64cfba", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 0, + "y": 5 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "typedproperties.${key}_${dataType}:* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Values Over Time for '${key}'", + "type": "timeseries" + } + ], + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "employee.isactive", + "value": "employee.isactive" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "key", + "options": [ + { + "selected": true, + "text": "employee.isactive", + "value": "employee.isactive" + } + ], + "query": "employee.isactive", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "b22ce7c1-137d-4466-9a3e-c62238079e47", + "value": "b22ce7c1-137d-4466-9a3e-c62238079e47" + }, + "hide": 0, + "includeAll": false, + "label": "Organization Id", + "multi": false, + "name": "organizationId", + "options": [ + { + "selected": true, + "text": "b22ce7c1-137d-4466-9a3e-c62238079e47", + "value": "b22ce7c1-137d-4466-9a3e-c62238079e47" + } + ], + "query": "b22ce7c1-137d-4466-9a3e-c62238079e47", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "Boolean", + "value": "Boolean" + }, + "hide": 0, + "includeAll": false, + "label": "Data Type", + "multi": false, + "name": "dataType", + "options": [ + { + "selected": true, + "text": "Boolean", + "value": "Boolean" + } + ], + "query": "Boolean", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-90d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Boolean Vocabulary Key", + "uid": "3lAj47zSz", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/env/default/grafana/profiling/datetime-vocabulary-key.json b/env/default/grafana/profiling/datetime-vocabulary-key.json new file mode 100644 index 0000000..e6af38f --- /dev/null +++ b/env/default/grafana/profiling/datetime-vocabulary-key.json @@ -0,0 +1,588 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 11, + "iteration": 1699614987545, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#476ca2", + "value": 80 + } + ] + }, + "unit": "dateTimeAsIso" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "min" + ], + "fields": "/.*/", + "limit": 1, + "values": true + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "typedproperties.${key}_${dataType}", + "id": "2", + "settings": { + "min_doc_count": "1", + "order": "asc", + "orderBy": "_term", + "size": "1" + }, + "type": "terms" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Minimum Value", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#476ca2", + "value": 80 + } + ] + }, + "unit": "dateTimeAsIso" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 5, + "y": 0 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "/.*/", + "limit": 1, + "values": true + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "typedproperties.${key}_${dataType}", + "id": "2", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Maximum Value", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 10, + "y": 0 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "+typedproperties.${key}_${dataType}:* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Number of Values", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 15, + "y": 0 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "-typedproperties.${key}_${dataType}:* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Empty Values", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#64cfba", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 20, + "x": 0, + "y": 6 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "-typedproperties.${key}_${dataType}:Cat AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Values Over Time for '${key}'", + "type": "timeseries" + } + ], + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "employee.startDate", + "value": "employee.startDate" + }, + "hide": 0, + "includeAll": false, + "label": "Vocabulary Key", + "multi": false, + "name": "key", + "options": [ + { + "selected": false, + "text": "employee.Birthday", + "value": "employee.Birthday" + } + ], + "query": "employee.Birthday", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "ddda4082-5023-4ac9-a982-da6c4fa60835", + "value": "ddda4082-5023-4ac9-a982-da6c4fa60835" + }, + "hide": 0, + "includeAll": false, + "label": "Organization Id", + "multi": false, + "name": "organizationId", + "options": [ + { + "selected": false, + "text": "b22ce7c1-137d-4466-9a3e-c62238079e47", + "value": "b22ce7c1-137d-4466-9a3e-c62238079e47" + } + ], + "query": "b22ce7c1-137d-4466-9a3e-c62238079e47", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "Date", + "value": "Date" + }, + "hide": 0, + "includeAll": false, + "label": "Data Type", + "multi": false, + "name": "dataType", + "options": [ + { + "selected": true, + "text": "Date", + "value": "Date" + } + ], + "query": "Date", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "1s", + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "DateTime Vocabulary Key", + "uid": "b9Q0i7kSz", + "version": 4, + "weekStart": "" +} \ No newline at end of file diff --git a/env/default/grafana/profiling/float-vocabulary-key.json b/env/default/grafana/profiling/float-vocabulary-key.json new file mode 100644 index 0000000..4700462 --- /dev/null +++ b/env/default/grafana/profiling/float-vocabulary-key.json @@ -0,0 +1,764 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 3, + "iteration": 1699614866154, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "min" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "field": "typedproperties.${key}_${dataType}", + "id": "1", + "type": "min" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Minimum Value", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 5, + "y": 0 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "min" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "field": "typedproperties.${key}_${dataType}", + "id": "1", + "meta": { + "std_deviation": true, + "std_deviation_bounds_lower": false, + "std_deviation_bounds_upper": false + }, + "type": "extended_stats" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Standard Deviation", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 10, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "field": "typedproperties.${key}_${dataType}", + "id": "1", + "type": "max" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Maximum Value", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 15, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "field": "typedproperties.${key}_${dataType}", + "id": "1", + "type": "avg" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Average Value", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 0, + "y": 6 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "+typedproperties.${key}_${dataType}:* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Number of Values", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 5, + "y": 6 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "field": "typedproperties.${key}_${dataType}", + "id": "1", + "type": "sum" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Sum Value", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#64cfba", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 10, + "y": 6 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "+typedproperties.${key}_${dataType}:* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Values Over Time for '${key}'", + "type": "timeseries" + } + ], + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "employee.age", + "value": "employee.age" + }, + "hide": 0, + "includeAll": false, + "label": "Vocabulary Key", + "multi": false, + "name": "key", + "options": [ + { + "selected": true, + "text": "employee.age", + "value": "employee.age" + }, + { + "selected": false, + "text": "deal.Size", + "value": "deal.Size" + } + ], + "query": "employee.age,deal.Size", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "ddda4082-5023-4ac9-a982-da6c4fa60835", + "value": "ddda4082-5023-4ac9-a982-da6c4fa60835" + }, + "hide": 0, + "includeAll": false, + "label": "Organization Id", + "multi": false, + "name": "organizationId", + "options": [ + { + "selected": false, + "text": "b22ce7c1-137d-4466-9a3e-c62238079e47", + "value": "b22ce7c1-137d-4466-9a3e-c62238079e47" + } + ], + "query": "b22ce7c1-137d-4466-9a3e-c62238079e47", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "Integer", + "value": "Integer" + }, + "hide": 0, + "includeAll": false, + "label": "Data Type", + "multi": false, + "name": "dataType", + "options": [ + { + "selected": true, + "text": "Integer", + "value": "Integer" + } + ], + "query": "Integer", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Float Vocabulary Key", + "uid": "LFRqeMkIk", + "version": 4, + "weekStart": "" +} \ No newline at end of file diff --git a/env/default/grafana/profiling/general-stats.json b/env/default/grafana/profiling/general-stats.json new file mode 100644 index 0000000..1aea4d5 --- /dev/null +++ b/env/default/grafana/profiling/general-stats.json @@ -0,0 +1,2119 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "iteration": 1694680282375, + "links": [], + "liveNow": false, + "panels": [ + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 41, + "title": "Stats", + "type": "row" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 1 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "+entityType:* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Total Entities", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 3, + "y": 1 + }, + "id": 26, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "min" + ], + "fields": "/^Count$/", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "isShadowEntity", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Total Shadow Entities", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "dateFormat": "YYYY-DD-MM", + "destinationType": "time", + "targetField": "typedproperties.employee.startDate_Date" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "mssql", + "uid": "JmNKFSkSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 1 + }, + "id": 43, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "datasource": { + "type": "mssql", + "uid": "JmNKFSkSz" + }, + "format": "table", + "rawSql": "SELECT SUM (d.Persistence)\r\n FROM [DataStore.Db.Metrics].[dbo].Metric m\r\n INNER JOIN [DataStore.Db.Metrics].[dbo].Dimension d on\r\n m.Id = d.MetricId\r\n WHERE m.Name = 'counter.billable-record'", + "refId": "A" + } + ], + "title": "Billable Record Count", + "type": "stat" + }, + { + "datasource": { + "type": "mssql", + "uid": "JmNKFSkSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 47, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "datasource": { + "type": "mssql", + "uid": "JmNKFSkSz" + }, + "format": "table", + "rawSql": "SELECT COUNT(*) FROM [DataStore.Db.BlobStorage].[dbo].Blobs \r\nWhere MimeType = 'application/vnd.cluedin.entity.xml'", + "refId": "A" + } + ], + "title": "Total Blobs", + "type": "stat" + }, + { + "datasource": { + "type": "mssql", + "uid": "JmNKFSkSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 49, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "datasource": { + "type": "mssql", + "uid": "JmNKFSkSz" + }, + "format": "table", + "rawSql": "SELECT COUNT(GroupId)\r\n FROM [DataStore.Db.OpenCommunication].[dbo].[DeduplicationGroupEntry]\r\n GROUP BY GroupId", + "refId": "A" + } + ], + "title": "Possible Duplicate Count", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + }, + "unit": "bool" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 30, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "isExternalData", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Total External Data", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "dateFormat": "YYYY-DD-MM", + "destinationType": "time", + "targetField": "typedproperties.employee.startDate_Date" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 39, + "title": "Entity Types", + "type": "row" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 18, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "values": [] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "entityType", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Entity Type Distribution", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "dateFormat": "YYYY-DD-MM", + "destinationType": "time", + "targetField": "typedproperties.employee.startDate_Date" + } + ], + "fields": {} + } + } + ], + "type": "piechart" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#64cfba", + "value": null + }, + { + "color": "#476ca2", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 6, + "y": 7 + }, + "id": 15, + "options": { + "displayMode": "lcd", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": true + }, + "showUnfilled": true + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "entityType", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Entity Type Heatmap", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "dateFormat": "YYYY-DD-MM", + "destinationType": "time", + "targetField": "typedproperties.employee.startDate_Date" + } + ], + "fields": {} + } + } + ], + "type": "bargauge" + }, + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 37, + "title": "Data Sources", + "type": "row" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#64cfba", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 0, + "y": 19 + }, + "id": 28, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xField": "providers", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "providers", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Data by Source", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "dateFormat": "YYYY-DD-MM", + "destinationType": "time", + "targetField": "typedproperties.employee.startDate_Date" + } + ], + "fields": {} + } + } + ], + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 35, + "panels": [], + "title": "Processing", + "type": "row" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#64cfba", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 18, + "x": 0, + "y": 28 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "-*:Cat AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Incoming Values Over Time", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#64cfba", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 9, + "x": 0, + "y": 34 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "lastProcessedDate", + "id": "3", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Last Processed Date", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "dateFormat": "YYYY-DD-MM", + "destinationType": "time", + "targetField": "typedproperties.employee.startDate_Date" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#64cfba", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 9, + "x": 9, + "y": 34 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "3", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Discovery Date", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "dateFormat": "YYYY-DD-MM", + "destinationType": "time", + "targetField": "typedproperties.employee.startDate_Date" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#64cfba", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 9, + "x": 0, + "y": 40 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "3", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "+isShadowEntity:true AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Shadow Entities Over Time", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "dateFormat": "YYYY-DD-MM", + "destinationType": "time", + "targetField": "typedproperties.employee.startDate_Date" + } + ], + "fields": {} + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "mssql", + "uid": "JmNKFSkSz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "value" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#64cfba", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 9, + "x": 9, + "y": 40 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "none", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "datasource": { + "type": "mssql", + "uid": "JmNKFSkSz" + }, + "format": "time_series", + "rawSql": "SELECT\n CreatedDate time,\n COUNT(*) value\nFROM [DataStore.Db.OpenCommunication].[dbo].GDPRMessages\nWHERE $__timeFilter(CreatedDate)\nGROUP BY CreatedDate\nORDER BY CreatedDate", + "refId": "A" + } + ], + "title": "Mesh Commands Over Time", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 33, + "panels": [], + "title": "Properties", + "type": "row" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#64cfba", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "propertyCount" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#64cfba", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 9, + "x": 0, + "y": 48 + }, + "id": 24, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xField": "propertyCount", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "propertyCount", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Property Count Distribution", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "dateFormat": "YYYY-DD-MM", + "destinationType": "time", + "targetField": "typedproperties.employee.startDate_Date" + } + ], + "fields": {} + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "density" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#64cfba", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 9, + "x": 9, + "y": 48 + }, + "id": 25, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xField": "density", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "density", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Record Density", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "dateFormat": "YYYY-DD-MM", + "destinationType": "time", + "targetField": "typedproperties.employee.startDate_Date" + } + ], + "fields": {} + } + } + ], + "type": "barchart" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Unique Count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#56A64B", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 9, + "x": 0, + "y": 58 + }, + "id": 29, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xField": "codes", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "codes", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "field": "codes", + "id": "1", + "type": "cardinality" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Number of Entity Codes", + "transformations": [], + "type": "barchart" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "persistVersion" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#64cfba", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 9, + "x": 9, + "y": 58 + }, + "id": 27, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xField": "persistVersion", + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "persistVersion", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Versions", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "dateFormat": "YYYY-DD-MM", + "destinationType": "time", + "targetField": "typedproperties.employee.startDate_Date" + } + ], + "fields": {} + } + } + ], + "type": "barchart" + } + ], + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "a3f62dd0-115e-455b-9862-74505bc304e0", + "value": "a3f62dd0-115e-455b-9862-74505bc304e0" + }, + "hide": 0, + "includeAll": false, + "label": "Organization Id", + "multi": false, + "name": "organizationId", + "options": [ + { + "selected": true, + "text": "b22ce7c1-137d-4466-9a3e-c62238079e47", + "value": "b22ce7c1-137d-4466-9a3e-c62238079e47" + } + ], + "query": "b22ce7c1-137d-4466-9a3e-c62238079e47", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "General Dashboard CluedIn", + "uid": "TWiqNSkSk", + "version": 12, + "weekStart": "" +} \ No newline at end of file diff --git a/env/default/grafana/profiling/money-vocabulary-key.json b/env/default/grafana/profiling/money-vocabulary-key.json new file mode 100644 index 0000000..1033d48 --- /dev/null +++ b/env/default/grafana/profiling/money-vocabulary-key.json @@ -0,0 +1,767 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 5, + "iteration": 1699614726587, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 0, + "y": 0 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "min" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "field": "typedproperties.${key}_${dataType}", + "id": "1", + "type": "min" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Minimum Value", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 5, + "y": 0 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "min" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "field": "typedproperties.${key}_${dataType}", + "id": "1", + "meta": { + "std_deviation": true, + "std_deviation_bounds_lower": false, + "std_deviation_bounds_upper": false + }, + "type": "extended_stats" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Standard Deviation", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 10, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "field": "typedproperties.${key}_${dataType}", + "id": "1", + "type": "max" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Maximum Value", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 15, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "field": "typedproperties.${key}_${dataType}", + "id": "1", + "type": "avg" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Average Value", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 0, + "y": 6 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "+typedproperties.${key}_${dataType}:* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Number of Values", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 5, + "y": 6 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "field": "typedproperties.${key}_${dataType}", + "id": "1", + "type": "sum" + } + ], + "query": "* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Sum Value", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#64cfba", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 10, + "y": 6 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "+typedproperties.${key}_${dataType}:* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Values Over Time for '${key}'", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "employee.salary", + "value": "employee.salary" + }, + "hide": 0, + "includeAll": false, + "label": "Vocabulary Key", + "multi": false, + "name": "key", + "options": [ + { + "selected": false, + "text": "employee.Salary", + "value": "employee.Salary" + } + ], + "query": "employee.Salary", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "ddda4082-5023-4ac9-a982-da6c4fa60835", + "value": "ddda4082-5023-4ac9-a982-da6c4fa60835" + }, + "hide": 0, + "includeAll": false, + "label": "Organization Id", + "multi": false, + "name": "organizationId", + "options": [ + { + "selected": false, + "text": "b22ce7c1-137d-4466-9a3e-c62238079e47", + "value": "b22ce7c1-137d-4466-9a3e-c62238079e47" + } + ], + "query": "b22ce7c1-137d-4466-9a3e-c62238079e47", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "Money", + "value": "Money" + }, + "hide": 0, + "includeAll": false, + "label": "Data Type", + "multi": false, + "name": "dataType", + "options": [ + { + "selected": true, + "text": "Money", + "value": "Money" + } + ], + "query": "Money", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Money Vocabulary Key", + "uid": "yElkNIkIk", + "version": 4, + "weekStart": "" +} \ No newline at end of file diff --git a/env/default/grafana/profiling/processing-over-time.json b/env/default/grafana/profiling/processing-over-time.json new file mode 100644 index 0000000..6d2f806 --- /dev/null +++ b/env/default/grafana/profiling/processing-over-time.json @@ -0,0 +1,714 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "A new RabbitMQ Management Overview", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 10991, + "graphTooltip": 1, + "id": 14, + "iteration": 1694462816392, + "links": [ + { + "icon": "doc", + "tags": [], + "targetBlank": true, + "title": "Monitoring with Prometheus & Grafana", + "tooltip": "", + "type": "link", + "url": "https://www.rabbitmq.com/prometheus.html" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 27, + "panels": [], + "title": "QUEUED MESSAGES", + "type": "row" + }, + { + "aliasColors": { + "rabbit@localhost": "#64cfba" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "Total number of ready messages ready to be delivered to consumers.\n\nAim to keep this value as low as possible. RabbitMQ behaves best when messages are flowing through it. It's OK for publishers to occasionally outpace consumers, but the expectation is that consumers will eventually process all ready messages.\n\nIf this metric keeps increasing, your system will eventually run out of memory and/or disk space. Consider using TTL or Queue Length Limit to prevent unbounded message growth.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Consumers](https://www.rabbitmq.com/consumers.html)\n* [Queue Length Limit](https://www.rabbitmq.com/maxlength.html)\n* [Time-To-Live and Expiration](https://www.rabbitmq.com/ttl.html)", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 16, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 70, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.4.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "$$hashKey": "object:185", + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#64CFBA" + }, + { + "$$hashKey": "object:186", + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "$$hashKey": "object:187", + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "$$hashKey": "object:188", + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "$$hashKey": "object:189", + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "$$hashKey": "object:190", + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "$$hashKey": "object:191", + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "$$hashKey": "object:192", + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "$$hashKey": "object:193", + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "$$hashKey": "object:194", + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rabbitmq_queue_messages_ready * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages ready to be delivered to consumers", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "$$hashKey": "object:235", + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "$$hashKey": "object:236", + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "rabbit@localhost": "#64cfba" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "description": "The rate of messages received from publishers and successfully routed to the master queue replicas.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Publishers](https://www.rabbitmq.com/publishers.html)", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 16, + "x": 0, + "y": 6 + }, + "hiddenSeries": false, + "id": 66, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.4.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#64CFBA" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "exemplar": true, + "expr": "rabbitmq_queue_messages{queue=\"CluedIn.Core.Messages.Processing.DeadLetterCommand, CluedIn.Core_CluedIn_Retry\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages in Dead Letter Queue", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 11, + "panels": [], + "title": "INCOMING MESSAGES", + "type": "row" + }, + { + "aliasColors": { + "rabbit@localhost": "#64cfba" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The incoming message rate before any routing rules are applied.\n\nIf this value is lower than the number of messages published to queues, it may indicate that some messages are delivered to more than one queue.\n\nIf this value is higher than the number of messages published to queues, messages cannot be routed and will either be dropped or returned to publishers.\n\n* [Publishers](https://www.rabbitmq.com/publishers.html)", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 16, + "x": 0, + "y": 12 + }, + "hiddenSeries": false, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.4.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#64CFBA" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_channel_messages_published_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages published / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "rabbit@localhost": "#64cfba" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "The rate of messages received from publishers and successfully routed to the master queue replicas.\n\n* [Queues](https://www.rabbitmq.com/queues.html)\n* [Publishers](https://www.rabbitmq.com/publishers.html)", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 10, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 16, + "x": 0, + "y": 17 + }, + "hiddenSeries": false, + "id": 61, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.4.7", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^rabbit@[\\w.-]+0/", + "color": "#64CFBA" + }, + { + "alias": "/^rabbit@[\\w.-]+1/", + "color": "#F2CC0C" + }, + { + "alias": "/^rabbit@[\\w.-]+2/", + "color": "#3274D9" + }, + { + "alias": "/^rabbit@[\\w.-]+3/", + "color": "#A352CC" + }, + { + "alias": "/^rabbit@[\\w.-]+4/", + "color": "#FF780A" + }, + { + "alias": "/^rabbit@[\\w.-]+5/", + "color": "#96D98D" + }, + { + "alias": "/^rabbit@[\\w.-]+6/", + "color": "#FFEE52" + }, + { + "alias": "/^rabbit@[\\w.-]+7/", + "color": "#8AB8FF" + }, + { + "alias": "/^rabbit@[\\w.-]+8/", + "color": "#CA95E5" + }, + { + "alias": "/^rabbit@[\\w.-]+9/", + "color": "#FFB357" + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(rabbitmq_queue_messages_published_total[60s]) * on(instance) group_left(rabbitmq_cluster, rabbitmq_node) rabbitmq_identity_info{rabbitmq_cluster=\"$rabbitmq_cluster\"}) by(rabbitmq_node)", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "legendFormat": "{{rabbitmq_node}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Messages routed to queues / s", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "15s", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "rabbit@cluedin-dev", + "value": "rabbit@cluedin-dev" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(rabbitmq_identity_info, rabbitmq_cluster)", + "hide": 0, + "includeAll": false, + "label": "RabbitMQ Cluster", + "multi": false, + "name": "rabbitmq_cluster", + "options": [], + "query": { + "query": "label_values(rabbitmq_identity_info, rabbitmq_cluster)", + "refId": "Prometheus-rabbitmq_cluster-Variable-Query" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "1s", + "15s", + "30s", + "1m", + "5m", + "10m" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Processing Dashboard", + "uid": "3evNhSkIk", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/env/default/grafana/profiling/text-vocabulary-key.json b/env/default/grafana/profiling/text-vocabulary-key.json new file mode 100644 index 0000000..5057f7f --- /dev/null +++ b/env/default/grafana/profiling/text-vocabulary-key.json @@ -0,0 +1,519 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "iteration": 1699534274842, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "1", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "+typedproperties.${key}_${dataType}:* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Total Values for '${key}'", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#64cfba", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 4, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "1", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "+typedproperties.${key}_${dataType}:* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Values Over Time for '${key}'", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 15, + "x": 0, + "y": 6 + }, + "id": 15, + "options": { + "displayMode": "lcd", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": true + }, + "showUnfilled": true + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "typedproperties.${key}_${dataType}", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "+typedproperties.${key}_${dataType}:* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Distribution of Values for '${key}'", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "dateFormat": "YYYY-DD-MM", + "destinationType": "time", + "targetField": "properties.${key}" + } + ], + "fields": {} + } + } + ], + "type": "bargauge" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 5, + "x": 15, + "y": 6 + }, + "id": 18, + "options": { + "displayLabels": [ + "name", + "value" + ], + "legend": { + "displayMode": "hidden", + "placement": "right", + "values": [] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "typedproperties.${key}_${dataType}", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "+typedproperties.${key}_${dataType}:* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Distribution of Values for '${key}'", + "transformations": [], + "type": "piechart" + } + ], + "refresh": "5s", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "employee.firstName", + "value": "employee.firstName" + }, + "hide": 0, + "includeAll": false, + "label": "Vocabulary Key", + "multi": false, + "name": "key", + "options": [ + { + "selected": false, + "text": "user.firstName", + "value": "user.firstName" + } + ], + "query": "user.firstName", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "e8ed9853-d1f6-48c6-8470-3511f6d55af9", + "value": "e8ed9853-d1f6-48c6-8470-3511f6d55af9" + }, + "hide": 0, + "includeAll": false, + "label": "Organization Id", + "multi": false, + "name": "organizationId", + "options": [ + { + "selected": false, + "text": "bea4363a-db25-4762-a52f-2c2104ac2d00", + "value": "bea4363a-db25-4762-a52f-2c2104ac2d00" + } + ], + "query": "bea4363a-db25-4762-a52f-2c2104ac2d00", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "Keyword", + "value": "Keyword" + }, + "hide": 0, + "includeAll": false, + "label": "Data Type", + "multi": false, + "name": "dataType", + "options": [ + { + "selected": false, + "text": "String", + "value": "String" + } + ], + "query": "String", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Text Vocabulary Key", + "uid": "xKKNHSzIk", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/env/default/grafana/profiling/unmapped-vocabulary-key.json b/env/default/grafana/profiling/unmapped-vocabulary-key.json new file mode 100644 index 0000000..8c1859c --- /dev/null +++ b/env/default/grafana/profiling/unmapped-vocabulary-key.json @@ -0,0 +1,475 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 8, + "iteration": 1694594835696, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "elasticsearch-unmapped", + "uid": "28kyjMzJk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "cluedincreated", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "1", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch-unmapped", + "uid": "28kyjMzJk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "+${key}:*", + "refId": "A", + "timeField": "cluedincreated" + } + ], + "title": "Total Values for '${key}'", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch-unmapped", + "uid": "28kyjMzJk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#64cfba", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 4, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "cluedincreated", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "1", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch-unmapped", + "uid": "28kyjMzJk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "+${key}:*", + "refId": "A", + "timeField": "cluedincreated" + } + ], + "title": "Values Over Time for '${key}'", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzJk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 15, + "x": 0, + "y": 6 + }, + "id": 15, + "options": { + "displayMode": "lcd", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": true + }, + "showUnfilled": true + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "${key}.keyword", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_count", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "elasticsearch-unmapped", + "uid": "28kyjMzJk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "+${key}:*", + "refId": "A", + "timeField": "cluedincreated" + } + ], + "title": "Distribution of Values for '${key}'", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "dateFormat": "YYYY-DD-MM", + "destinationType": "time", + "targetField": "${key}" + } + ], + "fields": {} + } + } + ], + "type": "bargauge" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzJk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 5, + "x": 15, + "y": 6 + }, + "id": 18, + "options": { + "displayLabels": [ + "name", + "value" + ], + "legend": { + "displayMode": "hidden", + "placement": "right", + "values": [] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "${key}.keyword", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "+${key}:*", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Distribution of Values for '${key}'", + "transformations": [], + "type": "piechart" + } + ], + "refresh": "5s", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "first_name", + "value": "first_name" + }, + "hide": 0, + "includeAll": false, + "label": "Vocabulary Key", + "multi": false, + "name": "key", + "options": [ + { + "selected": true, + "text": "first_name", + "value": "first_name" + } + ], + "query": "first_name", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Unmapped Vocabulary Key", + "uid": "xKKNHSAIk", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/env/default/grafana/profiling/untyped-vocabulary-key.json b/env/default/grafana/profiling/untyped-vocabulary-key.json new file mode 100644 index 0000000..7110e77 --- /dev/null +++ b/env/default/grafana/profiling/untyped-vocabulary-key.json @@ -0,0 +1,497 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 13, + "iteration": 1699872772868, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "1", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "+properties.${key}:* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Total Values for '${key}'", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "#64cfba", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Count" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#64cfba", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 16, + "x": 4, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "discoverydate", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "1", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "+properties.${key}:* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Values Over Time for '${key}'", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#476ca2", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 15, + "x": 0, + "y": 6 + }, + "id": 15, + "options": { + "displayMode": "lcd", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": true + }, + "showUnfilled": true + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "properties.${key}", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "+properties.${key}:* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Distribution of Values for '${key}'", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "dateFormat": "YYYY-DD-MM", + "destinationType": "time", + "targetField": "properties.${key}" + } + ], + "fields": {} + } + } + ], + "type": "bargauge" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 5, + "x": 15, + "y": 6 + }, + "id": 18, + "options": { + "displayLabels": [ + "name", + "value" + ], + "legend": { + "displayMode": "hidden", + "placement": "right", + "values": [] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.4.7", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "properties.${key}", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "28kyjMzIk" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "+properties.${key}:* AND organizationId:${organizationId}", + "refId": "A", + "timeField": "discoverydate" + } + ], + "title": "Distribution of Values for '${key}'", + "transformations": [], + "type": "piechart" + } + ], + "refresh": "5s", + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "employee.firstName", + "value": "employee.firstName" + }, + "hide": 0, + "includeAll": false, + "label": "Vocabulary Key", + "multi": false, + "name": "key", + "options": [ + { + "selected": true, + "text": "user.firstName", + "value": "user.firstName" + } + ], + "query": "user.firstName", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": { + "selected": false, + "text": "e8ed9853-d1f6-48c6-8470-3511f6d55af9", + "value": "e8ed9853-d1f6-48c6-8470-3511f6d55af9" + }, + "hide": 0, + "includeAll": false, + "label": "Organization Id", + "multi": false, + "name": "organizationId", + "options": [ + { + "selected": true, + "text": "bea4363a-db25-4762-a52f-2c2104ac2d00", + "value": "bea4363a-db25-4762-a52f-2c2104ac2d00" + } + ], + "query": "bea4363a-db25-4762-a52f-2c2104ac2d00", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "Untyped Vocabulary Key", + "uid": "xKKNWOTIk", + "version": 2, + "weekStart": "" +} \ No newline at end of file diff --git a/env/default/prometheus/plugins.conf b/env/default/prometheus/plugins.conf index 318ea04..5b647c9 100644 --- a/env/default/prometheus/plugins.conf +++ b/env/default/prometheus/plugins.conf @@ -1 +1 @@ -[rabbitmq_management,rabbitmq_prometheus]. \ No newline at end of file +[rabbitmq_management,rabbitmq_prometheus,rabbitmq_shovel,rabbitmq_shovel_management]. \ No newline at end of file diff --git a/env/default/prometheus/rabbitmq.conf b/env/default/prometheus/rabbitmq.conf index 659dbbc..df9d55f 100644 --- a/env/default/prometheus/rabbitmq.conf +++ b/env/default/prometheus/rabbitmq.conf @@ -1,2 +1,3 @@ prometheus.return_per_object_metrics = true -loopback_users.guest = false \ No newline at end of file +loopback_users.guest = false +classic_queue.default_version = 2 \ No newline at end of file