Release 3.2.0

CluedIn-io · Nov 24, 2020 · 239756d · 239756d
1 parent 2de7e05
commit 239756d
Show file tree

Hide file tree

Showing 11 changed files with 112 additions and 40 deletions.
diff --git a/README.md b/README.md
@@ -261,7 +261,7 @@ This sub-command prints the current version information for this script.
 | datasource    | 9011       | 8888          | DataSource API          |
 | submitter     | 9012       | 8888          | Submitter API           |
 | gql           | 8888       | 8888          | GraphQL                 |
-| ui            | 80         | 80            | UI/System Entrypoint    |
+| ui            | 9080       | 80            | UI/System Entrypoint    |
 | neo4j         | 7474       | 7474          | Neo4J Data              |
 |               | 7687       | 7687          | Neo4J Bolt              |
 | elasticsearch | 9200       | 9200          | ElasticSearch Data      |

diff --git a/docker/compose/docker-compose.micro.yml b/docker/compose/docker-compose.micro.yml
@@ -13,13 +13,18 @@ services:
     - DbUserName=${CLUEDIN_SQLSERVER_USER}
     - DbName=${CLUEDIN_CLEAN_DBNAME:-DataStore.Db.MicroServices}
     - ForceTableCreation=${CLUEDIN_CLEAN_FORCETABLECREATION:-false}
+    - AnnotationURL=http://${CLUEDIN_ANNOTATION_HOST:-annotation}:${CLUEDIN_ANNOTATION_PORT:-8888}
+    - DataSourceURL=http://${CLUEDIN_DATASOURCE_HOST:-datasource}:${CLUEDIN_DATASOURCE_PORT:-8888}
+    - NODE_ENV=${CLUEDIN_ENVIRONMENT}
     ports:
       - "${CLUEDIN_CLEAN_LOCALPORT:-9009}:8888"
     depends_on:
     - elasticsearch
     - rabbitmq
     - sqlserver
     - seq
+    - annotation
+    - datasource
 
   annotation:
     image: cluedin/cluedin-micro-annotation:${CLUEDIN_ANNOTATION_TAG}
@@ -32,6 +37,7 @@ services:
     - SeqURL=http://${CLUEDIN_SEQ_HOST:-seq}:5341
     - SeqLoggingLevel=debug
     - ForceTableCreation=${CLUEDIN_ANNOTATION_FORCETABLECREATION:-false}
+    - NODE_ENV=${CLUEDIN_ENVIRONMENT}
     ports:
       - "${CLUEDIN_ANNOTATION_LOCALPORT:-9010}:8888"
     depends_on:
@@ -54,6 +60,7 @@ services:
     - AnnotationURL=http://${CLUEDIN_ANNOTATION_HOST:-annotation}:${CLUEDIN_ANNOTATION_PORT:-8888}
     - ConsoleLoggingLevel=info
     - ForceTableCreation=${CLUEDIN_DATASOURCE_FORCETABLECREATION:-false}
+    - NODE_ENV=${CLUEDIN_ENVIRONMENT}
     ports:
       - "${CLUEDIN_DATASOURCE_LOCALPORT:-9011}:8888"
     depends_on:

diff --git a/docker/compose/docker-compose.server.yml b/docker/compose/docker-compose.server.yml
@@ -5,7 +5,7 @@ services:
     environment:
       - CLUEDIN_appSettings__AgentServerUrl=http://${CLUEDIN_DOMAIN}:9000
       - CLUEDIN_appSettings__AuthReturnUrl=http://${CLUEDIN_DOMAIN}:9001
-      - CLUEDIN_appSettings__AuthServerUrl=http://*:9001
+      - CLUEDIN_appSettings__AuthServerUrl=http://${CLUEDIN_DOMAIN}:9001
       - CLUEDIN_appSettings__Domain=${CLUEDIN_DOMAIN}
       - CLUEDIN_appSettings__JobServerDashboardUrl=http://*:9003
       - CLUEDIN_appSettings__Notifications_AdminNotifications_Recipient=admin@foobar.com
@@ -50,8 +50,11 @@ services:
       - CLUEDIN_appSettings__EmailServer=${CLUEDIN_EMAIL_HOST}
       - CLUEDIN_appSettings__EmailPort=${CLUEDIN_EMAIL_PORT}
       - CLUEDIN_appSettings__EmailSender=${CLUEDIN_EMAIL_SENDER}
+      - CLUEDIN_appSettings__EmailDirectory=${CLUEDIN_EMAIL_DIR:-}
+      - CLUEDIN_appSettings__JobServer_DashboardVisible=${CLUEDIN_JOBS_DASHBOARD:-localOnly}
     volumes:
         - "./components:/components"
+        - ".${CLUEDIN_EMAIL_DIR:-}:/app/${CLUEDIN_EMAIL_DIR:-emails}"
     ports:
       - "${CLUEDIN_SERVER_LOCALPORT:-9000}:9000"
       - "${CLUEDIN_SERVER_AUTH_LOCALPORT:-9001}:9001"

diff --git a/docker/compose/docker-compose.ui.yml b/docker/compose/docker-compose.ui.yml
@@ -3,19 +3,20 @@ services:
   ui:
     image: cluedin/ui:${CLUEDIN_UI_TAG}
     ports:
-    - "${CLUEDIN_UI_LOCALPORT:-80}:80"
+    - "${CLUEDIN_UI_LOCALPORT:-9080}:80"
     environment:
     - REACT_APP_UI_APP=http://app.${CLUEDIN_DOMAIN}
     - REACT_APP_UI_APP_SUBDOMAIN=app
     - REACT_APP_UI_BUCKET=/public/img/providers/
     - REACT_APP_UI_CDN_URL=/
     - REACT_APP_UI_CLUEDIN_APP_DOMAIN=${CLUEDIN_DOMAIN}
-    - REACT_APP_UI_CLUEDIN_FEATURES=follow;savedSearches;users;data-governance;developer;settings;integrations;extended-sar;cluedinClean;annotation;datasource
+    - REACT_APP_UI_CLUEDIN_FEATURES=${CLUEDIN_UI_FEATURES_ANNOTATION};${CLUEDIN_UI_FEATURES_CLEAN};${CLUEDIN_UI_FEATURES_CONNECTOR};${CLUEDIN_UI_FEATURES_DATAGOVERNANCE};${CLUEDIN_UI_FEATURES_DATASOURCE};${CLUEDIN_UI_FEATURES_DEVELOPER};${CLUEDIN_UI_FEATURES_ENDPOINT};${CLUEDIN_UI_FEATURES_ENRICHER};${CLUEDIN_UI_FEATURES_EXTENDEDSAR};${CLUEDIN_UI_FEATURES_FOLLOW};${CLUEDIN_UI_FEATURES_GLOSSARY};${CLUEDIN_UI_FEATURES_INTEGRATIONS};${CLUEDIN_UI_FEATURES_METRICS};${CLUEDIN_UI_FEATURES_OPENREFINE};${CLUEDIN_UI_FEATURES_PREPARATION};${CLUEDIN_UI_FEATURES_SAVEDSEARCHES};${CLUEDIN_UI_FEATURES_SETTINGS};${CLUEDIN_UI_FEATURES_STREAMS};${CLUEDIN_UI_FEATURES_USERS}
     - REACT_APP_UI_CLUEDIN_CLEAN_URL=http://${CLUEDIN_DOMAIN}:${CLUEDIN_OPENREFINE_LOCALPORT:-3333}/
     - REACT_APP_UI_GQL_URL=http://${CLUEDIN_DOMAIN}:${CLUEDIN_GQL_LOCALPORT:-8888}/
+    - REACT_APP_UI_INTERNAL_GQL_URL=${CLUEDIN_UI_ALT_UPLOAD:-http://gql:8888}
     - REACT_APP_UI_COOKIE_DOMAIN_NAME=${CLUEDIN_DOMAIN}
     - REACT_APP_UI_ENV=production
-    - REACT_APP_UI_MAIN_DOMAIN=${CLUEDIN_DOMAIN}
+    - REACT_APP_UI_MAIN_DOMAIN=${CLUEDIN_DOMAIN}:${CLUEDIN_UI_LOCALPORT:-9080}
     - REACT_APP_UI_PROTOCOL=http://
     depends_on:
     - server
@@ -31,6 +32,7 @@ services:
         - "APP_API_URL=http://${CLUEDIN_WEBAPI_HOST:-webapi}:9008/"
         - "PUBLIC_API=http://${CLUEDIN_SERVER_HOST:-server}:9007/"
         - "WEBHOOKS_URL=http://${CLUEDIN_SERVER_HOST:-server}:9006/"
+        - "NODE_ENV=${CLUEDIN_ENVIRONMENT}"
     ports:
     - "${CLUEDIN_GQL_LOCALPORT:-8888}:8888"
     depends_on:

diff --git a/docker/compose/docker-compose.yml b/docker/compose/docker-compose.yml
@@ -26,7 +26,7 @@ services:
         ports:
             - "${CLUEDIN_REDIS_LOCALPORT:-6379}:6379"
     seq:
-        image: datalust/seq:5
+        image: datalust/seq:latest
         environment:
             ACCEPT_EULA: 'Y'
         ports:

diff --git a/env/default/.env b/env/default/.env
@@ -32,6 +32,7 @@ CLUEDIN_GQL_HOST=
 CLUEDIN_GQL_LOCALPORT=
 CLUEDIN_GQL_TAG=latest
 CLUEDIN_INSTALLER_TAG=latest
+CLUEDIN_JOBS_DASHBOARD=true
 CLUEDIN_NEO4J_BOLT_LOCALPORT=
 CLUEDIN_NEO4J_HOST=
 CLUEDIN_NEO4J_HTTP_LOCALPORT=
@@ -65,6 +66,26 @@ CLUEDIN_SUBMITTER_PORT=
 CLUEDIN_SUBMITTER_TAG=latest
 CLUEDIN_UI_LOCALPORT=
 CLUEDIN_UI_TAG=latest
+CLUEDIN_UI_ALT_UPLOAD=
+CLUEDIN_UI_FEATURES_ANNOTATION=annotation
+CLUEDIN_UI_FEATURES_CLEAN=
+CLUEDIN_UI_FEATURES_CONNECTOR=connector
+CLUEDIN_UI_FEATURES_DATAGOVERNANCE=data-governance
+CLUEDIN_UI_FEATURES_DATASOURCE=datasource
+CLUEDIN_UI_FEATURES_DEVELOPER=developer
+CLUEDIN_UI_FEATURES_ENDPOINT=endpoint
+CLUEDIN_UI_FEATURES_ENRICHER=enricher
+CLUEDIN_UI_FEATURES_EXTENDEDSAR=extended-sar
+CLUEDIN_UI_FEATURES_FOLLOW=
+CLUEDIN_UI_FEATURES_GLOSSARY=glossary
+CLUEDIN_UI_FEATURES_INTEGRATIONS=integrations
+CLUEDIN_UI_FEATURES_METRICS=metrics
+CLUEDIN_UI_FEATURES_OPENREFINE=cluedinClean
+CLUEDIN_UI_FEATURES_PREPARATION=preparation
+CLUEDIN_UI_FEATURES_SAVEDSEARCHES=
+CLUEDIN_UI_FEATURES_SETTINGS=settings
+CLUEDIN_UI_FEATURES_STREAMS=streams;stream
+CLUEDIN_UI_FEATURES_USERS=users
 CLUEDIN_WEBAPI_HOST=
 CLUEDIN_WEBAPI_LOCALPORT=
 CLUEDIN_WEBAPI_TAG=latest
diff --git a/powershell/bootstrap.ps1 b/powershell/bootstrap.ps1
@@ -1,3 +1,23 @@
+function Invoke-Open {
+    [CluedInAction(Action = 'open', Header = 'Opening Cluedin')]
+    [CmdletBinding()]
+    param(
+        [Parameter(Position=0)]
+        [String]$Env = 'default',
+        [String]$Org = 'app'
+    )
+
+    $env = GetEnvironment -Name $Env
+    if(!$env) {
+        throw "Environment ${Env} could not be found"
+    }
+
+    $port  = $env.CLUEDIN_UI_LOCALPORT ?? '9080'
+    $domain = $env.CLUEDIN_DOMAIN ?? '127.0.0.1.xip.io'
+    $address = "http://${Org}.${domain}:${port}"
+    Start-Process $address
+}
+
 function Invoke-CreateOrg {
     [CluedInAction(Action = 'createorg', Header = 'Create Organization')]
     [CmdletBinding()]
@@ -8,21 +28,22 @@ function Invoke-CreateOrg {
         [string]$Name,
         [ValidatePattern("^[a-zA-Z0-9.!#$%&*+\/=?^_{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$")]
         [string]$Email = "admin@${Name}.com",
-        [string]$Pass = "P@ssword!123"
+        [string]$Pass = "P@ssword!123",
+        [switch]$AllowEmailSignup = $false
     )
 
     $serverPort  = GetEnvironmentValue -Name $Env -Key 'CLUEDIN_SERVER_AUTH_LOCALPORT' -DefaultValue '9001'
-
+    $domain = $env.CLUEDIN_DOMAIN ?? 'localhost'
     $requestArgs = @{
         Method = 'POST'
-        Uri = "http://localhost:${serverPort}/api/account/new"
+        Uri = "http://${domain}:${serverPort}/api/account/new"
         Headers = @{
             'Content-Type' = 'application/x-www-form-urlencoded'
             'cache-control' = 'no-cache'
         }
         Body = @{
             grant_type = 'password'
-            allowEmailDomainSignup = 'false'
+            allowEmailDomainSignup = $AllowEmailSignup
             username = $Email
             email = $Email
             password = $Pass
@@ -41,8 +62,19 @@ function Invoke-CreateOrg {
     } catch {
         $success = $false
         Write-Host "Create organization was not successful"
-        if($_.ErrorDetails.Message) {
-            ($_.ErrorDetails.Message | ConvertFrom-Json).password.'$values'
+        $ex = $_
+        if($ex.ErrorDetails.Message) {
+            try {
+                $json = $_.ErrorDetails.Message | ConvertFrom-Json
+                $json.psobject.members.name |
+                    ForEach-Object {
+                        if($json.$_.psobject.members.name -contains '$values') {
+                            $json.$_.'$values'
+                        }
+                    }
+            } catch {
+                Write-Host $ex.ErrorDetails.Message
+            }
         }
     } finally {
         If($success) {

diff --git a/powershell/checks.ps1 b/powershell/checks.ps1
@@ -5,7 +5,12 @@ function Test-Environment {
         [String]$Env = 'default'
     )
 
-    $cluedinDomain  = GetEnvironmentValue -Name $Env -Key 'CLUEDIN_DOMAIN' -DefaultValue 'localhost'
+    $env = GetEnvironment -Name $Env
+    if(!$env) {
+        throw "Environment ${Env} could not be found"
+    }
+
+    $cluedinDomain = $env.CLUEDIN_DOMAIN ?? 'localhost'
 
     $appChecks = [CheckCollection]::new('Installed Applications', @(
         [Check]::new('PowerShell', {
@@ -36,29 +41,29 @@ function Test-Environment {
     ))
 
     $portChecks = [CheckCollection]::new('Available Ports', @(
-        [PortCheck]::new('CluedIn UI', 80, $cluedinDomain)
-        [PortCheck]::new('CluedIn API', 9000, $cluedinDomain)
-        [PortCheck]::new('CluedIn Auth', 9001, $cluedinDomain)
-        [PortCheck]::new('CluedIn Jobs', 9003, $cluedinDomain)
-        [PortCheck]::new('CluedIn WebHooks', 9006, $cluedinDomain)
-        [PortCheck]::new('CluedIn Public', 9007, $cluedinDomain)
-        [PortCheck]::new('CluedIn Web Api', 9008, $cluedinDomain)
-        [PortCheck]::new('CluedIn Clean', 9009, $cluedinDomain)
-        [PortCheck]::new('CluedIn Annotation', 9010, $cluedinDomain)
-        [PortCheck]::new('CluedIn Datasource', 9011, $cluedinDomain)
-        [PortCheck]::new('CluedIn Submitter', 9012, $cluedinDomain)
-        [PortCheck]::new('CluedIn Gql', 8888, $cluedinDomain)
-        [PortCheck]::new('Neo4j Data', 7474, 'localhost')
-        [PortCheck]::new('Neo4j Bolt', 7687, 'localhost')
-        [PortCheck]::new('Elasticsearch Data', 9200, 'localhost')
-        [PortCheck]::new('Elasticsearch Http', 9300, 'localhost')
-        [PortCheck]::new('RabbitMQ Data', 5672, 'localhost')
-        [PortCheck]::new('RabbitMQ Http', 15672, 'localhost')
-        [PortCheck]::new('Redis', 6379, 'localhost')
-        [PortCheck]::new('Seq Http', 3200, 'localhost')
-        [PortCheck]::new('Seq Data', 5341, 'localhost')
-        [PortCheck]::new('Sql Server', 1433, 'localhost')
-        [PortCheck]::new('OpenRefine', 3333, 'localhost')
+        [PortCheck]::new('CluedIn UI', $env.CLUEDIN_UI_LOCALPORT ?? 9080, $cluedinDomain)
+        [PortCheck]::new('CluedIn API', $env.CLUEDIN_SERVER_LOCALPORT ?? 9000, $cluedinDomain)
+        [PortCheck]::new('CluedIn Auth', $env.CLUEDIN_SERVER_AUTH_LOCALPORT ?? 9001, $cluedinDomain)
+        [PortCheck]::new('CluedIn Jobs', $env.CLUEDIN_SERVER_JOB_LOCALPORT ?? 9003, $cluedinDomain)
+        [PortCheck]::new('CluedIn WebHooks', $env.CLUEDIN_SERVER_WEBHOOK_LOCALPORT ?? 9006, $cluedinDomain)
+        [PortCheck]::new('CluedIn Public', $env.CLUEDIN_SERVER_PUBLIC_LOCALPORT ?? 9007, $cluedinDomain)
+        [PortCheck]::new('CluedIn Web Api', $env.CLUEDIN_WEBAPI_LOCALPORT ?? 9008, $cluedinDomain)
+        [PortCheck]::new('CluedIn Clean', $env.CLUEDIN_CLEAN_LOCALPORT ?? 9009, $cluedinDomain)
+        [PortCheck]::new('CluedIn Annotation', $env.CLUEDIN_ANNOTATION_LOCALPORT ?? 9010, $cluedinDomain)
+        [PortCheck]::new('CluedIn Datasource', $env.CLUEDIN_DATASOURCE_LOCALPORT ?? 9011, $cluedinDomain)
+        [PortCheck]::new('CluedIn Submitter', $env.CLUEDIN_SUBMITTER_LOCALPORT ?? 9012, $cluedinDomain)
+        [PortCheck]::new('CluedIn Gql', $env.CLUEDIN_GQL_LOCALPORT ?? 8888, $cluedinDomain)
+        [PortCheck]::new('Neo4j Http', $env.CLUEDIN_NEO4J_HTTP_LOCALPORT ?? 7474, 'localhost')
+        [PortCheck]::new('Neo4j Bolt', $env.CLUEDIN_NEO4J_BOLT_LOCALPORT ?? 7687, 'localhost')
+        [PortCheck]::new('Elasticsearch Data', $env.CLUEDIN_ELASTIC_DATA_LOCALPORT ?? 9200, 'localhost')
+        [PortCheck]::new('Elasticsearch Http', $env.CLUEDIN_ELASTIC_HTTP_LOCALPORT ?? 9300, 'localhost')
+        [PortCheck]::new('RabbitMQ Data', $env.CLUEDIN_RABBITMQ_DATA_LOCALPORT ?? 5672, 'localhost')
+        [PortCheck]::new('RabbitMQ Http', $env.CLUEDIN_RABBITMQ_HTTP_LOCALPORT ?? 15672, 'localhost')
+        [PortCheck]::new('Redis', $env.CLUEDIN_REDIS_LOCALPORT ?? 6379, 'localhost')
+        [PortCheck]::new('Seq UI', $env.CLUEDIN_SEQ_UI_LOCALPORT ?? 3200, 'localhost')
+        [PortCheck]::new('Seq Data', $env.CLUEDIN_SEQ_DATA_LOCALPORT ?? 5341, 'localhost')
+        [PortCheck]::new('Sql Server', $env.CLUEDIN_SQLSERVER_LOCALPORT ?? 1433, 'localhost')
+        [PortCheck]::new('OpenRefine', $env.CLUEDIN_OPENREFINE_LOCALPORT ?? 3333, 'localhost')
     ))
 
     $authChecks = [CheckCollection]::new('Authentication', @(

diff --git a/powershell/cluedin.classes.ps1 b/powershell/cluedin.classes.ps1
@@ -197,7 +197,7 @@ class EnvironmentToggle {
             else { $current = $current.Value }
             $this.Container[$Variable] = $current
         }
-        Set-Item $path $Value
+        Set-Item $path.ToUpper() $Value
     }
 
     [void] Reset() {

diff --git a/powershell/docker.ps1 b/powershell/docker.ps1
@@ -25,8 +25,8 @@ function DockerCompose {
 
     $compose = "docker-compose $projectName $composeFiles $envPath $action"
 
-    $disableFags = $Disable | Where-Object { $_ } | ForEach-Object { "--scale ${_}=0" }
-    if($disableFags) { $compose += " ${disableFags}" }
+    $disableFlags = $Disable | Where-Object { $_ } | ForEach-Object { "--scale ${_}=0" }
+    if($disableFlags) { $compose += " ${disableFlags}" }
 
     if($AdditionalArgs) { $compose += " ${AdditionalArgs}" }
 

diff --git a/powershell/environment.ps1 b/powershell/environment.ps1
@@ -10,6 +10,7 @@ function Invoke-Environment {
         [Parameter(Position=0, ParameterSetName='get')]
         [Parameter(Position=0, ParameterSetName='unset')]
         [Parameter(Position=0, ParameterSetName='remove')]
+        [Parameter(Position=0, ParameterSetName='setTagOverride')]
         [string]$Name = 'default',
         [Parameter(ParameterSetName='set', Mandatory)]
         [string[]]$Set,
@@ -18,6 +19,7 @@ function Invoke-Environment {
         [string]$Tag = [string]::Empty,
         [Parameter(ParameterSetName='set')]
         [Parameter(ParameterSetName='setTag')]
+        [Parameter(ParameterSetName='setTagOverride', Mandatory)]
         [string[]]$TagOverride = @(),
         [Parameter(ParameterSetName='get')]
         [switch]$Get,
@@ -31,7 +33,7 @@ function Invoke-Environment {
     process {
 
         switch ($PSCmdlet.ParameterSetName) {
-            {$_ -in @('set', 'setTag')} {
+            {$_ -in @('set', 'setTag', 'setTagOverride')} {
                 $env = (GetEnvironment $Name) ?? (GetEnvironment 'default')
 
                 if($Set) {