From 9354afb55c55105f11d1f0acfb032e73b517ea48 Mon Sep 17 00:00:00 2001 From: Fernando Arreola Date: Thu, 17 Aug 2023 15:13:54 -0700 Subject: [PATCH 1/4] Add support for external connections. --- package-lock.json | 177 ++++++++++--- package.json | 13 +- src/common/connection_manager.ts | 20 +- src/common/connection_manager_types.ts | 34 ++- .../connections/browser/connection_factory.ts | 9 + .../external_connection_factory.ts | 122 +++++++++ .../connections/node/connection_factory.ts | 40 ++- src/common/connections/types.ts | 5 + src/common/errors.ts | 2 +- src/common/message_types.ts | 40 ++- .../browser/commands/edit_connections.ts | 1 + .../node/commands/edit_connections.ts | 22 ++ .../webviews/connections_page/App.tsx | 28 ++ .../ConnectionEditor/ ConnectionEditor.tsx | 21 +- .../ExternalConnectionEditor.tsx | 245 ++++++++++++++++++ .../ExternalConnectionEditor/index.ts | 24 ++ .../ConnectionEditorList.tsx | 16 +- 17 files changed, 765 insertions(+), 54 deletions(-) create mode 100644 src/common/connections/external_connection_factory.ts create mode 100644 src/extension/webviews/connections_page/ConnectionEditor/ExternalConnectionEditor/ExternalConnectionEditor.tsx create mode 100644 src/extension/webviews/connections_page/ConnectionEditor/ExternalConnectionEditor/index.ts diff --git a/package-lock.json b/package-lock.json index 35dc4591..ab84ee4f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,16 +9,17 @@ "version": "0.2.0", "license": "MIT", "dependencies": { - "@malloydata/db-bigquery": "0.0.71", - "@malloydata/db-duckdb": "0.0.71", - "@malloydata/db-postgres": "0.0.71", - "@malloydata/malloy": "0.0.71", - "@malloydata/malloy-sql": "0.0.71", - "@malloydata/render": "0.0.71", + "@malloydata/db-bigquery": "0.0.77", + "@malloydata/db-duckdb": "0.0.77", + "@malloydata/db-postgres": "0.0.77", + "@malloydata/malloy": "0.0.77", + "@malloydata/malloy-sql": "0.0.77", + "@malloydata/render": "0.0.77", "@popperjs/core": "^2.11.6", "@vscode/webview-ui-toolkit": "^1.2.1", "duckdb": "0.8.1", "keytar": "7.7.0", + "live-plugin-manager": "^0.18.1", "lodash": "^4.17.21", "node-fetch": "^2.6.6", "prismjs": "^1.28.0", @@ -2374,14 +2375,14 @@ } }, "node_modules/@malloydata/db-bigquery": { - "version": "0.0.71", - "resolved": "https://registry.npmjs.org/@malloydata/db-bigquery/-/db-bigquery-0.0.71.tgz", - "integrity": "sha512-XD0sntgQuyyFQlBUyNtY8QRV2cn/M5LvUy2b163M8WzIz/IjXALM/DJhioR0Y7BJY9Ra1Yxvu4PWBV08pQpceA==", + "version": "0.0.77", + "resolved": "https://registry.npmjs.org/@malloydata/db-bigquery/-/db-bigquery-0.0.77.tgz", + "integrity": "sha512-+/Yf/d/annZYVSSrqc+195nR2z4CNK/T2kDGOm8bs94644J8PdaDJiBHmDkDe4Jr+si22lw+/5x2Nnqd5CjZIQ==", "dependencies": { "@google-cloud/bigquery": "^5.5.0", "@google-cloud/common": "^3.6.0", "@google-cloud/paginator": "^4.0.1", - "@malloydata/malloy": "^0.0.71", + "@malloydata/malloy": "^0.0.77", "gaxios": "^4.2.0" }, "engines": { @@ -2389,12 +2390,12 @@ } }, "node_modules/@malloydata/db-duckdb": { - "version": "0.0.71", - "resolved": "https://registry.npmjs.org/@malloydata/db-duckdb/-/db-duckdb-0.0.71.tgz", - "integrity": "sha512-IDH5cVJUCmzSdqFmxjZC3DlhyXUvZQWIGZX9E2OcHu1zYp9KDHrFoR0TIf7BhmaUZIjmI7FdcNp8JZrhq66tXw==", + "version": "0.0.77", + "resolved": "https://registry.npmjs.org/@malloydata/db-duckdb/-/db-duckdb-0.0.77.tgz", + "integrity": "sha512-5ykLEqjM+848Ey3fJIWMfMsCocTdgGf/eH7dVA1NDZCr4c/jOsZjcD/OeL7YidULF1jtfTu7YUdCmrJFtwNsNA==", "dependencies": { "@malloydata/duckdb-wasm": "0.0.2", - "@malloydata/malloy": "^0.0.71", + "@malloydata/malloy": "^0.0.77", "apache-arrow": "^11.0.0", "duckdb": "0.8.1", "web-worker": "^1.2.0" @@ -2404,11 +2405,11 @@ } }, "node_modules/@malloydata/db-postgres": { - "version": "0.0.71", - "resolved": "https://registry.npmjs.org/@malloydata/db-postgres/-/db-postgres-0.0.71.tgz", - "integrity": "sha512-XfDiYq/GKZMOgj1SnXnxqDkH83Bei+sZQ9l74wTtmotDxSg4Zi86FDM7KD56xBvLEbR54Qb7gmWWuoJxR66eXQ==", + "version": "0.0.77", + "resolved": "https://registry.npmjs.org/@malloydata/db-postgres/-/db-postgres-0.0.77.tgz", + "integrity": "sha512-NWr9NHzjsuPYZ3MV3Fw/7YdqD2a2vJtTRmqGmqEbL1SRtfFquQdgjcf8YGC/GTGNzI2oOug0b3GY2bWpsBAm5A==", "dependencies": { - "@malloydata/malloy": "^0.0.71", + "@malloydata/malloy": "^0.0.77", "@types/pg": "^8.6.1", "pg": "^8.7.1", "pg-query-stream": "4.2.3" @@ -2422,9 +2423,9 @@ } }, "node_modules/@malloydata/malloy": { - "version": "0.0.71", - "resolved": "https://registry.npmjs.org/@malloydata/malloy/-/malloy-0.0.71.tgz", - "integrity": "sha512-svaXp2TSP9Tp0aA/sDbfuAFS2qtLY3pRNF4VCGX6JABiOCMZwmb2glWj9o4BqSeAnrtrkN4gHGdDq2ERcTEKBA==", + "version": "0.0.77", + "resolved": "https://registry.npmjs.org/@malloydata/malloy/-/malloy-0.0.77.tgz", + "integrity": "sha512-7v3cDrdVqonDoc5uQclMCmxzfcocNLiJFFjHixiXC3+RkN2MHtHKRYhlsZ1rkIkS6so7Gh855MlAfXo2pQztzw==", "dependencies": { "antlr4ts": "^0.5.0-alpha.4", "assert": "^2.0.0", @@ -2438,19 +2439,19 @@ } }, "node_modules/@malloydata/malloy-sql": { - "version": "0.0.71", - "resolved": "https://registry.npmjs.org/@malloydata/malloy-sql/-/malloy-sql-0.0.71.tgz", - "integrity": "sha512-Y6+xGu58qHa1Jrnz7Ws34AKCqirVdIh2gAdFgZKM+Om0t+3KneQ9IblPCUWW90+kD8zoWNys6jDPRL5w7ytCkQ==", + "version": "0.0.77", + "resolved": "https://registry.npmjs.org/@malloydata/malloy-sql/-/malloy-sql-0.0.77.tgz", + "integrity": "sha512-od3FWDERunvCwDq6urs41//VnLywojm+znx8LmsHnCWS3yG/EtAHCS8vIz5OZuhvCQmIlRtQsOLOU08GzpzPXQ==", "dependencies": { - "@malloydata/malloy": "^0.0.71" + "@malloydata/malloy": "^0.0.77" } }, "node_modules/@malloydata/render": { - "version": "0.0.71", - "resolved": "https://registry.npmjs.org/@malloydata/render/-/render-0.0.71.tgz", - "integrity": "sha512-eXe+V5+TlKQpmRHhf/kT1tGVsgwkw9bYMoyetnm0YJHDq+b3d1IXxpdO/z3KyWShyv7WqZyCfA9egLVrm88MKA==", + "version": "0.0.77", + "resolved": "https://registry.npmjs.org/@malloydata/render/-/render-0.0.77.tgz", + "integrity": "sha512-QAJrl8RERaFTUbW+NSCYjGelyYBe1Rb9f/V5G1V2We5lYLev9AUqIcIN3crPZHsunWKJT9P2xMkPh1lMZun+gA==", "dependencies": { - "@malloydata/malloy": "^0.0.71", + "@malloydata/malloy": "^0.0.77", "@types/luxon": "^2.4.0", "lodash": "^4.17.20", "luxon": "^2.4.0", @@ -2887,6 +2888,14 @@ "version": "5.0.2", "license": "MIT" }, + "node_modules/@types/debug": { + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.8.tgz", + "integrity": "sha512-/vPO1EPOs306Cvhwv7KfVfYvOJqA/S/AXjaHQiJboCZzcNDb+TIJFN9/2C9DZ//ijSKWioNyUxD792QmDJ+HKQ==", + "dependencies": { + "@types/ms": "*" + } + }, "node_modules/@types/estree": { "version": "1.0.1", "license": "MIT" @@ -2895,6 +2904,14 @@ "version": "1.10.0", "license": "MIT" }, + "node_modules/@types/fs-extra": { + "version": "9.0.13", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.13.tgz", + "integrity": "sha512-nEnwB++1u5lVDM2UI4c1+5R+FYaKfaAzS4OococimjVm3nQw3TuzH5UNsocrcTBbhnerblyHj4A49qXbIiZdpA==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/geojson": { "version": "7946.0.4", "license": "MIT" @@ -3136,6 +3153,11 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/lockfile": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@types/lockfile/-/lockfile-1.0.2.tgz", + "integrity": "sha512-jD5VbvhfMhaYN4M3qPJuhMVUg3Dfc4tvPvLEAXn6GXbs/ajDFtCQahX37GIE65ipTI3I+hEvNaXS3MYAn9Ce3Q==" + }, "node_modules/@types/lodash": { "version": "4.14.191", "dev": true, @@ -3164,13 +3186,17 @@ "integrity": "sha512-/fvYntiO1GeICvqbQ3doGDIP97vWmvFt83GKguJ6prmQM2iXZfFcq6YE8KteFyRtX2/h5Hf91BYvPodJKFYv5Q==", "dev": true }, + "node_modules/@types/ms": { + "version": "0.7.31", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.31.tgz", + "integrity": "sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==" + }, "node_modules/@types/node": { "version": "14.18.37", "license": "MIT" }, "node_modules/@types/node-fetch": { "version": "2.6.2", - "dev": true, "license": "MIT", "dependencies": { "@types/node": "*", @@ -3245,7 +3271,6 @@ }, "node_modules/@types/semver": { "version": "7.3.13", - "dev": true, "license": "MIT" }, "node_modules/@types/stack-utils": { @@ -3263,6 +3288,15 @@ "csstype": "^3.0.2" } }, + "node_modules/@types/tar": { + "version": "6.1.5", + "resolved": "https://registry.npmjs.org/@types/tar/-/tar-6.1.5.tgz", + "integrity": "sha512-qm2I/RlZij5RofuY7vohTpYNaYcrSQlN2MyjucQc7ZweDwaEWkdN/EeNh6e9zjK6uEm6PwjdMXkcj05BxZdX1Q==", + "dependencies": { + "@types/node": "*", + "minipass": "^4.0.0" + } + }, "node_modules/@types/tar-stream": { "version": "2.2.2", "dev": true, @@ -3271,11 +3305,24 @@ "@types/node": "*" } }, + "node_modules/@types/tar/node_modules/minipass": { + "version": "4.2.8", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-4.2.8.tgz", + "integrity": "sha512-fNzuVyifolSLFL4NzpF+wEF4qrgqaaKX0haXPQEdQ7NKAN+WecoKMHV09YcuL/DHxrUsYQOK3MiuDf7Ip2OXfQ==", + "engines": { + "node": ">=8" + } + }, "node_modules/@types/tough-cookie": { "version": "4.0.2", "dev": true, "license": "MIT" }, + "node_modules/@types/url-join": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@types/url-join/-/url-join-4.0.1.tgz", + "integrity": "sha512-wDXw9LEEUHyV+7UWy7U315nrJGJ7p1BzaCxDpEoLr789Dk1WDVMMlf3iBfbG2F8NdWnYyFbtTxUn2ZNbm1Q4LQ==" + }, "node_modules/@types/uuid": { "version": "8.3.4", "dev": true, @@ -4008,7 +4055,6 @@ }, "node_modules/asynckit": { "version": "0.4.0", - "dev": true, "license": "MIT" }, "node_modules/available-typed-arrays": { @@ -4882,7 +4928,6 @@ }, "node_modules/combined-stream": { "version": "1.0.8", - "dev": true, "license": "MIT", "dependencies": { "delayed-stream": "~1.0.0" @@ -5607,7 +5652,6 @@ }, "node_modules/delayed-stream": { "version": "1.0.0", - "dev": true, "license": "MIT", "engines": { "node": ">=0.4.0" @@ -7120,7 +7164,6 @@ }, "node_modules/form-data": { "version": "3.0.1", - "dev": true, "license": "MIT", "dependencies": { "asynckit": "^0.4.0", @@ -7151,6 +7194,19 @@ "version": "1.0.0", "license": "MIT" }, + "node_modules/fs-extra": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", + "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/fs-minipass": { "version": "2.1.0", "license": "ISC", @@ -11873,6 +11929,17 @@ "version": "3.2.0", "license": "MIT" }, + "node_modules/jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, "node_modules/jwa": { "version": "2.0.0", "license": "MIT", @@ -12133,6 +12200,27 @@ "dev": true, "license": "ISC" }, + "node_modules/live-plugin-manager": { + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/live-plugin-manager/-/live-plugin-manager-0.18.1.tgz", + "integrity": "sha512-GvLMSaZ1Cc18o91NiHLRuPXm1z7xDiUXUGgQ6jAwGM/x0FY8vXXHa/+LMNb2zrkAV2bWULCs0FEwX9yRsmFZmw==", + "dependencies": { + "@types/debug": "^4.1.7", + "@types/fs-extra": "^9.0.13", + "@types/lockfile": "^1.0.2", + "@types/node-fetch": "^2.5.12", + "@types/semver": "^7.3.9", + "@types/tar": "^6.1.1", + "@types/url-join": "4.0.1", + "debug": "^4.3.3", + "fs-extra": "^10.0.0", + "lockfile": "^1.0.4", + "node-fetch": "^2.6.6", + "semver": "^7.3.5", + "tar": "^6.1.11", + "url-join": "^4.0.1" + } + }, "node_modules/locate-path": { "version": "6.0.0", "dev": true, @@ -12147,6 +12235,14 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/lockfile": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/lockfile/-/lockfile-1.0.4.tgz", + "integrity": "sha512-cvbTwETRfsFh4nHsL1eGWapU1XFi5Ot9E85sWAwia7Y7EgB7vfqcZhTKZ+l7hCGxSPoushMv5GKhT5PdLv03WA==", + "dependencies": { + "signal-exit": "^3.0.2" + } + }, "node_modules/lodash": { "version": "4.17.21", "license": "MIT" @@ -12602,7 +12698,6 @@ }, "node_modules/mime-db": { "version": "1.52.0", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.6" @@ -12610,7 +12705,6 @@ }, "node_modules/mime-types": { "version": "2.1.35", - "dev": true, "license": "MIT", "dependencies": { "mime-db": "1.52.0" @@ -16689,6 +16783,14 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "engines": { + "node": ">= 10.0.0" + } + }, "node_modules/unzipper": { "version": "0.10.11", "dev": true, @@ -16768,7 +16870,6 @@ }, "node_modules/url-join": { "version": "4.0.1", - "dev": true, "license": "MIT" }, "node_modules/us-atlas": { diff --git a/package.json b/package.json index d14010e4..a25da1bf 100644 --- a/package.json +++ b/package.json @@ -549,16 +549,17 @@ ] }, "dependencies": { - "@malloydata/db-bigquery": "0.0.71", - "@malloydata/db-duckdb": "0.0.71", - "@malloydata/db-postgres": "0.0.71", - "@malloydata/malloy": "0.0.71", - "@malloydata/malloy-sql": "0.0.71", - "@malloydata/render": "0.0.71", + "@malloydata/db-bigquery": "0.0.77", + "@malloydata/db-duckdb": "0.0.77", + "@malloydata/db-postgres": "0.0.77", + "@malloydata/malloy": "0.0.77", + "@malloydata/malloy-sql": "0.0.77", + "@malloydata/render": "0.0.77", "@popperjs/core": "^2.11.6", "@vscode/webview-ui-toolkit": "^1.2.1", "duckdb": "0.8.1", "keytar": "7.7.0", + "live-plugin-manager": "^0.18.1", "lodash": "^4.17.21", "node-fetch": "^2.6.6", "prismjs": "^1.28.0", diff --git a/src/common/connection_manager.ts b/src/common/connection_manager.ts index aa7ecca7..900627ef 100644 --- a/src/common/connection_manager.ts +++ b/src/common/connection_manager.ts @@ -26,7 +26,11 @@ import { LookupConnection, TestableConnection, } from '@malloydata/malloy'; -import {ConfigOptions, ConnectionConfig} from './connection_manager_types'; +import { + ConfigOptions, + ConnectionConfig, + ExternalConnectionConfig, +} from './connection_manager_types'; import {ConnectionFactory} from './connections/types'; const DEFAULT_CONFIG = Symbol('default-config'); @@ -53,7 +57,11 @@ export class DynamicConnectionLookup implements LookupConnection { ...this.options, }); } else { - throw new Error(`No connection found with name ${connectionName}`); + throw new Error( + `No connection found with name ${connectionName} ${ + new Error('abc').stack + }` + ); } } return this.connections[connectionKey]; @@ -121,6 +129,14 @@ export class ConnectionManager { return this.filterUnavailableConnectionBackends(this.configList); } + public async installExternalConnectionPackage( + connectionConfig: ExternalConnectionConfig + ): Promise { + return this.connectionFactory.installExternalConnectionPackage( + connectionConfig + ); + } + public getAvailableBackends() { return this.connectionFactory.getAvailableBackends(); } diff --git a/src/common/connection_manager_types.ts b/src/common/connection_manager_types.ts index 39961637..4276c0c6 100644 --- a/src/common/connection_manager_types.ts +++ b/src/common/connection_manager_types.ts @@ -21,11 +21,18 @@ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +import { + ConnectionConfigSchema, + ConnectionConfig as MalloyConnectionConfig, +} from '@malloydata/malloy'; +import {ExternalConnectionPackageInfo} from './connections/external_connection_factory'; + export enum ConnectionBackend { BigQuery = 'bigquery', Postgres = 'postgres', DuckDB = 'duckdb', DuckDBWASM_DEPRECATED = 'duckdb_wasm', + External = 'external', } export const ConnectionBackendNames: Record = { @@ -33,6 +40,8 @@ export const ConnectionBackendNames: Record = { [ConnectionBackend.Postgres]: 'Postgres', [ConnectionBackend.DuckDB]: 'DuckDB', [ConnectionBackend.DuckDBWASM_DEPRECATED]: 'duckdDuckDBb_wasm', + // TODO(figutierrez): Remove beta once ready. + [ConnectionBackend.External]: 'External (Beta)', }; /* @@ -77,11 +86,34 @@ export interface DuckDBWASMConnectionConfigDeprecated workingDirectory?: string; } +export enum ExternalConnectionSource { + NPM = 'npm', + LocalNPM = 'local_npm', +} + +export const ExternalConnectionSourceNames: Record< + ExternalConnectionSource, + string +> = { + [ExternalConnectionSource.NPM]: 'NPM package', + [ExternalConnectionSource.LocalNPM]: 'Local package', +}; + +export interface ExternalConnectionConfig extends BaseConnectionConfig { + backend: ConnectionBackend.External; + source?: ExternalConnectionSource; + path?: string; + packageInfo?: ExternalConnectionPackageInfo; + connectionSchema?: ConnectionConfigSchema; + configParameters?: MalloyConnectionConfig; +} + export type ConnectionConfig = | BigQueryConnectionConfig | PostgresConnectionConfig | DuckDBConnectionConfig - | DuckDBWASMConnectionConfigDeprecated; + | DuckDBWASMConnectionConfigDeprecated + | ExternalConnectionConfig; export interface ConfigOptions { workingDirectory: string; diff --git a/src/common/connections/browser/connection_factory.ts b/src/common/connections/browser/connection_factory.ts index 24b0429d..f6049f2b 100644 --- a/src/common/connections/browser/connection_factory.ts +++ b/src/common/connections/browser/connection_factory.ts @@ -27,6 +27,7 @@ import { ConfigOptions, ConnectionBackend, ConnectionConfig, + ExternalConnectionConfig, } from '../../connection_manager_types'; import {createDuckDbWasmConnection} from '../duckdb_wasm_connection'; import {DuckDBWASMConnection} from '@malloydata/db-duckdb/wasm'; @@ -38,6 +39,14 @@ export class WebConnectionFactory implements ConnectionFactory { constructor(private fetchBinaryFile?: FetchCallback) {} + installExternalConnectionPackage( + _connectionConfig: ExternalConnectionConfig + ): Promise { + throw new Error( + 'Can not install external packages in the browser for now.' + ); + } + async reset() { await Promise.all( Object.values(this.connectionCache).map(connection => connection.close()) diff --git a/src/common/connections/external_connection_factory.ts b/src/common/connections/external_connection_factory.ts new file mode 100644 index 00000000..ef5ca97d --- /dev/null +++ b/src/common/connections/external_connection_factory.ts @@ -0,0 +1,122 @@ +import { + Connection, + ConnectionFactory, + TestableConnection, +} from '@malloydata/malloy'; +import {registerDialect} from '@malloydata/malloy/dist/dialect'; +import {PluginManager} from 'live-plugin-manager'; +import { + ExternalConnectionConfig, + ExternalConnectionSource, +} from '../connection_manager_types'; + +// TODO(figutierrez): This could be a class instead of static consts, have a common plugin manager. +// TODO(figutierrez): Can we check if package deps are in sync with these? + +export interface ExternalConnectionPackageInfo { + packageName: string; + version: string; +} + +export class ExternalConnectionFactory { + readonly pm = new PluginManager(); + readonly sessionInstalledPackages: Array = []; + + async createOtherConnection( + config: ExternalConnectionConfig + ): Promise { + if (!config.packageInfo) { + throw new Error( + 'Can not instantiate external connection, package info not provided.' + ); + } + await this.installExternalConnectionPackage(config); + const externalConnection = this.pm.require(config.packageInfo.packageName); + // TODO: Cast factory to factory type when available. + if (!externalConnection.connectionFactory) { + throw new Error( + `External package ${config.packageInfo.packageName} does not export factory connectionFactory` + ); + } + + const connection = externalConnection.connectionFactory.createConnection( + config.configParameters ?? {}, + registerDialect + ) as Connection & TestableConnection; + return connection; + } + + async fetchConnectionFactory( + packageInfo: ExternalConnectionPackageInfo + ): Promise { + try { + const externalConnectionPackage = this.pm.require( + packageInfo.packageName + ); + if (externalConnectionPackage.connectionFactory) { + return externalConnectionPackage.connectionFactory as ConnectionFactory; + } + } catch (error) { + throw new Error( + `Could not load connection factory for external connection package ${packageInfo.packageName}. Error: ${error}` + ); + } + + throw new Error( + `Seems like ${packageInfo.packageName} is not a valid connection since it does not export a connectionFactory.` + ); + } + + async installExternalConnectionPackage( + connectionConfig: ExternalConnectionConfig + ): Promise { + // TODO(figutierrez): Explore if force is needed. + // TODO(figutierrez): Add Support for github. + + if (connectionConfig.packageInfo) { + if ( + this.sessionInstalledPackages.indexOf( + connectionConfig.packageInfo.packageName + ) >= 0 + ) { + return connectionConfig.packageInfo; + } + } + + if (!connectionConfig.path) { + throw new Error( + 'Could not install external package, path was not provided.' + ); + } + + switch (connectionConfig.source) { + // TODO(figutierrez): Test this path. + case ExternalConnectionSource.NPM: { + try { + const pi = await this.pm.installFromNpm(connectionConfig.path); + this.sessionInstalledPackages.push(pi.name); + return {packageName: pi.name, version: pi.version}; + } catch (error) { + throw new Error(`Could not install package from NPM. ${error}`); + } + } + case ExternalConnectionSource.LocalNPM: { + try { + const pi = await this.pm.installFromPath(connectionConfig.path, { + force: true, + }); + this.sessionInstalledPackages.push(pi.name); + return {packageName: pi.name, version: pi.version}; + } catch (error) { + throw new Error( + `Could not install package from local path. ${error}` + ); + } + } + } + + throw new Error( + `Could not install package, source ${connectionConfig.source} is not supported.` + ); + } +} diff --git a/src/common/connections/node/connection_factory.ts b/src/common/connections/node/connection_factory.ts index 4f594ae8..d524787c 100644 --- a/src/common/connections/node/connection_factory.ts +++ b/src/common/connections/node/connection_factory.ts @@ -27,6 +27,7 @@ import { ConfigOptions, ConnectionBackend, ConnectionConfig, + ExternalConnectionConfig, } from '../../connection_manager_types'; import {createBigQueryConnection} from '../bigquery_connection'; import {createDuckDbConnection} from '../duckdb_connection'; @@ -34,9 +35,12 @@ import {createPostgresConnection} from '../postgres_connection'; import {isDuckDBAvailable} from '../../duckdb_availability'; import {fileURLToPath} from 'url'; +import {ExternalConnectionFactory} from '../external_connection_factory'; +import {random} from 'lodash'; export class DesktopConnectionFactory implements ConnectionFactory { connectionCache: Record = {}; + externalConnectionFactory = new ExternalConnectionFactory(); reset() { Object.values(this.connectionCache).forEach(connection => @@ -46,7 +50,11 @@ export class DesktopConnectionFactory implements ConnectionFactory { } getAvailableBackends(): ConnectionBackend[] { - const available = [ConnectionBackend.BigQuery, ConnectionBackend.Postgres]; + const available = [ + ConnectionBackend.BigQuery, + ConnectionBackend.Postgres, + ConnectionBackend.External, + ]; if (isDuckDBAvailable) { available.push(ConnectionBackend.DuckDB); } @@ -87,19 +95,44 @@ export class DesktopConnectionFactory implements ConnectionFactory { ); break; } + case ConnectionBackend.External: { + connection = await this.externalConnectionFactory.createOtherConnection( + connectionConfig + ); + break; + } } if (useCache && connection) { this.connectionCache[cacheKey] = connection; } if (!connection) { throw new Error( - `Unsupported connection back end "${connectionConfig.backend}"` + `Unsupported connection back end "${connectionConfig.backend}" ${ + new Error('abc').stack + }` ); } return connection; } + async installExternalConnectionPackage( + connectionConfig: ExternalConnectionConfig + ): Promise { + const packageInfo = + await this.externalConnectionFactory.installExternalConnectionPackage( + connectionConfig + ); + const connectionSpec = + await this.externalConnectionFactory.fetchConnectionFactory(packageInfo); + return { + ...connectionConfig, + packageInfo: packageInfo, + name: connectionSpec.connectionName, + connectionSchema: connectionSpec.configSchema, + }; + } + getWorkingDirectory(url: URL): string { try { const baseUrl = new URL('.', url); @@ -111,6 +144,9 @@ export class DesktopConnectionFactory implements ConnectionFactory { } addDefaults(configs: ConnectionConfig[]): ConnectionConfig[] { + // Create a default bigquery connection if one isn't configured + // TODO(figutierrez): remove. + // Create a default bigquery connection if one isn't configured if ( !configs.find(config => config.backend === ConnectionBackend.BigQuery) diff --git a/src/common/connections/types.ts b/src/common/connections/types.ts index ef06072b..a7585684 100644 --- a/src/common/connections/types.ts +++ b/src/common/connections/types.ts @@ -25,6 +25,7 @@ import { ConfigOptions, ConnectionBackend, ConnectionConfig, + ExternalConnectionConfig, } from '../connection_manager_types'; import {TestableConnection} from '@malloydata/malloy'; @@ -41,4 +42,8 @@ export interface ConnectionFactory { getWorkingDirectory(url: URL): string; addDefaults(configs: ConnectionConfig[]): ConnectionConfig[]; + + installExternalConnectionPackage( + connectionConfig: ExternalConnectionConfig + ): Promise; } diff --git a/src/common/errors.ts b/src/common/errors.ts index 1c329c31..267094a7 100644 --- a/src/common/errors.ts +++ b/src/common/errors.ts @@ -22,7 +22,7 @@ */ export const errorMessage = (error: unknown): string => { - let message = 'Something went wrong'; + let message = `Something went wrong ${JSON.stringify(error)}`; if (error instanceof Error) { message = error.message; } else if (typeof error === 'string') { diff --git a/src/common/message_types.ts b/src/common/message_types.ts index eaffe07a..da41be78 100755 --- a/src/common/message_types.ts +++ b/src/common/message_types.ts @@ -23,7 +23,11 @@ import {MalloyError, MalloyQueryData, ResultJSON} from '@malloydata/malloy'; import {DataStyles} from '@malloydata/render'; -import {ConnectionBackend, ConnectionConfig} from './connection_manager_types'; +import { + ConnectionBackend, + ConnectionConfig, + ExternalConnectionConfig, +} from './connection_manager_types'; import {ProgressType} from 'vscode-jsonrpc'; /* @@ -228,6 +232,7 @@ export enum ConnectionMessageType { AppReady = 'app-ready', TestConnection = 'test-connection', RequestBigQueryServiceAccountKeyFile = 'request-bigquery-service-account-key-file', + InstallExternalConnection = 'install-external-connection', } interface ConnectionMessageSetConnections { @@ -292,11 +297,42 @@ export type ConnectionMessageServiceAccountKeyRequest = | ConnectionMessageServiceAccountKeyRequestWaiting | ConnectionMessageServiceAccountKeyRequestSuccess; +export enum InstallExternalConnectionStatus { + Waiting = 'waiting', + Success = 'success', + Error = 'error', +} + +interface ConnectionMessageInstallExternalConnectionWaiting { + type: ConnectionMessageType.InstallExternalConnection; + status: InstallExternalConnectionStatus.Waiting; + connection: ExternalConnectionConfig; +} + +interface ConnectionMessageInstallExternalConnectionSuccess { + type: ConnectionMessageType.InstallExternalConnection; + status: InstallExternalConnectionStatus.Success; + connection: ExternalConnectionConfig; +} + +interface ConnectionMessageInstallExternalConnectionError { + type: ConnectionMessageType.InstallExternalConnection; + status: InstallExternalConnectionStatus.Error; + error: string; + connection: ExternalConnectionConfig; +} + +export type ConnectionMessageInstallExternalConnection = + | ConnectionMessageInstallExternalConnectionWaiting + | ConnectionMessageInstallExternalConnectionSuccess + | ConnectionMessageInstallExternalConnectionError; + export type ConnectionPanelMessage = | ConnectionMessageAppReady | ConnectionMessageSetConnections | ConnectionMessageTest - | ConnectionMessageServiceAccountKeyRequest; + | ConnectionMessageServiceAccountKeyRequest + | ConnectionMessageInstallExternalConnection; export enum HelpMessageType { AppReady = 'app-ready', diff --git a/src/extension/browser/commands/edit_connections.ts b/src/extension/browser/commands/edit_connections.ts index 514f5d60..d5a63419 100644 --- a/src/extension/browser/commands/edit_connections.ts +++ b/src/extension/browser/commands/edit_connections.ts @@ -74,6 +74,7 @@ export function editConnectionsCommand(): void { messageManager.onReceiveMessage(async message => { switch (message.type) { case ConnectionMessageType.SetConnections: { + console.log(`==== CONNS ${message.connections}`); const connections = await handleConnectionsPreSave(message.connections); const malloyConfig = vscode.workspace.getConfiguration('malloy'); const hasWorkspaceConfig = diff --git a/src/extension/node/commands/edit_connections.ts b/src/extension/node/commands/edit_connections.ts index f96356b7..2c4b8504 100644 --- a/src/extension/node/commands/edit_connections.ts +++ b/src/extension/node/commands/edit_connections.ts @@ -30,6 +30,7 @@ import { ConnectionPanelMessage, ConnectionServiceAccountKeyRequestStatus, ConnectionTestStatus, + InstallExternalConnectionStatus, } from '../../../common/message_types'; import {WebviewMessageManager} from '../../webview_message_manager'; import {connectionManager} from '../connection_manager'; @@ -137,6 +138,27 @@ export function editConnectionsCommand(): void { } break; } + case ConnectionMessageType.InstallExternalConnection: { + try { + const installResult = + await connectionManager.installExternalConnectionPackage( + message.connection + ); + messageManager.postMessage({ + type: ConnectionMessageType.InstallExternalConnection, + status: InstallExternalConnectionStatus.Success, + connection: {...installResult}, + }); + } catch (error) { + messageManager.postMessage({ + type: ConnectionMessageType.InstallExternalConnection, + status: InstallExternalConnectionStatus.Error, + connection: message.connection, + error: errorMessage(error), + }); + } + break; + } } }); } diff --git a/src/extension/webviews/connections_page/App.tsx b/src/extension/webviews/connections_page/App.tsx index a198a0df..07f73359 100644 --- a/src/extension/webviews/connections_page/App.tsx +++ b/src/extension/webviews/connections_page/App.tsx @@ -25,6 +25,7 @@ import React, {useEffect, useState} from 'react'; import { ConnectionBackend, ConnectionConfig, + ExternalConnectionConfig, } from '../../../common/connection_manager_types'; import { ConnectionMessageType, @@ -32,6 +33,8 @@ import { ConnectionMessageTest, ConnectionTestStatus, ConnectionServiceAccountKeyRequestStatus, + InstallExternalConnectionStatus, + ConnectionMessageInstallExternalConnection, } from '../../../common/message_types'; import {useConnectionsVSCodeContext} from './connections_vscode_context'; import {ConnectionEditorList} from './ConnectionEditorList'; @@ -48,6 +51,10 @@ export const App: React.FC = () => { ConnectionConfig[] | undefined >(); const [testStatuses, setTestStatuses] = useState([]); + const [ + installExternalConnectionStatuses, + setInstallExternalConnectionStatuses, + ] = useState([]); const [availableBackends, setAvailableBackends] = useState< ConnectionBackend[] >([]); @@ -70,6 +77,19 @@ export const App: React.FC = () => { setTestStatuses([...testStatuses, message]); }; + const installExternalConnection = (connection: ExternalConnectionConfig) => { + const message: ConnectionMessageInstallExternalConnection = { + type: ConnectionMessageType.InstallExternalConnection, + connection, + status: InstallExternalConnectionStatus.Waiting, + }; + vscode.postMessage(message); + setInstallExternalConnectionStatuses([ + ...installExternalConnectionStatuses, + message, + ]); + }; + const requestServiceAccountKeyPath = (connectionId: string) => { vscode.postMessage({ type: ConnectionMessageType.RequestBigQueryServiceAccountKeyFile, @@ -90,6 +110,12 @@ export const App: React.FC = () => { case ConnectionMessageType.TestConnection: setTestStatuses([...testStatuses, message]); break; + case ConnectionMessageType.InstallExternalConnection: + setInstallExternalConnectionStatuses([ + ...installExternalConnectionStatuses, + message, + ]); + break; case ConnectionMessageType.RequestBigQueryServiceAccountKeyFile: { if ( message.status === ConnectionServiceAccountKeyRequestStatus.Success @@ -130,6 +156,8 @@ export const App: React.FC = () => { testStatuses={testStatuses} requestServiceAccountKeyPath={requestServiceAccountKeyPath} availableBackends={availableBackends} + installExternalConnection={installExternalConnection} + installExternalConnectionStatuses={installExternalConnectionStatuses} /> diff --git a/src/extension/webviews/connections_page/ConnectionEditor/ ConnectionEditor.tsx b/src/extension/webviews/connections_page/ConnectionEditor/ ConnectionEditor.tsx index d921733b..7d2eebf7 100644 --- a/src/extension/webviews/connections_page/ConnectionEditor/ ConnectionEditor.tsx +++ b/src/extension/webviews/connections_page/ConnectionEditor/ ConnectionEditor.tsx @@ -27,8 +27,12 @@ import { ConnectionBackend, ConnectionBackendNames, ConnectionConfig, + ExternalConnectionConfig, } from '../../../../common/connection_manager_types'; -import {ConnectionMessageTest} from '../../../../common/message_types'; +import { + ConnectionMessageInstallExternalConnection, + ConnectionMessageTest, +} from '../../../../common/message_types'; import {Dropdown} from '../../components'; import { VSCodeButton, @@ -41,6 +45,7 @@ import {Label} from './Label'; import {LabelCell} from './LabelCell'; import {PostgresConnectionEditor} from './PostgresConnectionEditor'; import {DuckDBConnectionEditor} from './DuckDBConnectionEditor'; +import {ExternalConnectionEditor} from './ExternalConnectionEditor'; interface ConnectionEditorProps { config: ConnectionConfig; @@ -52,6 +57,10 @@ interface ConnectionEditorProps { isDefault: boolean; makeDefault: () => void; availableBackends: ConnectionBackend[]; + installExternalConnection: (config: ExternalConnectionConfig) => void; + installExternalConnectionStatus: + | ConnectionMessageInstallExternalConnection + | undefined; } export const ConnectionEditor: React.FC = ({ @@ -64,11 +73,14 @@ export const ConnectionEditor: React.FC = ({ isDefault, makeDefault, availableBackends, + installExternalConnection, + installExternalConnectionStatus, }) => { const allBackendOptions: ConnectionBackend[] = [ ConnectionBackend.BigQuery, ConnectionBackend.Postgres, ConnectionBackend.DuckDB, + ConnectionBackend.External, ]; const backendOptions = allBackendOptions @@ -128,6 +140,13 @@ export const ConnectionEditor: React.FC = ({ ) : config.backend === ConnectionBackend.DuckDB ? ( + ) : config.backend === ConnectionBackend.External ? ( + ) : (
Unknown Connection Type
)} diff --git a/src/extension/webviews/connections_page/ConnectionEditor/ExternalConnectionEditor/ExternalConnectionEditor.tsx b/src/extension/webviews/connections_page/ConnectionEditor/ExternalConnectionEditor/ExternalConnectionEditor.tsx new file mode 100644 index 00000000..1f0541a0 --- /dev/null +++ b/src/extension/webviews/connections_page/ConnectionEditor/ExternalConnectionEditor/ExternalConnectionEditor.tsx @@ -0,0 +1,245 @@ +/* + * Copyright 2023 Google LLC + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +import React from 'react'; +import { + ExternalConnectionConfig, + ExternalConnectionSource, + ExternalConnectionSourceNames, +} from '../../../../../common/connection_manager_types'; +import {Dropdown, TextField, VSCodeButton} from '../../../components'; +import {Label} from '../Label'; +import {LabelCell} from '../LabelCell'; +import {ButtonGroup} from '../../ButtonGroup'; +import {ConnectionMessageInstallExternalConnection} from '../../../../../common/message_types'; +import {TextFieldType} from '@vscode/webview-ui-toolkit'; +import {ConnectionConfig, ConnectionParameterValue} from '@malloydata/malloy'; + +interface ExternalConnectionEditorProps { + config: ExternalConnectionConfig; + setConfig: (config: ExternalConnectionConfig) => void; + installExternalConnection: (config: ExternalConnectionConfig) => void; + installExternalConnectionStatus: + | ConnectionMessageInstallExternalConnection + | undefined; +} + +export const ExternalConnectionEditor: React.FC< + ExternalConnectionEditorProps +> = ({ + config, + setConfig, + installExternalConnection, + installExternalConnectionStatus, +}) => { + const allConnectionSources = [ + ExternalConnectionSource.NPM, + ExternalConnectionSource.LocalNPM, + ]; + + const sourceOptions = allConnectionSources.map(value => ({ + value, + label: ExternalConnectionSourceNames[value], + })); + + if (!config.source) { + config.source = ExternalConnectionSource.NPM; + } + + if ( + installExternalConnectionStatus?.status === 'success' && + config.packageInfo !== + installExternalConnectionStatus.connection.packageInfo + ) { + config.packageInfo = installExternalConnectionStatus.connection.packageInfo; + config.connectionSchema = + installExternalConnectionStatus.connection.connectionSchema; + config.name = installExternalConnectionStatus.connection.name; + } + + return ( + + + + + + + + + + + + + + + + + + {config.packageInfo && ( + + + + + + + )} + {config.connectionSchema?.map(parameter => { + // TODO(figutierrez): Move this to its own component. + return ( + + + + + + + ); + })} + +
+ + setConfig({ + backend: config.backend, + isDefault: config.isDefault, + id: config.id, + name: config.name, + source: source as ExternalConnectionSource, + }) + } + options={sourceOptions} + /> +
+ { + setConfig({ + backend: config.backend, + isDefault: config.isDefault, + id: config.id, + name: config.name, + source: config.source, + path: path.trim(), + }); + }} + /> +
+ + + installExternalConnection(config)} + appearance="secondary" + > + {installButtonLabel(installExternalConnectionStatus, config)} + + {installExternalConnectionStatus?.status === 'error' && + installExternalConnectionStatus.error} + +
+ { + setConfig({...config, name: value}); + }} + /> +
+ { + const configParameters = + config.configParameters ?? ({} as ConnectionConfig); + let configParameterValue: ConnectionParameterValue = value; + if (parameter.type === 'number') { + try { + configParameterValue = parseInt(value); + } catch { + // TODO(figutierrez): Show error when this happens. + configParameterValue = 0; + } + } else if (parameter.type === 'boolean') { + configParameterValue = value === 'false'; + } + configParameters[parameter.name] = configParameterValue; + setConfig({ + ...config, + configParameters: configParameters, + }); + }} + type={ + parameter.isSecret + ? TextFieldType.password + : TextFieldType.text + } + > +
+ ); +}; + +const installButtonLabel = ( + installConnection: ConnectionMessageInstallExternalConnection | undefined, + config: ExternalConnectionConfig +) => { + if (installConnection?.status === 'waiting') { + return 'Installing...'; + } + + if (config.packageInfo) { + return 'Installed'; + } + + return 'Install Plugin'; +}; + +const shouldDisableInstallButton = ( + installConnection: ConnectionMessageInstallExternalConnection | undefined, + config: ExternalConnectionConfig +) => { + if (installConnection?.status === 'waiting') { + return true; + } + + if ((config.path?.trim()?.length ?? 0) === 0) { + return true; + } + + if (config.packageInfo) { + return true; + } + + return false; +}; diff --git a/src/extension/webviews/connections_page/ConnectionEditor/ExternalConnectionEditor/index.ts b/src/extension/webviews/connections_page/ConnectionEditor/ExternalConnectionEditor/index.ts new file mode 100644 index 00000000..0cea925a --- /dev/null +++ b/src/extension/webviews/connections_page/ConnectionEditor/ExternalConnectionEditor/index.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2023 Google LLC + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files + * (the "Software"), to deal in the Software without restriction, + * including without limitation the rights to use, copy, modify, merge, + * publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +export {ExternalConnectionEditor} from './ExternalConnectionEditor'; diff --git a/src/extension/webviews/connections_page/ConnectionEditorList/ConnectionEditorList.tsx b/src/extension/webviews/connections_page/ConnectionEditorList/ConnectionEditorList.tsx index 5c1ef195..a7d741a5 100644 --- a/src/extension/webviews/connections_page/ConnectionEditorList/ConnectionEditorList.tsx +++ b/src/extension/webviews/connections_page/ConnectionEditorList/ConnectionEditorList.tsx @@ -27,9 +27,13 @@ import {v4 as uuidv4} from 'uuid'; import { ConnectionBackend, ConnectionConfig, + ExternalConnectionConfig, getDefaultIndex, } from '../../../../common/connection_manager_types'; -import {ConnectionMessageTest} from '../../../../common/message_types'; +import { + ConnectionMessageInstallExternalConnection, + ConnectionMessageTest, +} from '../../../../common/message_types'; import {VSCodeButton} from '../../components'; import {ButtonGroup} from '../ButtonGroup'; import {ConnectionEditor} from '../ConnectionEditor'; @@ -42,6 +46,8 @@ interface ConnectionEditorListProps { testStatuses: ConnectionMessageTest[]; requestServiceAccountKeyPath: (connectionId: string) => void; availableBackends: ConnectionBackend[]; + installExternalConnection: (config: ExternalConnectionConfig) => void; + installExternalConnectionStatuses: ConnectionMessageInstallExternalConnection[]; } export const ConnectionEditorList: React.FC = ({ @@ -52,6 +58,8 @@ export const ConnectionEditorList: React.FC = ({ testStatuses, requestServiceAccountKeyPath, availableBackends, + installExternalConnection, + installExternalConnectionStatuses, }) => { const [dirty, setDirty] = useState(false); const defaultConnectionIndex = getDefaultIndex(connections); @@ -110,6 +118,12 @@ export const ConnectionEditorList: React.FC = ({ isDefault={index === defaultConnectionIndex} makeDefault={() => makeDefault(index)} availableBackends={availableBackends} + installExternalConnection={installExternalConnection} + installExternalConnectionStatus={[ + ...installExternalConnectionStatuses, + ] + .reverse() + .find(message => message.connection.id === config.id)} /> ))} {connections.length === 0 && ( From 4050d1516aedc78b046abd88495fd37dc02caac9 Mon Sep 17 00:00:00 2001 From: Fernando Arreola Date: Thu, 17 Aug 2023 15:20:39 -0700 Subject: [PATCH 2/4] Remove some unnecesary stuff. --- src/common/connection_manager.ts | 6 +----- src/common/connections/external_connection_factory.ts | 5 ++--- src/common/connections/node/connection_factory.ts | 7 +------ src/common/errors.ts | 2 +- 4 files changed, 5 insertions(+), 15 deletions(-) diff --git a/src/common/connection_manager.ts b/src/common/connection_manager.ts index 900627ef..9bc67e7f 100644 --- a/src/common/connection_manager.ts +++ b/src/common/connection_manager.ts @@ -57,11 +57,7 @@ export class DynamicConnectionLookup implements LookupConnection { ...this.options, }); } else { - throw new Error( - `No connection found with name ${connectionName} ${ - new Error('abc').stack - }` - ); + throw new Error(`No connection found with name ${connectionName}`); } } return this.connections[connectionKey]; diff --git a/src/common/connections/external_connection_factory.ts b/src/common/connections/external_connection_factory.ts index ef5ca97d..d7a431c8 100644 --- a/src/common/connections/external_connection_factory.ts +++ b/src/common/connections/external_connection_factory.ts @@ -10,7 +10,6 @@ import { ExternalConnectionSource, } from '../connection_manager_types'; -// TODO(figutierrez): This could be a class instead of static consts, have a common plugin manager. // TODO(figutierrez): Can we check if package deps are in sync with these? export interface ExternalConnectionPackageInfo { @@ -39,7 +38,8 @@ export class ExternalConnectionFactory { ); } - const connection = externalConnection.connectionFactory.createConnection( + const factory = externalConnection.connectionFactory as ConnectionFactory; + const connection = factory.createConnection( config.configParameters ?? {}, registerDialect ) as Connection & TestableConnection; @@ -90,7 +90,6 @@ export class ExternalConnectionFactory { } switch (connectionConfig.source) { - // TODO(figutierrez): Test this path. case ExternalConnectionSource.NPM: { try { const pi = await this.pm.installFromNpm(connectionConfig.path); diff --git a/src/common/connections/node/connection_factory.ts b/src/common/connections/node/connection_factory.ts index d524787c..e987616e 100644 --- a/src/common/connections/node/connection_factory.ts +++ b/src/common/connections/node/connection_factory.ts @@ -107,9 +107,7 @@ export class DesktopConnectionFactory implements ConnectionFactory { } if (!connection) { throw new Error( - `Unsupported connection back end "${connectionConfig.backend}" ${ - new Error('abc').stack - }` + `Unsupported connection back end "${connectionConfig.backend}"` ); } @@ -144,9 +142,6 @@ export class DesktopConnectionFactory implements ConnectionFactory { } addDefaults(configs: ConnectionConfig[]): ConnectionConfig[] { - // Create a default bigquery connection if one isn't configured - // TODO(figutierrez): remove. - // Create a default bigquery connection if one isn't configured if ( !configs.find(config => config.backend === ConnectionBackend.BigQuery) diff --git a/src/common/errors.ts b/src/common/errors.ts index 267094a7..1c329c31 100644 --- a/src/common/errors.ts +++ b/src/common/errors.ts @@ -22,7 +22,7 @@ */ export const errorMessage = (error: unknown): string => { - let message = `Something went wrong ${JSON.stringify(error)}`; + let message = 'Something went wrong'; if (error instanceof Error) { message = error.message; } else if (typeof error === 'string') { From 0edc8e84bd10f697d6e2c3909fe1ef63fcda4404 Mon Sep 17 00:00:00 2001 From: Fernando Arreola Date: Thu, 17 Aug 2023 15:38:25 -0700 Subject: [PATCH 3/4] Fix circular dep. --- src/common/connection_manager_types.ts | 6 +++++- src/common/connections/external_connection_factory.ts | 6 +----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/common/connection_manager_types.ts b/src/common/connection_manager_types.ts index 4276c0c6..465f1c86 100644 --- a/src/common/connection_manager_types.ts +++ b/src/common/connection_manager_types.ts @@ -25,7 +25,6 @@ import { ConnectionConfigSchema, ConnectionConfig as MalloyConnectionConfig, } from '@malloydata/malloy'; -import {ExternalConnectionPackageInfo} from './connections/external_connection_factory'; export enum ConnectionBackend { BigQuery = 'bigquery', @@ -86,6 +85,11 @@ export interface DuckDBWASMConnectionConfigDeprecated workingDirectory?: string; } +export interface ExternalConnectionPackageInfo { + packageName: string; + version: string; +} + export enum ExternalConnectionSource { NPM = 'npm', LocalNPM = 'local_npm', diff --git a/src/common/connections/external_connection_factory.ts b/src/common/connections/external_connection_factory.ts index d7a431c8..bfc85cd5 100644 --- a/src/common/connections/external_connection_factory.ts +++ b/src/common/connections/external_connection_factory.ts @@ -7,16 +7,12 @@ import {registerDialect} from '@malloydata/malloy/dist/dialect'; import {PluginManager} from 'live-plugin-manager'; import { ExternalConnectionConfig, + ExternalConnectionPackageInfo, ExternalConnectionSource, } from '../connection_manager_types'; // TODO(figutierrez): Can we check if package deps are in sync with these? -export interface ExternalConnectionPackageInfo { - packageName: string; - version: string; -} - export class ExternalConnectionFactory { readonly pm = new PluginManager(); readonly sessionInstalledPackages: Array = []; From 55a4ac0805a673bf5e575993a9f93876d8fb86d1 Mon Sep 17 00:00:00 2001 From: Fernando Arreola Date: Thu, 17 Aug 2023 15:41:38 -0700 Subject: [PATCH 4/4] Fix package.json --- package-lock.json | 49 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index 0d40d865..913221cc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2375,6 +2375,14 @@ } }, "node_modules/@malloydata/db-bigquery": { + "version": "0.0.77", + "resolved": "https://registry.npmjs.org/@malloydata/db-bigquery/-/db-bigquery-0.0.77.tgz", + "integrity": "sha512-+/Yf/d/annZYVSSrqc+195nR2z4CNK/T2kDGOm8bs94644J8PdaDJiBHmDkDe4Jr+si22lw+/5x2Nnqd5CjZIQ==", + "dependencies": { + "@google-cloud/bigquery": "^5.5.0", + "@google-cloud/common": "^3.6.0", + "@google-cloud/paginator": "^4.0.1", + "@malloydata/malloy": "^0.0.77", "gaxios": "^4.2.0" }, "engines": { @@ -2382,6 +2390,12 @@ } }, "node_modules/@malloydata/db-duckdb": { + "version": "0.0.77", + "resolved": "https://registry.npmjs.org/@malloydata/db-duckdb/-/db-duckdb-0.0.77.tgz", + "integrity": "sha512-5ykLEqjM+848Ey3fJIWMfMsCocTdgGf/eH7dVA1NDZCr4c/jOsZjcD/OeL7YidULF1jtfTu7YUdCmrJFtwNsNA==", + "dependencies": { + "@malloydata/duckdb-wasm": "0.0.2", + "@malloydata/malloy": "^0.0.77", "apache-arrow": "^11.0.0", "duckdb": "0.8.1", "web-worker": "^1.2.0" @@ -2391,6 +2405,11 @@ } }, "node_modules/@malloydata/db-postgres": { + "version": "0.0.77", + "resolved": "https://registry.npmjs.org/@malloydata/db-postgres/-/db-postgres-0.0.77.tgz", + "integrity": "sha512-NWr9NHzjsuPYZ3MV3Fw/7YdqD2a2vJtTRmqGmqEbL1SRtfFquQdgjcf8YGC/GTGNzI2oOug0b3GY2bWpsBAm5A==", + "dependencies": { + "@malloydata/malloy": "^0.0.77", "@types/pg": "^8.6.1", "pg": "^8.7.1", "pg-query-stream": "4.2.3" @@ -2404,6 +2423,9 @@ } }, "node_modules/@malloydata/malloy": { + "version": "0.0.77", + "resolved": "https://registry.npmjs.org/@malloydata/malloy/-/malloy-0.0.77.tgz", + "integrity": "sha512-7v3cDrdVqonDoc5uQclMCmxzfcocNLiJFFjHixiXC3+RkN2MHtHKRYhlsZ1rkIkS6so7Gh855MlAfXo2pQztzw==", "dependencies": { "antlr4ts": "^0.5.0-alpha.4", "assert": "^2.0.0", @@ -2417,6 +2439,19 @@ } }, "node_modules/@malloydata/malloy-sql": { + "version": "0.0.77", + "resolved": "https://registry.npmjs.org/@malloydata/malloy-sql/-/malloy-sql-0.0.77.tgz", + "integrity": "sha512-od3FWDERunvCwDq6urs41//VnLywojm+znx8LmsHnCWS3yG/EtAHCS8vIz5OZuhvCQmIlRtQsOLOU08GzpzPXQ==", + "dependencies": { + "@malloydata/malloy": "^0.0.77" + } + }, + "node_modules/@malloydata/render": { + "version": "0.0.77", + "resolved": "https://registry.npmjs.org/@malloydata/render/-/render-0.0.77.tgz", + "integrity": "sha512-QAJrl8RERaFTUbW+NSCYjGelyYBe1Rb9f/V5G1V2We5lYLev9AUqIcIN3crPZHsunWKJT9P2xMkPh1lMZun+gA==", + "dependencies": { + "@malloydata/malloy": "^0.0.77", "@types/luxon": "^2.4.0", "lodash": "^4.17.20", "luxon": "^2.4.0", @@ -7186,20 +7221,6 @@ "version": "1.0.0", "license": "ISC" }, - "node_modules/fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "dev": true, - "hasInstallScript": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, "node_modules/fstream": { "version": "1.0.12", "dev": true,