From 24e37e1637d0064d33299c16f780cd7ff0129d13 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Fri, 8 Dec 2023 09:35:12 +0000 Subject: [PATCH 1/7] Azure Blob Storage integration --- .../DatasourceNavigator/icons/Azure.svelte | 64 +++++++++ .../DatasourceNavigator/icons/index.js | 2 + packages/server/package.json | 1 + .../src/integrations/azureBlobStorage.ts | 129 ++++++++++++++++++ packages/server/src/integrations/index.ts | 3 + packages/types/src/sdk/datasources.ts | 1 + yarn.lock | 14 ++ 7 files changed, 214 insertions(+) create mode 100644 packages/builder/src/components/backend/DatasourceNavigator/icons/Azure.svelte create mode 100644 packages/server/src/integrations/azureBlobStorage.ts diff --git a/packages/builder/src/components/backend/DatasourceNavigator/icons/Azure.svelte b/packages/builder/src/components/backend/DatasourceNavigator/icons/Azure.svelte new file mode 100644 index 00000000000..27931e011d9 --- /dev/null +++ b/packages/builder/src/components/backend/DatasourceNavigator/icons/Azure.svelte @@ -0,0 +1,64 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js b/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js index 2486942dea7..a03cd5a677c 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js +++ b/packages/builder/src/components/backend/DatasourceNavigator/icons/index.js @@ -15,6 +15,7 @@ import GoogleSheets from "./GoogleSheets.svelte" import Firebase from "./Firebase.svelte" import Redis from "./Redis.svelte" import Snowflake from "./Snowflake.svelte" +import Azure from "./Azure.svelte" import Custom from "./Custom.svelte" import { integrations } from "stores/backend" import { get } from "svelte/store" @@ -28,6 +29,7 @@ const ICONS = { COUCHDB: CouchDB, SQL_SERVER: SqlServer, S3: S3, + AZURE: Azure, AIRTABLE: Airtable, MYSQL: MySQL, ARANGODB: ArangoDB, diff --git a/packages/server/package.json b/packages/server/package.json index c845f7889d0..e9c7cde953d 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -43,6 +43,7 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", + "@azure/storage-blob": "^12.17.0", "@budibase/backend-core": "0.0.0", "@budibase/client": "0.0.0", "@budibase/pro": "0.0.0", diff --git a/packages/server/src/integrations/azureBlobStorage.ts b/packages/server/src/integrations/azureBlobStorage.ts new file mode 100644 index 00000000000..2601ad382c1 --- /dev/null +++ b/packages/server/src/integrations/azureBlobStorage.ts @@ -0,0 +1,129 @@ +import { + Integration, + QueryType, + IntegrationBase, + DatasourceFieldType, + DatasourceFeature, + ConnectionInfo, +} from "@budibase/types" + +import { BlobServiceClient } from "@azure/storage-blob" + +interface BlobConfig { + connectionString: string +} + +const SCHEMA: Integration = { + docs: "https://learn.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction", + description: + "Azure Blob Storage is Microsoft's object storage solution for the cloud.", + friendlyName: "Azure Blob Storage", + type: "Object store", + features: { + [DatasourceFeature.CONNECTION_CHECKING]: true, + }, + datasource: { + connectionString: { + type: DatasourceFieldType.STRING, + required: true, + display: "Connection string", + }, + }, + query: { + create: { + type: QueryType.FIELDS, + fields: { + containerName: { + display: "Container name", + type: DatasourceFieldType.STRING, + required: true, + }, + }, + }, + read: { + type: QueryType.FIELDS, + fields: { + containerName: { + display: "Container name", + type: DatasourceFieldType.STRING, + required: true, + }, + }, + }, + delete: { + type: QueryType.FIELDS, + fields: { + containerName: { + display: "Container name", + type: DatasourceFieldType.STRING, + required: true, + }, + blobName: { + display: "Blob name", + type: DatasourceFieldType.STRING, + required: true, + }, + }, + }, + }, +} + +class AzureBlobIntegration implements IntegrationBase { + private readonly blobServiceClient: BlobServiceClient + + constructor(config: BlobConfig) { + this.blobServiceClient = BlobServiceClient.fromConnectionString( + config.connectionString + ) + } + + async testConnection() { + const response: ConnectionInfo = { + connected: false, + } + try { + await this.blobServiceClient.getProperties() + response.connected = true + } catch (e: any) { + response.error = e.message as string + } + return response + } + + async create(query: { containerName: string }) { + const containerClient = this.blobServiceClient.getContainerClient( + query.containerName + ) + return await containerClient.create() + } + + async read(query: { containerName: string }) { + const containerClient = this.blobServiceClient.getContainerClient( + query.containerName + ) + let blobs = [] + for await (const blob of containerClient.listBlobsFlat()) { + // Get Blob Client from name, to get the URL + const blobClient = containerClient.getBlockBlobClient(blob.name) + blobs.push({ + name: blob.name, + ...blob.properties, + url: blobClient.url, + }) + } + return blobs + } + + async delete(query: { containerName: string; blobName: string }) { + const containerClient = this.blobServiceClient.getContainerClient( + query.containerName + ) + const blockBlobClient = containerClient.getBlockBlobClient(query.blobName) + return await blockBlobClient.delete() + } +} + +export default { + schema: SCHEMA, + integration: AzureBlobIntegration, +} diff --git a/packages/server/src/integrations/index.ts b/packages/server/src/integrations/index.ts index 49761bac85d..db8c33f8145 100644 --- a/packages/server/src/integrations/index.ts +++ b/packages/server/src/integrations/index.ts @@ -5,6 +5,7 @@ import elasticsearch from "./elasticsearch" import couchdb from "./couchdb" import sqlServer from "./microsoftSqlServer" import s3 from "./s3" +import azureBlobStorage from "./azureBlobStorage" import airtable from "./airtable" import mysql from "./mysql" import arangodb from "./arangodb" @@ -28,6 +29,7 @@ const DEFINITIONS: Record = { [SourceName.COUCHDB]: couchdb.schema, [SourceName.SQL_SERVER]: sqlServer.schema, [SourceName.S3]: s3.schema, + [SourceName.AZURE]: azureBlobStorage.schema, [SourceName.AIRTABLE]: airtable.schema, [SourceName.MYSQL]: mysql.schema, [SourceName.ARANGODB]: arangodb.schema, @@ -47,6 +49,7 @@ const INTEGRATIONS: Record = { [SourceName.COUCHDB]: couchdb.integration, [SourceName.SQL_SERVER]: sqlServer.integration, [SourceName.S3]: s3.integration, + [SourceName.AZURE]: azureBlobStorage.integration, [SourceName.AIRTABLE]: airtable.integration, [SourceName.MYSQL]: mysql.integration, [SourceName.ARANGODB]: arangodb.integration, diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 7a335eb3b9b..a417a45508f 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -47,6 +47,7 @@ export enum SourceName { COUCHDB = "COUCHDB", SQL_SERVER = "SQL_SERVER", S3 = "S3", + AZURE = "AZURE", AIRTABLE = "AIRTABLE", MYSQL = "MYSQL", ARANGODB = "ARANGODB", diff --git a/yarn.lock b/yarn.lock index a09ae20de6d..0ee1ff0a986 100644 --- a/yarn.lock +++ b/yarn.lock @@ -822,6 +822,20 @@ events "^3.0.0" tslib "^2.2.0" +"@azure/storage-blob@^12.17.0": + version "12.17.0" + resolved "https://registry.yarnpkg.com/@azure/storage-blob/-/storage-blob-12.17.0.tgz#04aad7f59cb08dbbe5b1b672a9f5b6256c8c9006" + integrity sha512-sM4vpsCpcCApagRW5UIjQNlNylo02my2opgp0Emi8x888hZUvJ3dN69Oq20cEGXkMUWnoCrBaB0zyS3yeB87sQ== + dependencies: + "@azure/abort-controller" "^1.0.0" + "@azure/core-http" "^3.0.0" + "@azure/core-lro" "^2.2.0" + "@azure/core-paging" "^1.1.1" + "@azure/core-tracing" "1.0.0-preview.13" + "@azure/logger" "^1.0.0" + events "^3.0.0" + tslib "^2.2.0" + "@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.16.7", "@babel/code-frame@^7.22.5": version "7.22.5" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.5.tgz#234d98e1551960604f1246e6475891a570ad5658" From cee9152a77d2f63c0c0e003777c1772820599bf5 Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Fri, 8 Dec 2023 11:11:26 +0000 Subject: [PATCH 2/7] Azure Storage File upload --- .../design/settings/componentSettings.js | 4 +-- ...Select.svelte => ObjectStoreSelect.svelte} | 2 +- .../builder/src/constants/backend/index.js | 2 ++ packages/client/manifest.json | 6 ++-- packages/frontend-core/src/api/index.js | 2 ++ .../src/api/controllers/static/index.ts | 36 ++++++++++++++++++- .../src/integrations/azureBlobStorage.ts | 18 ++++++---- 7 files changed, 57 insertions(+), 13 deletions(-) rename packages/builder/src/components/design/settings/controls/{S3DataSourceSelect.svelte => ObjectStoreSelect.svelte} (78%) diff --git a/packages/builder/src/components/design/settings/componentSettings.js b/packages/builder/src/components/design/settings/componentSettings.js index b7402472943..ce314637aeb 100644 --- a/packages/builder/src/components/design/settings/componentSettings.js +++ b/packages/builder/src/components/design/settings/componentSettings.js @@ -1,6 +1,6 @@ import { Checkbox, Select, RadioGroup, Stepper, Input } from "@budibase/bbui" import DataSourceSelect from "./controls/DataSourceSelect/DataSourceSelect.svelte" -import S3DataSourceSelect from "./controls/S3DataSourceSelect.svelte" +import ObjectStoreSelect from "./controls/ObjectStoreSelect.svelte" import DataProviderSelect from "./controls/DataProviderSelect.svelte" import ButtonActionEditor from "./controls/ButtonActionEditor/ButtonActionEditor.svelte" import TableSelect from "./controls/TableSelect.svelte" @@ -32,7 +32,7 @@ const componentMap = { select: Select, radio: RadioGroup, dataSource: DataSourceSelect, - "dataSource/s3": S3DataSourceSelect, + "dataSource/s3": ObjectStoreSelect, dataProvider: DataProviderSelect, boolean: Checkbox, number: Stepper, diff --git a/packages/builder/src/components/design/settings/controls/S3DataSourceSelect.svelte b/packages/builder/src/components/design/settings/controls/ObjectStoreSelect.svelte similarity index 78% rename from packages/builder/src/components/design/settings/controls/S3DataSourceSelect.svelte rename to packages/builder/src/components/design/settings/controls/ObjectStoreSelect.svelte index 05b4058aa3a..de85b779e14 100644 --- a/packages/builder/src/components/design/settings/controls/S3DataSourceSelect.svelte +++ b/packages/builder/src/components/design/settings/controls/ObjectStoreSelect.svelte @@ -5,7 +5,7 @@ export let value = null $: dataSources = $datasources.list - .filter(ds => ds.source === "S3" && !ds.config?.endpoint) + .filter(ds => ["S3", "AZURE"].includes(ds.source) && !ds.config?.endpoint) .map(ds => ({ label: ds.name, value: ds._id, diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js index ac4079b69ed..1bfa568777e 100644 --- a/packages/builder/src/constants/backend/index.js +++ b/packages/builder/src/constants/backend/index.js @@ -229,6 +229,7 @@ export const IntegrationTypes = { MONGODB: "MONGODB", COUCHDB: "COUCHDB", S3: "S3", + AZURE: "AZURE", MYSQL: "MYSQL", REST: "REST", DYNAMODB: "DYNAMODB", @@ -249,6 +250,7 @@ export const IntegrationNames = { [IntegrationTypes.MONGODB]: "MongoDB", [IntegrationTypes.COUCHDB]: "CouchDB", [IntegrationTypes.S3]: "S3", + [IntegrationTypes.AZURE]: "AZURE", [IntegrationTypes.MYSQL]: "MySQL", [IntegrationTypes.REST]: "REST", [IntegrationTypes.DYNAMODB]: "DynamoDB", diff --git a/packages/client/manifest.json b/packages/client/manifest.json index fdb0ad9db11..0bd3ab6e9b3 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -4491,7 +4491,7 @@ ] }, "s3upload": { - "name": "S3 File Upload", + "name": "File Upload", "icon": "UploadToCloud", "styles": ["size"], "editable": true, @@ -4513,13 +4513,13 @@ }, { "type": "dataSource/s3", - "label": "S3 datasource", + "label": "Object store", "key": "datasourceId", "info": "This component can't be used with S3 datasources that use custom endpoints" }, { "type": "text", - "label": "Bucket", + "label": "Bucket / Container", "key": "bucket" }, { diff --git a/packages/frontend-core/src/api/index.js b/packages/frontend-core/src/api/index.js index aefc3522a75..10b67ee06d7 100644 --- a/packages/frontend-core/src/api/index.js +++ b/packages/frontend-core/src/api/index.js @@ -136,6 +136,8 @@ export const createAPIClient = config => { headers["x-budibase-session-id"] = APISessionID if (!external) { headers["x-budibase-api-version"] = ApiVersion + } else { + headers["x-ms-blob-type"] = "BlockBlob" } if (json) { headers["Content-Type"] = "application/json" diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index 2963546e7fa..469c526a442 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -21,6 +21,12 @@ import { BadRequestError, } from "@budibase/backend-core" import AWS from "aws-sdk" +import { + StorageSharedKeyCredential, + generateBlobSASQueryParameters, + BlobSASPermissions, + BlobServiceClient, +} from "@azure/storage-blob" import fs from "fs" import sdk from "../../../sdk" import * as pro from "@budibase/pro" @@ -273,7 +279,6 @@ export const getSignedUploadURL = async function (ctx: Ctx) { const { bucket, key } = ctx.request.body || {} if (!bucket || !key) { ctx.throw(400, "bucket and key values are required") - return } try { const s3 = new AWS.S3({ @@ -289,6 +294,35 @@ export const getSignedUploadURL = async function (ctx: Ctx) { } catch (error: any) { ctx.throw(400, error) } + } else if (datasource?.source === "AZURE") { + const { bucket, key } = ctx.request.body || {} + if (!bucket || !key) { + ctx.throw(400, "container and key values are required") + } + try { + const containerClient = BlobServiceClient.fromConnectionString( + `DefaultEndpointsProtocol=https;AccountName=${datasource?.config?.accountName};AccountKey=${datasource?.config?.accountKey};EndpointSuffix=core.windows.net` + ).getContainerClient(bucket) + + const blobSAS = generateBlobSASQueryParameters( + { + containerName: bucket, + blobName: key, + permissions: BlobSASPermissions.parse("racwd"), + startsOn: new Date(), + expiresOn: new Date(new Date().valueOf() + 86400), + }, + new StorageSharedKeyCredential( + datasource?.config?.accountName, + datasource?.config?.accountKey + ) + ).toString() + + publicUrl = containerClient.getBlobClient(key).url + signedUrl = `${publicUrl}?${blobSAS}` + } catch (error: any) { + ctx.throw(400, error) + } } ctx.body = { signedUrl, publicUrl } diff --git a/packages/server/src/integrations/azureBlobStorage.ts b/packages/server/src/integrations/azureBlobStorage.ts index 2601ad382c1..1ee9b6cae03 100644 --- a/packages/server/src/integrations/azureBlobStorage.ts +++ b/packages/server/src/integrations/azureBlobStorage.ts @@ -9,8 +9,9 @@ import { import { BlobServiceClient } from "@azure/storage-blob" -interface BlobConfig { - connectionString: string +interface AzureBlobStorageConfig { + accountName: string + accountKey: string } const SCHEMA: Integration = { @@ -23,10 +24,15 @@ const SCHEMA: Integration = { [DatasourceFeature.CONNECTION_CHECKING]: true, }, datasource: { - connectionString: { + accountName: { type: DatasourceFieldType.STRING, required: true, - display: "Connection string", + display: "Account name", + }, + accountKey: { + type: DatasourceFieldType.PASSWORD, + required: true, + display: "Account key", }, }, query: { @@ -71,9 +77,9 @@ const SCHEMA: Integration = { class AzureBlobIntegration implements IntegrationBase { private readonly blobServiceClient: BlobServiceClient - constructor(config: BlobConfig) { + constructor(config: AzureBlobStorageConfig) { this.blobServiceClient = BlobServiceClient.fromConnectionString( - config.connectionString + `DefaultEndpointsProtocol=https;AccountName=${config.accountName};AccountKey=${config.accountKey};EndpointSuffix=core.windows.net` ) } From 568a5e3c41a3eed737eed9ec1048165a3806ae5d Mon Sep 17 00:00:00 2001 From: Mel O'Hagan Date: Fri, 8 Dec 2023 11:33:31 +0000 Subject: [PATCH 3/7] Refactor manifest --- .../src/components/design/settings/componentSettings.js | 4 ++-- .../actions/{S3Upload.svelte => FileUpload.svelte} | 4 ++-- .../settings/controls/ButtonActionEditor/actions/index.js | 2 +- .../settings/controls/ButtonActionEditor/manifest.json | 4 ++-- .../[componentId]/new/_components/componentStructure.json | 2 +- packages/client/manifest.json | 6 +++--- .../app/forms/{S3Upload.svelte => FileUpload.svelte} | 2 +- packages/client/src/components/app/forms/index.js | 2 +- packages/client/src/utils/buttonActions.js | 4 ++-- 9 files changed, 15 insertions(+), 15 deletions(-) rename packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/{S3Upload.svelte => FileUpload.svelte} (88%) rename packages/client/src/components/app/forms/{S3Upload.svelte => FileUpload.svelte} (99%) diff --git a/packages/builder/src/components/design/settings/componentSettings.js b/packages/builder/src/components/design/settings/componentSettings.js index ce314637aeb..7b39faee5ff 100644 --- a/packages/builder/src/components/design/settings/componentSettings.js +++ b/packages/builder/src/components/design/settings/componentSettings.js @@ -32,7 +32,7 @@ const componentMap = { select: Select, radio: RadioGroup, dataSource: DataSourceSelect, - "dataSource/s3": ObjectStoreSelect, + "dataSource/objectStore": ObjectStoreSelect, dataProvider: DataProviderSelect, boolean: Checkbox, number: Stepper, @@ -63,7 +63,7 @@ const componentMap = { "field/longform": FormFieldSelect, "field/datetime": FormFieldSelect, "field/attachment": FormFieldSelect, - "field/s3": Input, + "field/upload": Input, "field/link": FormFieldSelect, "field/array": FormFieldSelect, "field/json": FormFieldSelect, diff --git a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/S3Upload.svelte b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/FileUpload.svelte similarity index 88% rename from packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/S3Upload.svelte rename to packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/FileUpload.svelte index 2e374f165ff..2e4e225811e 100644 --- a/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/S3Upload.svelte +++ b/packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/FileUpload.svelte @@ -6,12 +6,12 @@ export let parameters $: components = findAllMatchingComponents($currentAsset?.props, component => - component._component.endsWith("s3upload") + component._component.endsWith("fileupload") )
- +