From 482b5b7544f11bcdd122823d185ec35bd13e7900 Mon Sep 17 00:00:00 2001 From: Karthikeyan Chinnakonda Date: Thu, 12 Sep 2024 17:26:33 +0530 Subject: [PATCH] Change the file structure (#45) * Change the file structure * add fs-extra * install fs-extra --- package-lock.json | 98 +++- package.json | 4 +- src/cli/config.ts | 469 +++++++++--------- src/cli/index.ts | 2 +- src/connector.ts | 105 ----- src/connector/connector.ts | 121 +++++ src/{ => connector/db}/cosmosDb.ts | 0 src/{ => connector}/execution.ts | 4 +- src/connector/schema.ts | 494 +++++++++++++++++++ src/connector/sql/runSql.ts | 50 ++ src/connector/sql/sqlGeneration.ts | 730 +++++++++++++++++++++++++++++ src/index.ts | 4 +- src/schema.ts | 471 ------------------- src/sqlGeneration.ts | 655 -------------------------- 14 files changed, 1754 insertions(+), 1453 deletions(-) delete mode 100644 src/connector.ts create mode 100644 src/connector/connector.ts rename src/{ => connector/db}/cosmosDb.ts (100%) rename src/{ => connector}/execution.ts (99%) create mode 100644 src/connector/schema.ts create mode 100644 src/connector/sql/runSql.ts create mode 100644 src/connector/sql/sqlGeneration.ts delete mode 100644 src/schema.ts delete mode 100644 src/sqlGeneration.ts diff --git a/package-lock.json b/package-lock.json index ed5a264..4004758 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,9 @@ "@azure/cosmos": "^4.0.0", "@azure/identity": "^4.0.1", "@hasura/ndc-sdk-typescript": "^6.1.0", + "@types/fs-extra": "^11.0.4", "dotenv": "^16.4.5", + "fs-extra": "^11.2.0", "quicktype-core": "^23.0.104", "ts-node": "^10.9.2", "typescript": "^5.5.4" @@ -2029,6 +2031,15 @@ "@types/chai": "*" } }, + "node_modules/@types/fs-extra": { + "version": "11.0.4", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-11.0.4.tgz", + "integrity": "sha512-yTbItCNreRooED33qjunPthRcSjERP1r4MqCZc7wv0u2sUkzTFp45tgUfS5+r7FrZPdmCCNflLhVSP/o+SemsQ==", + "dependencies": { + "@types/jsonfile": "*", + "@types/node": "*" + } + }, "node_modules/@types/json-pointer": { "version": "1.0.34", "resolved": "https://registry.npmjs.org/@types/json-pointer/-/json-pointer-1.0.34.tgz", @@ -2041,6 +2052,14 @@ "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", "dev": true }, + "node_modules/@types/jsonfile": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/@types/jsonfile/-/jsonfile-6.1.4.tgz", + "integrity": "sha512-D5qGUYwjvnNNextdU59/+fI+spnwtTFmyQP0h+PfIOSkNfpU6AOICUOkm4i0OnSk+NyjdPJrxCDro0sJsWlRpQ==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/jsonpath": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/@types/jsonpath/-/jsonpath-0.2.4.tgz", @@ -3054,6 +3073,19 @@ } ] }, + "node_modules/fs-extra": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", + "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -3160,8 +3192,7 @@ "node_modules/graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" }, "node_modules/has-flag": { "version": "4.0.0", @@ -3748,6 +3779,17 @@ "node": ">=6" } }, + "node_modules/jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, "node_modules/jsonwebtoken": { "version": "9.0.2", "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz", @@ -5741,6 +5783,14 @@ "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz", "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==" }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "engines": { + "node": ">= 10.0.0" + } + }, "node_modules/update-browserslist-db": { "version": "1.0.13", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz", @@ -7504,6 +7554,15 @@ "@types/chai": "*" } }, + "@types/fs-extra": { + "version": "11.0.4", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-11.0.4.tgz", + "integrity": "sha512-yTbItCNreRooED33qjunPthRcSjERP1r4MqCZc7wv0u2sUkzTFp45tgUfS5+r7FrZPdmCCNflLhVSP/o+SemsQ==", + "requires": { + "@types/jsonfile": "*", + "@types/node": "*" + } + }, "@types/json-pointer": { "version": "1.0.34", "resolved": "https://registry.npmjs.org/@types/json-pointer/-/json-pointer-1.0.34.tgz", @@ -7516,6 +7575,14 @@ "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", "dev": true }, + "@types/jsonfile": { + "version": "6.1.4", + "resolved": "https://registry.npmjs.org/@types/jsonfile/-/jsonfile-6.1.4.tgz", + "integrity": "sha512-D5qGUYwjvnNNextdU59/+fI+spnwtTFmyQP0h+PfIOSkNfpU6AOICUOkm4i0OnSk+NyjdPJrxCDro0sJsWlRpQ==", + "requires": { + "@types/node": "*" + } + }, "@types/jsonpath": { "version": "0.2.4", "resolved": "https://registry.npmjs.org/@types/jsonpath/-/jsonpath-0.2.4.tgz", @@ -8243,6 +8310,16 @@ "integrity": "sha512-cHEpEQHUg0f8XdtZCc2ZAhrHzKzT0MrFUTcvx+hfxYu7rGMDc5SKoXFh+n4YigxsHXRzc6OrCshdR1bWH6HHyg==", "dev": true }, + "fs-extra": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.2.0.tgz", + "integrity": "sha512-PmDi3uwK5nFuXh7XDTlVnS17xJS7vW36is2+w3xcv8SVxiB4NyATf4ctkVY5bkSjX0Y4nbvZCq1/EjtEyr9ktw==", + "requires": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -8315,8 +8392,7 @@ "graceful-fs": { "version": "4.2.11", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" }, "has-flag": { "version": "4.0.0", @@ -8740,6 +8816,15 @@ "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", "dev": true }, + "jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, "jsonwebtoken": { "version": "9.0.2", "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz", @@ -10284,6 +10369,11 @@ "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz", "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==" }, + "universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==" + }, "update-browserslist-db": { "version": "1.0.13", "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.13.tgz", diff --git a/package.json b/package.json index 1ab9ce5..1443ab1 100644 --- a/package.json +++ b/package.json @@ -31,10 +31,12 @@ "rimraf": "^5.0.7" }, "dependencies": { - "@hasura/ndc-sdk-typescript": "^6.1.0", "@azure/cosmos": "^4.0.0", "@azure/identity": "^4.0.1", + "@hasura/ndc-sdk-typescript": "^6.1.0", + "@types/fs-extra": "^11.0.4", "dotenv": "^16.4.5", + "fs-extra": "^11.2.0", "quicktype-core": "^23.0.104", "ts-node": "^10.9.2", "typescript": "^5.5.4" diff --git a/src/cli/config.ts b/src/cli/config.ts index 776ec5a..ca55120 100644 --- a/src/cli/config.ts +++ b/src/cli/config.ts @@ -1,42 +1,54 @@ import { Database } from "@azure/cosmos"; -import { CollectionDefinition, CollectionDefinitions, CollectionsSchema, NamedObjectTypeDefinition, ScalarTypeDefinitions, getJSONScalarTypes } from "../schema" -import { BuiltInScalarTypeName, ObjectTypeDefinitions, TypeDefinition, ObjectTypePropertiesMap } from "../schema"; - -import { Container } from "@azure/cosmos" - -import { InputData, jsonInputForTargetLanguage, quicktype } from "quicktype-core"; -import { runSQLQuery, constructCosmosDbClient } from "../cosmosDb" +import { + CollectionDefinition, + CollectionDefinitions, + CollectionsSchema, + NamedObjectTypeDefinition, + ScalarTypeDefinitions, + getJSONScalarTypes, + BuiltInScalarTypeName, + ObjectTypeDefinitions, + TypeDefinition, + ObjectTypePropertiesMap, +} from "../connector/schema"; + +import { Container } from "@azure/cosmos"; + +import { + InputData, + jsonInputForTargetLanguage, + quicktype, +} from "quicktype-core"; +import { runSQLQuery, constructCosmosDbClient } from "../connector/db/cosmosDb"; import { exit } from "process"; import fs from "fs"; import { promisify } from "util"; import { $RefParser } from "@apidevtools/json-schema-ref-parser"; - export type RawConfiguration = { - azure_cosmos_key: string, - azure_cosmos_db_endpoint: string, - azure_cosmos_db_name: string, - azure_cosmos_no_of_rows_to_fetch: number | null -} + azure_cosmos_key: string; + azure_cosmos_db_endpoint: string; + azure_cosmos_db_name: string; + azure_cosmos_no_of_rows_to_fetch: number | null; +}; type JSONSchemaProperty = { - type: string, - $ref: string, - items?: JSONSchemaProperty -} + type: string; + $ref: string; + items?: JSONSchemaProperty; +}; type JSONSchemaDefinition = { - type: string, - additionalProperties: boolean, - properties?: Record, - title: string, -} + type: string; + additionalProperties: boolean; + properties?: Record; + title: string; +}; export type JSONSchema = { - definitions: Record, - $ref: string, -} - + definitions: Record; + $ref: string; +}; /** * Fetches at-most `n` latest updated rows from the given container @@ -46,144 +58,162 @@ export type JSONSchema = { * @returns The latest at-most `n` rows from the `container`. **/ -export async function fetchLatestNRowsFromContainer(n: number, container: Container): Promise { - const querySpec = { - query: `SELECT * FROM ${container.id} c ORDER BY c._ts DESC OFFSET 0 LIMIT ${n}`, - parameters: [] - } - - return await runSQLQuery(querySpec, container) +export async function fetchLatestNRowsFromContainer( + n: number, + container: Container, +): Promise { + const querySpec = { + query: `SELECT * FROM ${container.id} c ORDER BY c._ts DESC OFFSET 0 LIMIT ${n}`, + parameters: [], + }; + + return await runSQLQuery(querySpec, container); } +export async function inferJSONSchemaFromContainerRows( + rows: string[], + containerTypeName: string, +): Promise { + const jsonInput = jsonInputForTargetLanguage("schema"); -export async function inferJSONSchemaFromContainerRows(rows: string[], containerTypeName: string): Promise { - const jsonInput = jsonInputForTargetLanguage("schema"); - - await jsonInput.addSource({ - name: containerTypeName, - samples: rows.map(x => JSON.stringify(x)) - }); - - const inputData = new InputData(); - inputData.addInput(jsonInput); + await jsonInput.addSource({ + name: containerTypeName, + samples: rows.map((x) => JSON.stringify(x)), + }); - let jsonSchema = await quicktype({ - inputData, - lang: "schema" - }); + const inputData = new InputData(); + inputData.addInput(jsonInput); - let rawJSONSchemaOutput: any = jsonSchema.lines.join("\n"); + let jsonSchema = await quicktype({ + inputData, + lang: "schema", + }); - return JSON.parse(rawJSONSchemaOutput) + let rawJSONSchemaOutput: any = jsonSchema.lines.join("\n"); + return JSON.parse(rawJSONSchemaOutput); } -function getPropertyTypeDefn(property: JSONSchemaProperty, $refs: $RefParser): TypeDefinition | null { - if (property.$ref !== undefined && property.$ref !== null) { - - const referencedPropertyDefn = $refs.$refs.get(property.$ref) as JSONSchemaDefinition; - - if (referencedPropertyDefn.type === "object") { - return { - type: "named", - name: referencedPropertyDefn.title, - kind: "object" - } - } else if (referencedPropertyDefn.type === "string") { - return { - type: "named", - name: BuiltInScalarTypeName.String, - kind: "scalar" - } - } else { - console.log("Warning: Could not infer the type for referenced property", property); - - } - - - } else if (property.type == "null") { - // We don't have enough information to predict anything about the property. So, just - // return null. - return null - } else if (property.type == "array") { - if (property.items !== undefined) { - const elementType = getPropertyTypeDefn(property.items, $refs); - - if (elementType !== null) { - return { - "type": "array", - "elementType": elementType - } - } - - } - return null - } else if (property.type == "string") { - return { - "type": "named", - name: BuiltInScalarTypeName.String, - kind: "scalar" - } - } else if (property.type == "number") { - return { - "type": "named", - name: BuiltInScalarTypeName.Number, - kind: "scalar" - } +function getPropertyTypeDefn( + property: JSONSchemaProperty, + $refs: $RefParser, +): TypeDefinition | null { + if (property.$ref !== undefined && property.$ref !== null) { + const referencedPropertyDefn = $refs.$refs.get( + property.$ref, + ) as JSONSchemaDefinition; + + if (referencedPropertyDefn.type === "object") { + return { + type: "named", + name: referencedPropertyDefn.title, + kind: "object", + }; + } else if (referencedPropertyDefn.type === "string") { + return { + type: "named", + name: BuiltInScalarTypeName.String, + kind: "scalar", + }; + } else { + console.log( + "Warning: Could not infer the type for referenced property", + property, + ); } - else if (property.type == "integer") { - return { - "type": "named", - name: BuiltInScalarTypeName.Number, - kind: "scalar" - } - } else if (property.type == "boolean") { + } else if (property.type == "null") { + // We don't have enough information to predict anything about the property. So, just + // return null. + return null; + } else if (property.type == "array") { + if (property.items !== undefined) { + const elementType = getPropertyTypeDefn(property.items, $refs); + + if (elementType !== null) { return { - "type": "named", - name: BuiltInScalarTypeName.Boolean, - kind: "scalar" - } + type: "array", + elementType: elementType, + }; + } } + return null; + } else if (property.type == "string") { + return { + type: "named", + name: BuiltInScalarTypeName.String, + kind: "scalar", + }; + } else if (property.type == "number") { + return { + type: "named", + name: BuiltInScalarTypeName.Number, + kind: "scalar", + }; + } else if (property.type == "integer") { + return { + type: "named", + name: BuiltInScalarTypeName.Number, + kind: "scalar", + }; + } else if (property.type == "boolean") { + return { + type: "named", + name: BuiltInScalarTypeName.Boolean, + kind: "scalar", + }; + } - return null + return null; } -export async function getObjectTypeDefinitionsFromJSONSchema(containerJSONSchema: JSONSchema): Promise { - var objectTypeDefinitions: ObjectTypeDefinitions = {}; - let parser = new $RefParser(); - - const $refs = await parser.resolve(JSON.parse(JSON.stringify(containerJSONSchema))); - Object.entries(containerJSONSchema.definitions).forEach(([objectTypeName, objectTypeDefinition]) => { - if (objectTypeDefinition.type == "object") { - var objectTypeProperties: ObjectTypePropertiesMap = {}; - - if (objectTypeDefinition.properties !== undefined) { - - Object.entries(objectTypeDefinition.properties).map(([propertyName, propertyDefn]) => { - - let propertyTypeDefn = getPropertyTypeDefn(propertyDefn, parser); - - let legacyProperty = ['_rid', '_self', '_etag', '_attachments', '_ts'] - if (propertyTypeDefn !== null && !legacyProperty.includes(propertyName)) { - objectTypeProperties[propertyName] = { - propertyName: propertyName, - description: null, - type: propertyTypeDefn - }; - } - - }) - - objectTypeDefinitions[objectTypeName] = { - description: null, - properties: objectTypeProperties - } - } +export async function getObjectTypeDefinitionsFromJSONSchema( + containerJSONSchema: JSONSchema, +): Promise { + var objectTypeDefinitions: ObjectTypeDefinitions = {}; + let parser = new $RefParser(); + + const $refs = await parser.resolve( + JSON.parse(JSON.stringify(containerJSONSchema)), + ); + Object.entries(containerJSONSchema.definitions).forEach( + ([objectTypeName, objectTypeDefinition]) => { + if (objectTypeDefinition.type == "object") { + var objectTypeProperties: ObjectTypePropertiesMap = {}; + + if (objectTypeDefinition.properties !== undefined) { + Object.entries(objectTypeDefinition.properties).map( + ([propertyName, propertyDefn]) => { + let propertyTypeDefn = getPropertyTypeDefn(propertyDefn, parser); + + let legacyProperty = [ + "_rid", + "_self", + "_etag", + "_attachments", + "_ts", + ]; + if ( + propertyTypeDefn !== null && + !legacyProperty.includes(propertyName) + ) { + objectTypeProperties[propertyName] = { + propertyName: propertyName, + description: null, + type: propertyTypeDefn, + }; + } + }, + ); + objectTypeDefinitions[objectTypeName] = { + description: null, + properties: objectTypeProperties, + }; } - } - ) - return objectTypeDefinitions + } + }, + ); + return objectTypeDefinitions; } /** @@ -195,79 +225,94 @@ export async function getObjectTypeDefinitionsFromJSONSchema(containerJSONSchema * @param {number} nRows - Number of rows to be read per container to infer the schema of the container. * @returns {Promise { - - let collectionDefinitions: CollectionDefinitions = {}; - - let objectTypeDefinitions: ObjectTypeDefinitions = {}; - - const scalarTypeDefinitions: ScalarTypeDefinitions = getJSONScalarTypes(); - - const { resources: allContainers } = await database.containers.readAll().fetchAll(); - - - for (const container of allContainers) { - const dbContainer = database.container(container.id); - - const nContainerRows = await fetchLatestNRowsFromContainer(nRows, dbContainer); - nContainerRows.reverse(); - const containerJsonSchema = await inferJSONSchemaFromContainerRows(nContainerRows, container.id); - - const containerObjectTypeDefinitions = await getObjectTypeDefinitionsFromJSONSchema(containerJsonSchema); - - const collectionObjectType: NamedObjectTypeDefinition = { - type: "named", - name: containerJsonSchema.$ref.split('/').pop() as string, - kind: "object" - }; - - const collectionDefinition: CollectionDefinition = { - description: null, - arguments: [], - resultType: collectionObjectType - }; +async function getCollectionsSchema( + database: Database, + nRows: number, +): Promise { + let collectionDefinitions: CollectionDefinitions = {}; + + let objectTypeDefinitions: ObjectTypeDefinitions = {}; + + const scalarTypeDefinitions: ScalarTypeDefinitions = getJSONScalarTypes(); + + const { resources: allContainers } = await database.containers + .readAll() + .fetchAll(); + + for (const container of allContainers) { + const dbContainer = database.container(container.id); + + const nContainerRows = await fetchLatestNRowsFromContainer( + nRows, + dbContainer, + ); + nContainerRows.reverse(); + const containerJsonSchema = await inferJSONSchemaFromContainerRows( + nContainerRows, + container.id, + ); + + const containerObjectTypeDefinitions = + await getObjectTypeDefinitionsFromJSONSchema(containerJsonSchema); + + const collectionObjectType: NamedObjectTypeDefinition = { + type: "named", + name: containerJsonSchema.$ref.split("/").pop() as string, + kind: "object", + }; - objectTypeDefinitions = { ...objectTypeDefinitions, ...containerObjectTypeDefinitions }; - collectionDefinitions[container.id] = collectionDefinition; - } + const collectionDefinition: CollectionDefinition = { + description: null, + arguments: [], + resultType: collectionObjectType, + }; - let schema = { - collections: collectionDefinitions, - objectTypes: objectTypeDefinitions, - scalarTypes: scalarTypeDefinitions, + objectTypeDefinitions = { + ...objectTypeDefinitions, + ...containerObjectTypeDefinitions, }; + collectionDefinitions[container.id] = collectionDefinition; + } - return schema + let schema = { + collections: collectionDefinitions, + objectTypes: objectTypeDefinitions, + scalarTypes: scalarTypeDefinitions, + }; + return schema; } export async function generateConnectorConfig(outputConfigDir: string) { - const rowsToFetch = process.env["AZURE_COSMOS_NO_OF_ROWS_TO_FETCH"] ?? "100"; - - try { - const client = constructCosmosDbClient(); - const schema = await getCollectionsSchema(client.dbClient, parseInt(rowsToFetch)); - const cosmosKey = client.connectionDetails.key; - const cosmosEndpoint = client.connectionDetails.endpoint; - const cosmosDbName = client.connectionDetails.databaseName; - - const response: any = { - connection: { - endpoint: cosmosEndpoint, - key: cosmosKey, - databaseName: cosmosDbName - }, - schema - }; - - const writeFile = promisify(fs.writeFile); - - await writeFile(`${outputConfigDir}/config.json`, JSON.stringify(response, null, 2)); - + const rowsToFetch = process.env["AZURE_COSMOS_NO_OF_ROWS_TO_FETCH"] ?? "100"; + + try { + const client = constructCosmosDbClient(); + const schema = await getCollectionsSchema( + client.dbClient, + parseInt(rowsToFetch), + ); + const cosmosKey = client.connectionDetails.key; + const cosmosEndpoint = client.connectionDetails.endpoint; + const cosmosDbName = client.connectionDetails.databaseName; + + const response: any = { + connection: { + endpoint: cosmosEndpoint, + key: cosmosKey, + databaseName: cosmosDbName, + }, + schema, + }; - } catch (error) { - console.log("Error while generating the config", error); - exit(1) - } + const writeFile = promisify(fs.writeFile); + await writeFile( + `${outputConfigDir}/config.json`, + JSON.stringify(response, null, 2), + ); + } catch (error) { + console.log("Error while generating the config", error); + exit(1); + } } diff --git a/src/cli/index.ts b/src/cli/index.ts index 915c8e0..2b5e3eb 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -2,7 +2,7 @@ import { Command, Option } from "commander"; import * as updateCmd from "./update"; -import { createConnector } from "../connector"; +import { createConnector } from "../connector/connector"; import { version } from "../../package.json" import * as sdk from "@hasura/ndc-sdk-typescript"; diff --git a/src/connector.ts b/src/connector.ts deleted file mode 100644 index 31744b2..0000000 --- a/src/connector.ts +++ /dev/null @@ -1,105 +0,0 @@ -import * as sdk from "@hasura/ndc-sdk-typescript"; -import { CollectionsSchema, getNdcSchemaResponse } from "./schema" -import { constructCosmosDbClient } from "./cosmosDb"; -import { Database } from "@azure/cosmos"; -import { executeQuery } from "./execution"; -import { readFileSync } from "fs"; - - -export type Configuration = ConnectorConfig; - -export type ConnectorConfig = { - connection: { - endpoint: string, - key: string, - databaseName: string - } - schema: CollectionsSchema -} - -export type State = { - databaseClient: Database -}; - -export function createConnector(): sdk.Connector { - - const connector: sdk.Connector = { - parseConfiguration: async function(configurationDir: string): Promise { - - try { - const configLocation = `${configurationDir}/config.json`; - const fileContent = readFileSync(configLocation, 'utf8'); - const configObject: ConnectorConfig = JSON.parse(fileContent); - return Promise.resolve(configObject) - - } catch (error) { - console.error("Failed to parse configuration:", error); - throw new sdk.InternalServerError( - "Internal Server Error, server configuration is invalid", - {} - ); - } - - - }, - - tryInitState: async function(_: Configuration, __: unknown): Promise { - try { - const databaseClient = constructCosmosDbClient().dbClient; - return Promise.resolve({ - databaseClient - }) - } catch (error) { - console.error("Failed to initialize the state of the connector", error); - throw new sdk.InternalServerError( - `Internal server error, failed to initialize the state of the connector - ${error}`, {} - ) - } - - }, - - getSchema: async function(configuration: Configuration): Promise { - if (!configuration.schema) { - throw new sdk.Forbidden("Internal server error, server configuration not found") - } - return Promise.resolve(getNdcSchemaResponse(configuration.schema)) - - }, - - getCapabilities(_: Configuration): sdk.Capabilities { - return { - query: { - nested_fields: {}, - }, - mutation: {} - - } - }, - - query: async function(configuration: Configuration, state: State, request: sdk.QueryRequest): Promise { - return executeQuery(request, configuration.schema, state.databaseClient) - }, - - mutation: async function(configuration: Configuration, state: State, request: sdk.MutationRequest): Promise { - throw new Error("Not implemented") - }, - - queryExplain: function(configuration: Configuration, state: State, request: sdk.QueryRequest): Promise { - throw new Error("Function not implemented."); - }, - - mutationExplain: function(configuration: Configuration, state: State, request: sdk.MutationRequest): Promise { - throw new Error("Function not implemented."); - }, - - fetchMetrics: async function(configuration: Configuration, state: State): Promise { - return undefined; - }, - - - } - - - return connector; - -} diff --git a/src/connector/connector.ts b/src/connector/connector.ts new file mode 100644 index 0000000..35884ed --- /dev/null +++ b/src/connector/connector.ts @@ -0,0 +1,121 @@ +import * as sdk from "@hasura/ndc-sdk-typescript"; +import { CollectionsSchema, getNdcSchemaResponse } from "./schema"; +import { constructCosmosDbClient } from "./db/cosmosDb"; +import { Database } from "@azure/cosmos"; +import { executeQuery } from "./execution"; +import { readFileSync } from "fs"; + +export type Configuration = ConnectorConfig; + +export type ConnectorConfig = { + connection: { + endpoint: string; + key: string; + databaseName: string; + }; + schema: CollectionsSchema; +}; + +export type State = { + databaseClient: Database; +}; + +export function createConnector(): sdk.Connector { + const connector: sdk.Connector = { + parseConfiguration: async function ( + configurationDir: string, + ): Promise { + try { + const configLocation = `${configurationDir}/config.json`; + const fileContent = readFileSync(configLocation, "utf8"); + const configObject: ConnectorConfig = JSON.parse(fileContent); + return Promise.resolve(configObject); + } catch (error) { + console.error("Failed to parse configuration:", error); + throw new sdk.InternalServerError( + "Internal Server Error, server configuration is invalid", + {}, + ); + } + }, + + tryInitState: async function ( + _: Configuration, + __: unknown, + ): Promise { + try { + const databaseClient = constructCosmosDbClient().dbClient; + return Promise.resolve({ + databaseClient, + }); + } catch (error) { + console.error("Failed to initialize the state of the connector", error); + throw new sdk.InternalServerError( + `Internal server error, failed to initialize the state of the connector - ${error}`, + {}, + ); + } + }, + + getSchema: async function ( + configuration: Configuration, + ): Promise { + if (!configuration.schema) { + throw new sdk.Forbidden( + "Internal server error, server configuration not found", + ); + } + return Promise.resolve(getNdcSchemaResponse(configuration.schema)); + }, + + getCapabilities(_: Configuration): sdk.Capabilities { + return { + query: { + nested_fields: {}, + }, + mutation: {}, + }; + }, + + query: async function ( + configuration: Configuration, + state: State, + request: sdk.QueryRequest, + ): Promise { + return executeQuery(request, configuration.schema, state.databaseClient); + }, + + mutation: async function ( + configuration: Configuration, + state: State, + request: sdk.MutationRequest, + ): Promise { + throw new Error("Not implemented"); + }, + + queryExplain: function ( + configuration: Configuration, + state: State, + request: sdk.QueryRequest, + ): Promise { + throw new Error("Function not implemented."); + }, + + mutationExplain: function ( + configuration: Configuration, + state: State, + request: sdk.MutationRequest, + ): Promise { + throw new Error("Function not implemented."); + }, + + fetchMetrics: async function ( + configuration: Configuration, + state: State, + ): Promise { + return undefined; + }, + }; + + return connector; +} diff --git a/src/cosmosDb.ts b/src/connector/db/cosmosDb.ts similarity index 100% rename from src/cosmosDb.ts rename to src/connector/db/cosmosDb.ts diff --git a/src/execution.ts b/src/connector/execution.ts similarity index 99% rename from src/execution.ts rename to src/connector/execution.ts index fcbc91f..13ebcf8 100644 --- a/src/execution.ts +++ b/src/connector/execution.ts @@ -1,8 +1,8 @@ import * as sdk from "@hasura/ndc-sdk-typescript"; import * as schema from "./schema"; -import * as sql from "./sqlGeneration"; +import * as sql from "./sql/sqlGeneration"; import { Database } from "@azure/cosmos"; -import { runSQLQuery } from "./cosmosDb"; +import { runSQLQuery } from "./db/cosmosDb"; function validateOrderBy(orderBy: sdk.OrderBy, collectionObjectType: schema.ObjectTypeDefinition) { diff --git a/src/connector/schema.ts b/src/connector/schema.ts new file mode 100644 index 0000000..a6e3ab8 --- /dev/null +++ b/src/connector/schema.ts @@ -0,0 +1,494 @@ +import * as sdk from "@hasura/ndc-sdk-typescript"; +import { mapObjectValues } from "../utils"; +import { ScalarType } from "@hasura/ndc-sdk-typescript"; + +export type CollectionsSchema = { + collections: CollectionDefinitions; + objectTypes: ObjectTypeDefinitions; + scalarTypes: ScalarTypeDefinitions; +}; + +export type CollectionDefinitions = { + [collectionName: string]: CollectionDefinition; +}; + +export type CollectionDefinition = { + description: string | null; + arguments: ArgumentDefinition[]; + resultType: TypeDefinition; +}; + +export type ArgumentDefinition = { + argumentName: string; + description: string | null; + type: TypeDefinition; +}; + +export type ObjectTypeDefinitions = { + [objectTypeName: string]: ObjectTypeDefinition; +}; + +export type ObjectTypePropertiesMap = { + [propertyName: string]: ObjectPropertyDefinition; +}; + +export type ObjectTypeDefinition = { + description: string | null; + properties: ObjectTypePropertiesMap; +}; + +export type ObjectPropertyDefinition = { + propertyName: string; + description: string | null; + type: TypeDefinition; +}; + +export type ScalarTypeDefinitions = { + [scalarTypeName: string]: ScalarTypeDefinition; +}; + +export type ScalarTypeDefinition = BuiltInScalarTypeDefinition; // Empty object, for now + +export type TypeDefinition = + | ArrayTypeDefinition + | NullableTypeDefinition + | NamedTypeDefinition; + +export type ArrayTypeDefinition = { + type: "array"; + elementType: TypeDefinition; +}; + +export type NullableTypeDefinition = { + type: "nullable"; + underlyingType: TypeDefinition; +}; + +export type NamedTypeDefinition = + | NamedObjectTypeDefinition + | NamedScalarTypeDefinition; + +export type NamedObjectTypeDefinition = { + type: "named"; + name: string; + kind: "object"; +}; + +export type NamedScalarTypeDefinition = + | CustomNamedScalarTypeDefinition + | BuiltInScalarTypeDefinition; + +export type BuiltInScalarTypeDefinition = + | StringScalarTypeDefinition + | BooleanScalarTypeDefinition + | IntegerScalarTypeDefinition + | NumberScalarTypeDefinition + | DateTimeScalarTypeDefinition; + +export type CustomNamedScalarTypeDefinition = { + type: "named"; + name: string; + kind: "scalar"; +}; + +export type StringScalarTypeDefinition = { + type: "named"; + name: BuiltInScalarTypeName.String; + kind: "scalar"; + literalValue?: string; +}; + +export type NumberScalarTypeDefinition = { + type: "named"; + name: BuiltInScalarTypeName.Number; + kind: "scalar"; + literalValue?: number; +}; + +export type BooleanScalarTypeDefinition = { + type: "named"; + name: BuiltInScalarTypeName.Boolean; + kind: "scalar"; + literalValue?: boolean; +}; + +export type DateTimeScalarTypeDefinition = { + type: "named"; + name: BuiltInScalarTypeName.DateTime; + kind: "scalar"; +}; + +export type IntegerScalarTypeDefinition = { + type: "named"; + name: BuiltInScalarTypeName.Integer; + kind: "scalar"; +}; + +export enum BuiltInScalarTypeName { + String = "String", + Number = "Number", + Boolean = "Boolean", + DateTime = "DateTime", + Integer = "Integer", +} + +export type ScalarTypes = { + [k: string]: ScalarType; +}; + +export type ScalarOperatorMappings = { + [k: string]: string; +}; + +export const scalarTypes: ScalarTypes = { + Integer: { + aggregate_functions: { + count: { + result_type: { + type: "named", + name: BuiltInScalarTypeName.Integer, + }, + }, + sum: { + result_type: { + type: "named", + name: BuiltInScalarTypeName.Integer, + }, + }, + avg: { + result_type: { + type: "named", + name: BuiltInScalarTypeName.Integer, + }, + }, + min: { + result_type: { + type: "named", + name: BuiltInScalarTypeName.Integer, + }, + }, + max: { + result_type: { + type: "named", + name: BuiltInScalarTypeName.Integer, + }, + }, + }, + comparison_operators: { + eq: { + type: "equal", + }, + neq: { + type: "custom", + argument_type: { + type: "named", + name: "Integer", + }, + }, + gt: { + type: "custom", + argument_type: { + type: "named", + name: "Integer", + }, + }, + lt: { + type: "custom", + argument_type: { + type: "named", + name: "Integer", + }, + }, + gte: { + type: "custom", + argument_type: { + type: "named", + name: "Integer", + }, + }, + lte: { + type: "custom", + argument_type: { + type: "named", + name: "Integer", + }, + }, + }, + }, + Number: { + aggregate_functions: { + count: { + result_type: { + type: "named", + name: BuiltInScalarTypeName.Number, + }, + }, + sum: { + result_type: { + type: "named", + name: BuiltInScalarTypeName.Number, + }, + }, + avg: { + result_type: { + type: "named", + name: BuiltInScalarTypeName.Number, + }, + }, + min: { + result_type: { + type: "named", + name: BuiltInScalarTypeName.Number, + }, + }, + max: { + result_type: { + type: "named", + name: BuiltInScalarTypeName.Number, + }, + }, + }, + comparison_operators: { + eq: { + type: "equal", + }, + neq: { + type: "custom", + argument_type: { + type: "named", + name: "Number", + }, + }, + gt: { + type: "custom", + argument_type: { + type: "named", + name: "Number", + }, + }, + lt: { + type: "custom", + argument_type: { + type: "named", + name: "Number", + }, + }, + gte: { + type: "custom", + argument_type: { + type: "named", + name: "Number", + }, + }, + lte: { + type: "custom", + argument_type: { + type: "named", + name: "Number", + }, + }, + }, + }, + Boolean: { + aggregate_functions: { + bool_and: { + result_type: { + type: "named", + name: BuiltInScalarTypeName.Boolean, + }, + }, + bool_or: { + result_type: { + type: "named", + name: BuiltInScalarTypeName.Boolean, + }, + }, + bool_not: { + result_type: { + type: "named", + name: BuiltInScalarTypeName.Boolean, + }, + }, + }, + comparison_operators: { + eq: { + type: "equal", + }, + neq: { + type: "custom", + argument_type: { + type: "named", + name: "Boolean", + }, + }, + }, + }, + String: { + aggregate_functions: {}, + comparison_operators: { + eq: { + type: "equal", + }, + neq: { + type: "custom", + argument_type: { + type: "named", + name: "String", + }, + }, + gt: { + type: "custom", + argument_type: { + type: "named", + name: "String", + }, + }, + lt: { + type: "custom", + argument_type: { + type: "named", + name: "String", + }, + }, + gte: { + type: "custom", + argument_type: { + type: "named", + name: "String", + }, + }, + lte: { + type: "custom", + argument_type: { + type: "named", + name: "String", + }, + }, + contains: { + type: "custom", + argument_type: { + type: "named", + name: "String", + }, + }, + endswith: { + type: "custom", + argument_type: { + type: "named", + name: "String", + }, + }, + regexmatch: { + type: "custom", + argument_type: { + type: "named", + name: "String", + }, + }, + startswith: { + type: "custom", + argument_type: { + type: "named", + name: "String", + }, + }, + }, + }, +}; + +export function getJSONScalarTypes(): ScalarTypeDefinitions { + var scalarTypeDefinitions: ScalarTypeDefinitions = {}; + scalarTypeDefinitions["Integer"] = { + type: "named", + name: BuiltInScalarTypeName.Integer, + kind: "scalar", + }; + scalarTypeDefinitions["Number"] = { + type: "named", + name: BuiltInScalarTypeName.Number, + kind: "scalar", + }; + scalarTypeDefinitions["Boolean"] = { + type: "named", + name: BuiltInScalarTypeName.Boolean, + kind: "scalar", + }; + scalarTypeDefinitions["String"] = { + type: "named", + name: BuiltInScalarTypeName.String, + kind: "scalar", + }; + + return scalarTypeDefinitions; +} + +export function getNdcSchemaResponse( + collectionsSchema: CollectionsSchema, +): sdk.SchemaResponse { + const collections = Object.entries(collectionsSchema.collections); + + var collectionInfos = collections.map(([collectionName, collectionInfo]) => { + return { + name: collectionName, + description: null, + arguments: {}, + type: getBaseNamedType(collectionInfo.resultType), + uniqueness_constraints: {}, + foreign_keys: {}, + }; + }); + + const objectTypes = mapObjectValues( + collectionsSchema.objectTypes, + (objDef) => { + return { + fields: Object.fromEntries( + Object.values(objDef.properties).map((propDef) => { + const objField: sdk.ObjectField = { + type: convertTypeReferenceToSdkType(propDef.type), + description: null, + }; + return [propDef.propertyName, objField]; + }), + ), + ...(objDef.description ? { description: objDef.description } : {}), + }; + }, + ); + + return { + functions: [], + procedures: [], + collections: collectionInfos, + object_types: objectTypes, + scalar_types: scalarTypes, + }; +} + +function convertTypeReferenceToSdkType(typeRef: TypeDefinition): sdk.Type { + switch (typeRef.type) { + case "array": + return { + type: "array", + element_type: convertTypeReferenceToSdkType(typeRef.elementType), + }; + case "nullable": + return { + type: "nullable", + underlying_type: convertTypeReferenceToSdkType(typeRef.underlyingType), + }; + case "named": + return { type: "named", name: typeRef.name }; + } +} + +export function getBaseNamedType(typeRef: TypeDefinition): string { + switch (typeRef.type) { + case "array": + return getBaseNamedType(typeRef.elementType); + case "nullable": + return getBaseNamedType(typeRef.underlyingType); + case "named": + return typeRef.name; + } +} diff --git a/src/connector/sql/runSql.ts b/src/connector/sql/runSql.ts new file mode 100644 index 0000000..1fdcfc1 --- /dev/null +++ b/src/connector/sql/runSql.ts @@ -0,0 +1,50 @@ +// Function that accepts a SQL query and returns the result + +import { Container } from "@azure/cosmos"; +import { constructCosmosDbClient } from "../db/cosmosDb"; +import { Command } from "commander"; +import * as fs from "fs-extra"; + +// Function that accepts an arbitrary SQL query and returns the result +export async function runSQLQuery(sql: string, container: Container) { + const sqlQuerySpec = { + query: sql, + }; + + const result = await container.items + .query(sqlQuerySpec) + .fetchAll() + .catch((err) => { + console.error(err); + throw err; + }); + + console.log(JSON.stringify(result.resources, null, 2)); +} + +async function executeSqlFromFile(filePath: string, containerName: string) { + try { + const sqlQuery = await fs.readFile(filePath, "utf8"); + + let db = constructCosmosDbClient().dbClient; + const container = db.container(containerName); + + await runSQLQuery(sqlQuery, container); + } catch (err) { + console.error("Error running SQL from file:", err); + } +} + +const program = new Command(); + +program + .requiredOption( + "--container ", + "The name of the container to run the SQL in", + ) + .requiredOption("--file ", "The location of the SQL file to run") + .parse(process.argv); + +const options = program.opts(); + +executeSqlFromFile(options.file, options.container); diff --git a/src/connector/sql/sqlGeneration.ts b/src/connector/sql/sqlGeneration.ts new file mode 100644 index 0000000..1c170d3 --- /dev/null +++ b/src/connector/sql/sqlGeneration.ts @@ -0,0 +1,730 @@ +import * as sdk from "@hasura/ndc-sdk-typescript"; +import * as cosmos from "@azure/cosmos"; +import { SqlQuerySpec } from "@azure/cosmos"; +import * as schema from "../schema"; + +export type Column = { + name: string; + prefix: string; +}; + +export type SelectContainerColumn = { + kind: "column"; + column: Column; +}; + +export type SelectAggregate = { + kind: "aggregate"; + column: Column; + aggregateFunction: string; +}; + +export type SelectColumn = + | SelectContainerColumn + | SelectAggregate + | SqlQueryContext; + +/* + The key represents the alias of the request field and the + value represents the value to be selected from the container. +*/ +export type SelectColumns = { + [alias: string]: SelectColumn; +}; + +export type QueryVariable = { + [k: string]: unknown; +}; + +export type QueryVariables = QueryVariable[] | null | undefined; + +/* + Type to track the parameters used in the SQL query. + */ +type SqlParameters = { + [column: string]: any[]; +}; + +export type FromClause = { + source: string; + sourceAlias: string; + in?: string; +}; + +export type ContainerExpression = { + kind: "containerExpression"; + containerExpression: string; +}; + +export type SqlExpression = { + kind: "sqlExpression"; + sqlExpression: SqlQueryContext; +}; + +export type ArrayJoinTarget = ContainerExpression | SqlExpression; + +export type ArrayJoinClause = { + type: "array"; + joinIdentifier: string; + arrayJoinTarget: ArrayJoinTarget; +}; + +export type SubqueryJoinClause = { + type: "subquery"; + from: string; + subQuery: SqlQueryContext; + subQueryAs: string; +}; + +export type JoinClause = ArrayJoinClause | SubqueryJoinClause; + +type ComparisonScalarDbOperator = { + name: string; + isInfix: boolean; +}; + +type AggregateScalarDbOperator = { + operator: string; + resultType: string; +}; + +// Defines how the NDC's scalar operators map to the DB operators +type ScalarDBOperatorMappings = { + comparison: { + [operatorName: string]: ComparisonScalarDbOperator; + }; + aggregate?: + | { + [operatorName: string]: AggregateScalarDbOperator; + } + | undefined; +}; + +type ScalarOperatorMappings = { + [scalarTypeName: string]: ScalarDBOperatorMappings; +}; + +export const scalarComparisonOperatorMappings: ScalarOperatorMappings = { + Integer: { + comparison: { + eq: { + name: "=", + isInfix: true, + }, + neq: { + name: "!=", + isInfix: true, + }, + gt: { + name: ">", + isInfix: true, + }, + lt: { + name: "<", + isInfix: true, + }, + gte: { + name: ">=", + isInfix: true, + }, + lte: { + name: "<=", + isInfix: true, + }, + }, + aggregate: { + count: { + operator: "count", + resultType: "Integer", + }, + sum: { + operator: "sum", + resultType: "Integer", + }, + avg: { + operator: "sum", + resultType: "Number", + }, + min: { + operator: "sum", + resultType: "Integer", + }, + max: { + operator: "sum", + resultType: "Integer", + }, + }, + }, + Number: { + comparison: { + eq: { + name: "=", + isInfix: true, + }, + neq: { + name: "!=", + isInfix: true, + }, + gt: { + name: ">", + isInfix: true, + }, + lt: { + name: "<", + isInfix: true, + }, + gte: { + name: ">=", + isInfix: true, + }, + lte: { + name: "<=", + isInfix: true, + }, + }, + aggregate: { + count: { + operator: "count", + resultType: "Integer", + }, + sum: { + operator: "sum", + resultType: "Number", + }, + avg: { + operator: "sum", + resultType: "Number", + }, + min: { + operator: "sum", + resultType: "Number", + }, + max: { + operator: "sum", + resultType: "Number", + }, + }, + }, + Boolean: { + comparison: { + eq: { + name: "=", + isInfix: true, + }, + neq: { + name: "!=", + isInfix: true, + }, + }, + aggregate: { + bool_and: { + operator: "bool_and", + resultType: "Boolean", + }, + bool_or: { + operator: "bool_or", + resultType: "Boolean", + }, + bool_not: { + operator: "bool_or", + resultType: "Boolean", + }, + }, + }, + String: { + comparison: { + eq: { + name: "=", + isInfix: true, + }, + neq: { + name: "!=", + isInfix: true, + }, + gt: { + name: ">", + isInfix: true, + }, + lt: { + name: "<", + isInfix: true, + }, + gte: { + name: ">=", + isInfix: true, + }, + lte: { + name: "<=", + isInfix: true, + }, + contains: { + name: "CONTAINS", + isInfix: false, + }, + endswith: { + name: "ENDSWITH", + isInfix: false, + }, + regexmatch: { + name: "REGEXMATCH", + isInfix: false, + }, + startswith: { + name: "STARTSWITH", + isInfix: false, + }, + }, + }, +}; + +export function getDbComparisonOperator( + scalarTypeName: string, + operator: string, +): ComparisonScalarDbOperator { + const scalarOperators = scalarComparisonOperatorMappings[scalarTypeName]; + + if (scalarOperators === undefined && scalarOperators === null) { + throw new sdk.BadRequest( + `Couldn't find scalar type: ${scalarTypeName} in the schema`, + ); + } else { + const scalarDbOperator = scalarOperators.comparison[operator]; + + if (scalarDbOperator) { + return scalarDbOperator; + } else { + throw new sdk.BadRequest( + `Comparison Operator ${operator} is not supported on type ${scalarTypeName}`, + ); + } + } +} + +export type ComparisonTarget = + | { + type: "column"; + /** + * The name of the column + */ + name: string; + } + | { + type: "root_collection_column"; + /** + * The name of the column + */ + name: string; + }; + +export type ComparisonValue = + | { + type: "column"; + column: string; + } + | { + type: "scalar"; + value: unknown; + } + | { + type: "variable"; + name: string; + }; + +export type Expression = + | { + type: "and"; + expressions: Expression[]; + } + | { + type: "or"; + expressions: Expression[]; + } + | { + type: "not"; + expression: Expression; + } + | { + type: "unary_comparison_operator"; + column: string; + operator: "is_null"; + } + | { + type: "binary_comparison_operator"; + column: string; + value: ComparisonValue; + dbOperator: ComparisonScalarDbOperator; + }; + +export type SqlQueryContext = { + kind: "sqlQueryContext"; + select: SelectColumns; + /* Set to `true` to prevent the wrapping of the results into another JSON object. */ + selectAsValue: boolean; + from?: FromClause | null; + join?: JoinClause[] | null; + predicate?: Expression | null; + offset?: number | null; + limit?: number | null; + orderBy?: sdk.OrderBy | null; + isAggregateQuery: boolean; + selectAsArray?: boolean | undefined; +}; + +type VariablesMappings = { + /* + The variableTarget will be the name of the column + which gets the value of the variable + */ + [variableTarget: string]: string; +}; + +function formatJoinClause(joinClause: JoinClause): string { + if (joinClause.type === "array") { + let joinTarget = + joinClause.arrayJoinTarget.kind === "containerExpression" + ? joinClause.arrayJoinTarget.containerExpression + : constructSqlQuery( + joinClause.arrayJoinTarget.sqlExpression, + joinClause.joinIdentifier, + null, + ); + + return `${joinClause.joinIdentifier} in (${joinTarget})`; + } else { + return `(${constructSqlQuery(joinClause.subQuery, joinClause.from, null).query}) ${joinClause.subQueryAs}`; + } +} + +function formatFromClause(fromClause: FromClause): string { + if (fromClause.in !== undefined) { + return `${fromClause.in} IN ${fromClause.source}`; + } else { + return `${fromClause.source} ${fromClause.sourceAlias}`; + } +} + +/** Constructs a SQL query from the given `sqlQueryContext` + * @param sqlQueryCtx - `SqlQueryContext` which contains the data required to generate the SQL query. + * @param source - `source` to run the query on. Note that, the source can be a container or a nested field of a document of a container. + * @param queryVariables - values of the variables provided with the request. + + */ +function constructSqlQuery( + sqlQueryCtx: SqlQueryContext, + source: string, + queryVariables: QueryVariables, +): cosmos.SqlQuerySpec { + let selectColumns = formatSelectColumns(sqlQueryCtx.select); + + let fromClause = + sqlQueryCtx.from === null || sqlQueryCtx.from === undefined + ? null + : formatFromClause(sqlQueryCtx.from); + + let whereClause = null; + let predicateParameters: SqlParameters = {}; + let utilisedVariables: VariablesMappings = {}; // This will be used to add the join mappings to the where expression. + + let parameters: cosmos.SqlParameter[] = []; + + if (sqlQueryCtx.predicate != null && sqlQueryCtx.predicate != undefined) { + const whereExp = visitExpression( + predicateParameters, + utilisedVariables, + sqlQueryCtx.predicate, + source, + ); + + whereClause = `WHERE ${whereExp}`; + + parameters = serializeSqlParameters(predicateParameters); + + if (Object.keys(utilisedVariables).length > 0) { + if (queryVariables === null || queryVariables === undefined) { + throw new sdk.BadRequest( + `The variables (${JSON.stringify(Object.values(utilisedVariables))}) were referenced in the variable, but their values were not provided`, + ); + } else { + parameters.push({ + name: "@vars", + value: queryVariables as cosmos.JSONValue, + }); + } + } + } + + let joinClause = null; + + if (Object.keys(utilisedVariables).length > 0) { + let variablesJoinTarget: ArrayJoinTarget = { + kind: "containerExpression", + containerExpression: "SELECT VALUE @vars", + }; + let joinExp: JoinClause = { + type: "array", + joinIdentifier: "vars", + arrayJoinTarget: variablesJoinTarget, + }; + joinClause = `JOIN ${formatJoinClause(joinExp)}`; + } + + let orderByClause = null; + + if ( + sqlQueryCtx.orderBy != null && + sqlQueryCtx.orderBy != null && + sqlQueryCtx.orderBy.elements.length > 0 + ) { + orderByClause = visitOrderByElements(sqlQueryCtx.orderBy.elements, source); + } + + let offsetClause = null; + + if (sqlQueryCtx.offset != undefined && sqlQueryCtx.offset != null) { + offsetClause = `${sqlQueryCtx.offset}`; + } + + let limitClause = null; + + if (sqlQueryCtx.limit != undefined && sqlQueryCtx.limit != null) { + limitClause = `${sqlQueryCtx.limit}`; + } + + let query = `SELECT ${sqlQueryCtx.selectAsValue ? "VALUE" : ""} ${selectColumns} + ${fromClause ? "FROM " + fromClause : ""} + ${joinClause ?? ""} + ${whereClause ?? ""} + ${orderByClause ? "ORDER BY " + orderByClause : ""} + ${offsetClause ? "OFFSET " + offsetClause : ""} + ${limitClause ? "LIMIT " + limitClause : ""}`; + + return { + query, + parameters, + }; +} + +export function generateSqlQuerySpec( + sqlGenCtx: SqlQueryContext, + containerName: string, + queryVariables: QueryVariables, + schema: schema.CollectionsSchema, +): SqlQuerySpec { + return constructSqlQuery(sqlGenCtx, `root_${containerName}`, queryVariables); +} + +export function formatColumn(column: Column) { + return `${column.prefix}.${column.name}`; +} + +function formatSelectColumns(fieldsToSelect: SelectColumns): string { + if (Object.keys(fieldsToSelect).length === 0) { + return "VALUE {}"; + } + return Object.entries(fieldsToSelect) + .map(([alias, selectColumn]) => { + switch (selectColumn.kind) { + case "column": + return `${formatColumn(selectColumn.column)} ?? null as ${alias}`; + case "sqlQueryContext": + let query = constructSqlQuery(selectColumn, alias, null).query.trim(); + if (selectColumn.selectAsArray) { + return `(ARRAY(${query})) as ${alias}`; + } else { + return `(${query}) as ${alias}`; + } + case "aggregate": + return `${selectColumn.aggregateFunction} (${formatColumn(selectColumn.column)}) as ${alias} `; + } + }) + .join(","); +} + +/* + Traverses over the order by elements and generates the ORDER BY clause. + NOTE that this function expects the `values` parameter to be a non-empty list. + */ +function visitOrderByElements( + values: sdk.OrderByElement[], + containerAlias: string, +): string { + if (values.length === 0) { + throw new sdk.InternalServerError( + "visit_order_by_elements called with an empty list", + ); + } + return values + .map((element) => visitOrderByElement(element, containerAlias)) + .join(", "); +} + +function visitOrderByElement( + value: sdk.OrderByElement, + containerAlias: string, +): string { + const direction = value.order_direction === "asc" ? "ASC" : "DESC"; + + switch (value.target.type) { + case "column": + if (value.target.path.length > 0) { + throw new sdk.NotSupported( + "Relationships are not supported in order_by.", + ); + } else { + return `${containerAlias}.${value.target.name} ${direction} `; + } + + case "single_column_aggregate": + throw new sdk.NotSupported("Order by aggregate is not supported"); + + case "star_count_aggregate": + throw new sdk.NotSupported("Order by aggregate is not supported"); + } +} + +/* + Wraps the expression in parantheses to avoid generating SQL with wrong operator precedence. + */ +function visitExpressionWithParentheses( + parameters: SqlParameters, + variables: VariablesMappings, + expression: Expression, + containerAlias: string, +): string { + return `(${visitExpression(parameters, variables, expression, containerAlias)})`; +} + +function visitExpression( + parameters: SqlParameters, + variables: VariablesMappings, + expression: Expression, + containerAlias: string, +): string { + switch (expression.type) { + case "and": + if (expression.expressions.length > 0) { + return expression.expressions + .map((expr) => + visitExpressionWithParentheses( + parameters, + variables, + expr, + containerAlias, + ), + ) + .join(" AND "); + } else { + return "true"; + } + + case "or": + if (expression.expressions.length > 0) { + return expression.expressions + .map((expr) => + visitExpressionWithParentheses( + parameters, + variables, + expr, + containerAlias, + ), + ) + .join(" OR "); + } else { + return "false"; + } + + case "not": + return `NOT ${visitExpressionWithParentheses(parameters, variables, expression.expression, containerAlias)} `; + + case "unary_comparison_operator": + switch (expression.operator) { + case "is_null": + return `IS_NULL(${expression.column})`; + } + + case "binary_comparison_operator": + const comparisonValue = visitComparisonValue( + parameters, + variables, + expression.value, + expression.column, + containerAlias, + ); + + if (expression.dbOperator.isInfix) { + return `${containerAlias}.${expression.column} ${expression.dbOperator.name} ${comparisonValue}`; + } else { + return `${expression.dbOperator.name}(${containerAlias}.${expression.column}, ${comparisonValue}) `; + } + } +} + +export function visitComparisonTarget(target: sdk.ComparisonTarget): string { + switch (target.type) { + case "column": + if (target.path.length > 0) { + throw new sdk.NotSupported( + "Relationship fields are not supported in predicates.", + ); + } + return target.name; + case "root_collection_column": + throw new sdk.NotSupported( + "Root collection column comparison is not supported", + ); + } +} + +function visitComparisonValue( + parameters: SqlParameters, + variables: VariablesMappings, + target: ComparisonValue, + comparisonTarget: string, + containerAlias: string, +): string { + switch (target.type) { + case "scalar": + const comparisonTargetName = comparisonTarget.replace(".", "_"); + const comparisonTargetParameterValues = parameters[comparisonTargetName]; + if (comparisonTargetParameterValues != null) { + const index = comparisonTargetParameterValues.findIndex( + (element) => element === target.value, + ); + if (index !== -1) { + return `@${comparisonTargetName}_${index} `; + } else { + let newIndex = parameters[comparisonTargetName].push(target.value); + return `@${comparisonTargetName}_${newIndex} `; + } + } else { + parameters[comparisonTargetName] = [target.value]; + return `@${comparisonTargetName}_0`; + } + + case "column": + return `${containerAlias}.${target.column}`; + + case "variable": + variables[comparisonTarget] = `vars["${target.name}"]`; + return `vars["${target.name}"]`; + } +} + +function serializeSqlParameters( + parameters: SqlParameters, +): cosmos.SqlParameter[] { + let sqlParameters: cosmos.SqlParameter[] = []; + + for (const comparisonTarget in parameters) { + const comparisonTargetValues = parameters[comparisonTarget]; + + for (let i = 0; i < comparisonTargetValues.length; i++) { + sqlParameters.push({ + name: `@${comparisonTarget}_${i}`, + value: comparisonTargetValues[i], + }); + } + } + + return sqlParameters; +} diff --git a/src/index.ts b/src/index.ts index 0f3a3ae..3e27831 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,4 @@ import * as sdk from "@hasura/ndc-sdk-typescript"; -import { createConnector } from "./connector" +import { createConnector } from "./connector/connector"; -sdk.start(createConnector()) +sdk.start(createConnector()); diff --git a/src/schema.ts b/src/schema.ts deleted file mode 100644 index 15800b9..0000000 --- a/src/schema.ts +++ /dev/null @@ -1,471 +0,0 @@ -import * as sdk from "@hasura/ndc-sdk-typescript"; -import { mapObjectValues } from "./utils"; -import { ScalarType } from "@hasura/ndc-sdk-typescript"; - -export type CollectionsSchema = { - collections: CollectionDefinitions - objectTypes: ObjectTypeDefinitions - scalarTypes: ScalarTypeDefinitions -} - -export type CollectionDefinitions = { - [collectionName: string]: CollectionDefinition -} - -export type CollectionDefinition = { - description: string | null, - arguments: ArgumentDefinition[] - resultType: TypeDefinition -} - -export type ArgumentDefinition = { - argumentName: string, - description: string | null, - type: TypeDefinition -} - -export type ObjectTypeDefinitions = { - [objectTypeName: string]: ObjectTypeDefinition -} - -export type ObjectTypePropertiesMap = { - [propertyName: string]: ObjectPropertyDefinition -} - - -export type ObjectTypeDefinition = { - description: string | null, - properties: ObjectTypePropertiesMap -} - -export type ObjectPropertyDefinition = { - propertyName: string, - description: string | null, - type: TypeDefinition, -} - -export type ScalarTypeDefinitions = { - [scalarTypeName: string]: ScalarTypeDefinition -} - -export type ScalarTypeDefinition = BuiltInScalarTypeDefinition // Empty object, for now - -export type TypeDefinition = ArrayTypeDefinition | NullableTypeDefinition | NamedTypeDefinition - -export type ArrayTypeDefinition = { - type: "array" - elementType: TypeDefinition -} - -export type NullableTypeDefinition = { - type: "nullable", - underlyingType: TypeDefinition -} - -export type NamedTypeDefinition = NamedObjectTypeDefinition | NamedScalarTypeDefinition - -export type NamedObjectTypeDefinition = { - type: "named" - name: string - kind: "object" -} - -export type NamedScalarTypeDefinition = CustomNamedScalarTypeDefinition | BuiltInScalarTypeDefinition - -export type BuiltInScalarTypeDefinition = StringScalarTypeDefinition | BooleanScalarTypeDefinition | IntegerScalarTypeDefinition | NumberScalarTypeDefinition | DateTimeScalarTypeDefinition - -export type CustomNamedScalarTypeDefinition = { - type: "named" - name: string - kind: "scalar" -} - -export type StringScalarTypeDefinition = { - type: "named" - name: BuiltInScalarTypeName.String - kind: "scalar" - literalValue?: string -} - -export type NumberScalarTypeDefinition = { - type: "named" - name: BuiltInScalarTypeName.Number - kind: "scalar" - literalValue?: number -} - -export type BooleanScalarTypeDefinition = { - type: "named" - name: BuiltInScalarTypeName.Boolean - kind: "scalar" - literalValue?: boolean -} - -export type DateTimeScalarTypeDefinition = { - type: "named" - name: BuiltInScalarTypeName.DateTime - kind: "scalar" -} - -export type IntegerScalarTypeDefinition = { - type: "named" - name: BuiltInScalarTypeName.Integer - kind: "scalar" -} - - -export enum BuiltInScalarTypeName { - String = "String", - Number = "Number", - Boolean = "Boolean", - DateTime = "DateTime", - Integer = "Integer" -} - -export type ScalarTypes = { - [k: string]: ScalarType; -}; - -export type ScalarOperatorMappings = { - [k: string]: string -}; - -export const scalarTypes: ScalarTypes = { - "Integer": { - aggregate_functions: { - "count": { - result_type: { - type: "named", - name: BuiltInScalarTypeName.Integer, - } - }, - "sum": { - result_type: { - type: "named", - name: BuiltInScalarTypeName.Integer, - } - }, - "avg": { - result_type: { - type: "named", - name: BuiltInScalarTypeName.Integer, - } - }, - "min": { - result_type: { - type: "named", - name: BuiltInScalarTypeName.Integer, - } - }, - "max": { - result_type: { - type: "named", - name: BuiltInScalarTypeName.Integer, - } - } - }, - comparison_operators: { - eq: { - type: "equal" - }, - neq: { - type: "custom", - argument_type: { - type: "named", - name: "Integer" - } - }, - gt: { - type: "custom", - argument_type: { - type: "named", - name: "Integer" - } - }, - lt: { - type: "custom", - argument_type: { - type: "named", - name: "Integer" - } - }, - gte: { - type: "custom", - argument_type: { - type: "named", - name: "Integer" - } - }, - lte: { - type: "custom", - argument_type: { - type: "named", - name: "Integer" - } - } - }, - }, - "Number": { - aggregate_functions: { - "count": { - result_type: { - type: "named", - name: BuiltInScalarTypeName.Number, - } - }, - "sum": { - result_type: { - type: "named", - name: BuiltInScalarTypeName.Number, - } - }, - "avg": { - result_type: { - type: "named", - name: BuiltInScalarTypeName.Number, - } - }, - "min": { - result_type: { - type: "named", - name: BuiltInScalarTypeName.Number, - } - }, - "max": { - result_type: { - type: "named", - name: BuiltInScalarTypeName.Number, - } - } - }, - comparison_operators: { - eq: { - type: "equal" - }, - neq: { - type: "custom", - argument_type: { - type: "named", - name: "Number" - } - }, - gt: { - type: "custom", - argument_type: { - type: "named", - name: "Number" - } - }, - lt: { - type: "custom", - argument_type: { - type: "named", - name: "Number" - } - }, - gte: { - type: "custom", - argument_type: { - type: "named", - name: "Number" - } - }, - lte: { - type: "custom", - argument_type: { - type: "named", - name: "Number" - } - } - }, - }, - "Boolean": { - aggregate_functions: { - "bool_and": { - result_type: { - type: "named", - name: BuiltInScalarTypeName.Boolean, - } - }, - "bool_or": { - result_type: { - type: "named", - name: BuiltInScalarTypeName.Boolean, - } - }, - "bool_not": { - result_type: { - type: "named", - name: BuiltInScalarTypeName.Boolean, - } - } - }, - comparison_operators: { - eq: { - type: "equal" - }, - neq: { - type: "custom", - argument_type: { - type: "named", - name: "Boolean" - } - } - }, - }, - "String": { - aggregate_functions: {}, - comparison_operators: { - eq: { - type: "equal" - }, - neq: { - type: "custom", - argument_type: { - type: "named", - name: "String" - } - }, - gt: { - type: "custom", - argument_type: { - type: "named", - name: "String" - } - }, - lt: { - type: "custom", - argument_type: { - type: "named", - name: "String" - } - }, - gte: { - type: "custom", - argument_type: { - type: "named", - name: "String" - } - }, - lte: { - type: "custom", - argument_type: { - type: "named", - name: "String" - } - }, - contains: { - type: "custom", - argument_type: { - type: "named", - name: "String" - } - }, - endswith: { - type: "custom", - argument_type: { - type: "named", - name: "String" - } - }, - regexmatch: { - type: "custom", - argument_type: { - type: "named", - name: "String" - } - }, - startswith: { - type: "custom", - argument_type: { - type: "named", - name: "String" - } - } - }, - } -}; - -export function getJSONScalarTypes(): ScalarTypeDefinitions { - var scalarTypeDefinitions: ScalarTypeDefinitions = {}; - scalarTypeDefinitions["Integer"] = { - type: "named", - name: BuiltInScalarTypeName.Integer, - kind: "scalar" - }; - scalarTypeDefinitions["Number"] = { - type: "named", - name: BuiltInScalarTypeName.Number, - kind: "scalar" - }; - scalarTypeDefinitions["Boolean"] = { - type: "named", - name: BuiltInScalarTypeName.Boolean, - kind: "scalar" - }; - scalarTypeDefinitions["String"] = { - type: "named", - name: BuiltInScalarTypeName.String, - kind: "scalar" - }; - - return scalarTypeDefinitions -} - - - - -export function getNdcSchemaResponse(collectionsSchema: CollectionsSchema): sdk.SchemaResponse { - const collections = Object.entries(collectionsSchema.collections); - - var collectionInfos = collections.map(([collectionName, collectionInfo]) => { - return { - name: collectionName, - description: null, - arguments: {}, - type: getBaseNamedType(collectionInfo.resultType), - uniqueness_constraints: {}, - foreign_keys: {} - } - }) - - - const objectTypes = mapObjectValues(collectionsSchema.objectTypes, objDef => { - return { - fields: Object.fromEntries(Object.values(objDef.properties).map(propDef => { - const objField: sdk.ObjectField = { - type: convertTypeReferenceToSdkType(propDef.type), - description: null - } - return [propDef.propertyName, objField]; - })), - ...(objDef.description ? { description: objDef.description } : {}) - } - }); - - - - return { - functions: [], - procedures: [], - collections: collectionInfos, - object_types: objectTypes, - scalar_types: scalarTypes, - } -} - -function convertTypeReferenceToSdkType(typeRef: TypeDefinition): sdk.Type { - switch (typeRef.type) { - case "array": return { type: "array", element_type: convertTypeReferenceToSdkType(typeRef.elementType) } - case "nullable": return { type: "nullable", underlying_type: convertTypeReferenceToSdkType(typeRef.underlyingType) } - case "named": return { type: "named", name: typeRef.name } - } -} - -export function getBaseNamedType(typeRef: TypeDefinition): string { - switch (typeRef.type) { - case "array": return getBaseNamedType(typeRef.elementType) - case "nullable": return getBaseNamedType(typeRef.underlyingType) - case "named": return typeRef.name - } -} diff --git a/src/sqlGeneration.ts b/src/sqlGeneration.ts deleted file mode 100644 index 8c209b0..0000000 --- a/src/sqlGeneration.ts +++ /dev/null @@ -1,655 +0,0 @@ -import * as sdk from "@hasura/ndc-sdk-typescript"; -import * as cosmos from "@azure/cosmos"; -import { SqlQuerySpec } from "@azure/cosmos"; -import * as schema from "./schema"; - -export type Column = { - name: string, - prefix: string, -} - -export type SelectContainerColumn = { - kind: 'column', - column: Column -} - -export type SelectAggregate = { - kind: 'aggregate', - column: Column, - aggregateFunction: string -} - -export type SelectColumn = SelectContainerColumn | SelectAggregate | SqlQueryContext - -/* - The key represents the alias of the request field and the - value represents the value to be selected from the container. -*/ -export type SelectColumns = { - [alias: string]: SelectColumn -} - -export type QueryVariable = { - [k: string]: unknown -} - -export type QueryVariables = QueryVariable[] | null | undefined - -/* - Type to track the parameters used in the SQL query. - */ -type SqlParameters = { - [column: string]: any[] -} - -export type FromClause = { - source: string, - sourceAlias: string, - in?: string, -} - -export type ContainerExpression = { - kind: 'containerExpression', - containerExpression: string -} - -export type SqlExpression = { - kind: 'sqlExpression', - sqlExpression: SqlQueryContext -} - -export type ArrayJoinTarget = ContainerExpression | SqlExpression - -export type ArrayJoinClause = { - type: 'array', - joinIdentifier: string, - arrayJoinTarget: ArrayJoinTarget, -} - -export type SubqueryJoinClause = { - type: 'subquery', - from: string, - subQuery: SqlQueryContext, - subQueryAs: string, -} - -export type JoinClause = ArrayJoinClause | SubqueryJoinClause; - -type ComparisonScalarDbOperator = { - name: string, - isInfix: boolean -} - -type AggregateScalarDbOperator = { - operator: string, - resultType: string -} - -// Defines how the NDC's scalar operators map to the DB operators -type ScalarDBOperatorMappings = { - comparison: { - [operatorName: string]: ComparisonScalarDbOperator - }, - aggregate?: { - [operatorName: string]: AggregateScalarDbOperator - } | undefined - -}; - -type ScalarOperatorMappings = { - [scalarTypeName: string]: ScalarDBOperatorMappings -} - - -export const scalarComparisonOperatorMappings: ScalarOperatorMappings = { - "Integer": { - "comparison": { - "eq": { - "name": "=", - "isInfix": true - }, - "neq": { - "name": "!=", - "isInfix": true - }, - "gt": { - "name": ">", - "isInfix": true - }, - "lt": { - "name": "<", - "isInfix": true - }, - "gte": { - "name": ">=", - "isInfix": true - }, - "lte": { - "name": "<=", - "isInfix": true - } - }, - "aggregate": { - "count": { - "operator": "count", - "resultType": "Integer" - }, - "sum": { - "operator": "sum", - "resultType": "Integer" - }, - "avg": { - "operator": "sum", - "resultType": "Number" - }, - "min": { - "operator": "sum", - "resultType": "Integer" - }, - "max": { - "operator": "sum", - "resultType": "Integer" - }, - } - }, - "Number": { - "comparison": { - "eq": { - "name": "=", - "isInfix": true - }, - "neq": { - "name": "!=", - "isInfix": true - }, - "gt": { - "name": ">", - "isInfix": true - }, - "lt": { - "name": "<", - "isInfix": true - }, - "gte": { - "name": ">=", - "isInfix": true - }, - "lte": { - "name": "<=", - "isInfix": true - } - }, - "aggregate": { - "count": { - "operator": "count", - "resultType": "Integer" - }, - "sum": { - "operator": "sum", - "resultType": "Number" - }, - "avg": { - "operator": "sum", - "resultType": "Number" - }, - "min": { - "operator": "sum", - "resultType": "Number" - }, - "max": { - "operator": "sum", - "resultType": "Number" - }, - } - }, - "Boolean": { - "comparison": { - "eq": { - "name": "=", - "isInfix": true - }, - "neq": { - "name": "!=", - "isInfix": true - } - }, - "aggregate": { - "bool_and": { - "operator": "bool_and", - "resultType": "Boolean" - }, - "bool_or": { - "operator": "bool_or", - "resultType": "Boolean" - }, - "bool_not": { - "operator": "bool_or", - "resultType": "Boolean" - }, - } - - }, - "String": { - "comparison": { - "eq": { - "name": "=", - "isInfix": true - }, - "neq": { - "name": "!=", - "isInfix": true - }, - "gt": { - "name": ">", - "isInfix": true - }, - "lt": { - "name": "<", - "isInfix": true - }, - "gte": { - "name": ">=", - "isInfix": true - }, - "lte": { - "name": "<=", - "isInfix": true - }, - "contains": { - "name": "CONTAINS", - "isInfix": false, - }, - "endswith": { - "name": "ENDSWITH", - "isInfix": false, - }, - "regexmatch": { - "name": "REGEXMATCH", - "isInfix": false, - }, - "startswith": { - "name": "STARTSWITH", - "isInfix": false - } - } - - }, -}; - -export function getDbComparisonOperator(scalarTypeName: string, operator: string): ComparisonScalarDbOperator { - const scalarOperators = scalarComparisonOperatorMappings[scalarTypeName]; - - if (scalarOperators === undefined && scalarOperators === null) { - throw new sdk.BadRequest(`Couldn't find scalar type: ${scalarTypeName} in the schema`) - } else { - const scalarDbOperator = scalarOperators.comparison[operator]; - - if (scalarDbOperator) { - return scalarDbOperator - } else { - throw new sdk.BadRequest(`Comparison Operator ${operator} is not supported on type ${scalarTypeName}`) - } - } - - -} - - -export type ComparisonTarget = - | { - type: "column"; - /** - * The name of the column - */ - name: string; - } - | { - type: "root_collection_column"; - /** - * The name of the column - */ - name: string; - }; - -export type ComparisonValue = - | { - type: "column"; - column: string; - } - | { - type: "scalar"; - value: unknown; - } - | { - type: "variable"; - name: string; - }; - -export type Expression = - | { - type: "and"; - expressions: Expression[]; - } - | { - type: "or"; - expressions: Expression[]; - } - | { - type: "not"; - expression: Expression; - } - | { - type: "unary_comparison_operator"; - column: string; - operator: "is_null"; - } - | { - type: "binary_comparison_operator"; - column: string; - value: ComparisonValue; - dbOperator: ComparisonScalarDbOperator; - }; - - - - -export type SqlQueryContext = { - kind: 'sqlQueryContext', - select: SelectColumns, - /* Set to `true` to prevent the wrapping of the results into another JSON object. */ - selectAsValue: boolean, - from?: FromClause | null, - join?: JoinClause[] | null, - predicate?: Expression | null, - offset?: number | null, - limit?: number | null, - orderBy?: sdk.OrderBy | null, - isAggregateQuery: boolean, - selectAsArray?: boolean | undefined -} - -type VariablesMappings = { - /* - The variableTarget will be the name of the column - which gets the value of the variable - */ - [variableTarget: string]: string -} - -function formatJoinClause(joinClause: JoinClause): string { - if (joinClause.type === "array") { - let joinTarget = - joinClause.arrayJoinTarget.kind === 'containerExpression' - ? joinClause.arrayJoinTarget.containerExpression - : constructSqlQuery(joinClause.arrayJoinTarget.sqlExpression, joinClause.joinIdentifier, null); - - return `${joinClause.joinIdentifier} in (${joinTarget})` - } else { - return `(${constructSqlQuery(joinClause.subQuery, joinClause.from, null).query}) ${joinClause.subQueryAs}`; - } - -} - -function formatFromClause(fromClause: FromClause): string { - if (fromClause.in !== undefined) { - return `${fromClause.in} IN ${fromClause.source}` - } else { - return `${fromClause.source} ${fromClause.sourceAlias}` - } -} - -/** Constructs a SQL query from the given `sqlQueryContext` - * @param sqlQueryCtx - `SqlQueryContext` which contains the data required to generate the SQL query. - * @param source - `source` to run the query on. Note that, the source can be a container or a nested field of a document of a container. - * @param queryVariables - values of the variables provided with the request. - - */ -function constructSqlQuery(sqlQueryCtx: SqlQueryContext, source: string, queryVariables: QueryVariables): cosmos.SqlQuerySpec { - let selectColumns = formatSelectColumns(sqlQueryCtx.select); - - let fromClause = - sqlQueryCtx.from === null || sqlQueryCtx.from === undefined - ? null : - formatFromClause(sqlQueryCtx.from); - - let whereClause = null; - let predicateParameters: SqlParameters = {}; - let utilisedVariables: VariablesMappings = {}; // This will be used to add the join mappings to the where expression. - - let parameters: cosmos.SqlParameter[] = []; - - if (sqlQueryCtx.predicate != null && sqlQueryCtx.predicate != undefined) { - - const whereExp = visitExpression(predicateParameters, utilisedVariables, sqlQueryCtx.predicate, source); - - whereClause = `WHERE ${whereExp}` - - parameters = serializeSqlParameters(predicateParameters); - - if (Object.keys(utilisedVariables).length > 0) { - if (queryVariables === null || queryVariables === undefined) { - throw new sdk.BadRequest(`The variables (${JSON.stringify(Object.values(utilisedVariables))}) were referenced in the variable, but their values were not provided`) - } else { - parameters.push({ - name: '@vars', - value: queryVariables as cosmos.JSONValue - }); - } - - } - } - - let joinClause = null; - - if (Object.keys(utilisedVariables).length > 0) { - let variablesJoinTarget: ArrayJoinTarget = { - kind: 'containerExpression', - containerExpression: 'SELECT VALUE @vars' - }; - let joinExp: JoinClause = { - type: 'array', - joinIdentifier: "vars", - arrayJoinTarget: variablesJoinTarget, - }; - joinClause = `JOIN ${formatJoinClause(joinExp)}` - } - - let orderByClause = null; - - if (sqlQueryCtx.orderBy != null && sqlQueryCtx.orderBy != null && sqlQueryCtx.orderBy.elements.length > 0) { - orderByClause = visitOrderByElements(sqlQueryCtx.orderBy.elements, source); - } - - let offsetClause = null; - - if (sqlQueryCtx.offset != undefined && sqlQueryCtx.offset != null) { - offsetClause = `${sqlQueryCtx.offset}`; - } - - let limitClause = null; - - if (sqlQueryCtx.limit != undefined && sqlQueryCtx.limit != null) { - limitClause = `${sqlQueryCtx.limit}` - - } - - let query = - `SELECT ${sqlQueryCtx.selectAsValue ? 'VALUE' : ''} ${selectColumns} - ${fromClause ? 'FROM ' + fromClause : ''} - ${joinClause ?? ''} - ${whereClause ?? ''} - ${orderByClause ? 'ORDER BY ' + orderByClause : ''} - ${offsetClause ? 'OFFSET ' + offsetClause : ''} - ${limitClause ? 'LIMIT ' + limitClause : ''}`; - - - return { - query, - parameters - } -} - -export function generateSqlQuerySpec(sqlGenCtx: SqlQueryContext, containerName: string, queryVariables: QueryVariables, schema: schema.CollectionsSchema): SqlQuerySpec { - - return constructSqlQuery(sqlGenCtx, `root_${containerName}`, queryVariables); - -} - -export function formatColumn(column: Column) { - return `${column.prefix}.${column.name}` -} - - -function formatSelectColumns(fieldsToSelect: SelectColumns): string { - if (Object.keys(fieldsToSelect).length === 0) { - return "VALUE {}" - } - return Object.entries(fieldsToSelect).map(([alias, selectColumn]) => { - switch (selectColumn.kind) { - case 'column': - return `${formatColumn(selectColumn.column)} ?? null as ${alias}` - case 'sqlQueryContext': - let query = constructSqlQuery(selectColumn, alias, null).query.trim(); - if (selectColumn.selectAsArray) { - return `(ARRAY(${query})) as ${alias}` - } else { - return `(${query}) as ${alias}` - } - case 'aggregate': - return `${selectColumn.aggregateFunction} (${formatColumn(selectColumn.column)}) as ${alias} ` - } - }).join(","); - -} - -/* - Traverses over the order by elements and generates the ORDER BY clause. - NOTE that this function expects the `values` parameter to be a non-empty list. - */ -function visitOrderByElements(values: sdk.OrderByElement[], containerAlias: string): string { - if (values.length === 0) { - throw new sdk.InternalServerError("visit_order_by_elements called with an empty list") - } - return values.map(element => visitOrderByElement(element, containerAlias)).join(", "); - -} - -function visitOrderByElement(value: sdk.OrderByElement, containerAlias: string): string { - const direction = value.order_direction === 'asc' ? 'ASC' : 'DESC'; - - switch (value.target.type) { - case 'column': - if (value.target.path.length > 0) { - throw new sdk.NotSupported("Relationships are not supported in order_by.") - } else { - return `${containerAlias}.${value.target.name} ${direction} ` - } - - case 'single_column_aggregate': - throw new sdk.NotSupported("Order by aggregate is not supported") - - case 'star_count_aggregate': - throw new sdk.NotSupported("Order by aggregate is not supported") - } -} - -/* - Wraps the expression in parantheses to avoid generating SQL with wrong operator precedence. - */ -function visitExpressionWithParentheses(parameters: SqlParameters, variables: VariablesMappings, expression: Expression, containerAlias: string): string { - return `(${visitExpression(parameters, variables, expression, containerAlias)})` -} - -function visitExpression(parameters: SqlParameters, variables: VariablesMappings, expression: Expression, containerAlias: string): string { - switch (expression.type) { - case "and": - if (expression.expressions.length > 0) { - return expression.expressions.map(expr => visitExpressionWithParentheses(parameters, variables, expr, containerAlias)).join(" AND ") - } else { - return "true" - }; - - case "or": - if (expression.expressions.length > 0) { - return expression.expressions.map(expr => visitExpressionWithParentheses(parameters, variables, expr, containerAlias)).join(" OR ") - } else { - return "false" - }; - - case "not": - return `NOT ${visitExpressionWithParentheses(parameters, variables, expression.expression, containerAlias)} ` - - case "unary_comparison_operator": - switch (expression.operator) { - case "is_null": - return `IS_NULL(${expression.column})` - } - - case "binary_comparison_operator": - const comparisonValue = visitComparisonValue(parameters, variables, expression.value, expression.column, containerAlias); - - if (expression.dbOperator.isInfix) { - return `${containerAlias}.${expression.column} ${expression.dbOperator.name} ${comparisonValue}` - } else { - return `${expression.dbOperator.name}(${containerAlias}.${expression.column}, ${comparisonValue}) ` - } - - } -} - -export function visitComparisonTarget(target: sdk.ComparisonTarget): string { - switch (target.type) { - case 'column': - if (target.path.length > 0) { - throw new sdk.NotSupported("Relationship fields are not supported in predicates."); - } - return target.name; - case 'root_collection_column': - throw new sdk.NotSupported("Root collection column comparison is not supported"); - } -} - -function visitComparisonValue(parameters: SqlParameters, variables: VariablesMappings, target: ComparisonValue, comparisonTarget: string, containerAlias: string): string { - switch (target.type) { - case 'scalar': - const comparisonTargetName = comparisonTarget.replace(".", "_"); - const comparisonTargetParameterValues = parameters[comparisonTargetName]; - if (comparisonTargetParameterValues != null) { - const index = comparisonTargetParameterValues.findIndex((element) => element === target.value); - if (index !== -1) { - return `@${comparisonTargetName}_${index} ` - } else { - let newIndex = parameters[comparisonTargetName].push(target.value); - return `@${comparisonTargetName}_${newIndex} ` - } - } else { - parameters[comparisonTargetName] = [target.value]; - return `@${comparisonTargetName}_0` - } - - case 'column': - return `${containerAlias}.${target.column}` - - case 'variable': - variables[comparisonTarget] = `vars["${target.name}"]` - return `vars["${target.name}"]` - - } -} - -function serializeSqlParameters(parameters: SqlParameters): cosmos.SqlParameter[] { - let sqlParameters: cosmos.SqlParameter[] = []; - - for (const comparisonTarget in parameters) { - const comparisonTargetValues = parameters[comparisonTarget]; - - for (let i = 0; i < comparisonTargetValues.length; i++) { - sqlParameters.push({ - name: `@${comparisonTarget}_${i}`, - value: comparisonTargetValues[i] - }) - } - } - - return sqlParameters -}