From 985272012a7183fb690ecdd7bcfafd42e3103539 Mon Sep 17 00:00:00 2001 From: Mario Castro Squella Date: Mon, 30 Sep 2024 15:17:10 -0300 Subject: [PATCH] Fix read model queries when selecting deeply-nested arrays (local provider) (#1553) --- .../fix_nested_arrays_2024-09-27-19-40.json | 10 ++ common/config/rush/pnpm-lock.yaml | 106 ++++++------- .../src/services/read-model-registry.ts | 149 ++++++------------ .../test/services/read-model-registry.test.ts | 68 ++++++++ 4 files changed, 180 insertions(+), 153 deletions(-) create mode 100644 common/changes/@boostercloud/framework-core/fix_nested_arrays_2024-09-27-19-40.json diff --git a/common/changes/@boostercloud/framework-core/fix_nested_arrays_2024-09-27-19-40.json b/common/changes/@boostercloud/framework-core/fix_nested_arrays_2024-09-27-19-40.json new file mode 100644 index 000000000..187424cd3 --- /dev/null +++ b/common/changes/@boostercloud/framework-core/fix_nested_arrays_2024-09-27-19-40.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@boostercloud/framework-core", + "comment": "Fix handling of deeply-nested arrays an sub-array properties in Read Model queries in local provider", + "type": "patch" + } + ], + "packageName": "@boostercloud/framework-core" +} \ No newline at end of file diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index f75e90281..38ceee7df 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -8,8 +8,8 @@ importers: ../../packages/application-tester: specifiers: '@apollo/client': 3.7.13 - '@boostercloud/eslint-config': workspace:^2.18.1 - '@boostercloud/framework-types': workspace:^2.18.1 + '@boostercloud/eslint-config': workspace:^2.18.2 + '@boostercloud/framework-types': workspace:^2.18.2 '@effect-ts/core': ^0.60.4 '@types/jsonwebtoken': 9.0.1 '@types/node': ^18.18.2 @@ -70,10 +70,10 @@ importers: ../../packages/cli: specifiers: - '@boostercloud/application-tester': workspace:^2.18.1 - '@boostercloud/eslint-config': workspace:^2.18.1 - '@boostercloud/framework-core': workspace:^2.18.1 - '@boostercloud/framework-types': workspace:^2.18.1 + '@boostercloud/application-tester': workspace:^2.18.2 + '@boostercloud/eslint-config': workspace:^2.18.2 + '@boostercloud/framework-core': workspace:^2.18.2 + '@boostercloud/framework-types': workspace:^2.18.2 '@effect-ts/core': ^0.60.4 '@oclif/core': 3.15.0 '@oclif/plugin-help': ^5 @@ -183,8 +183,8 @@ importers: ../../packages/framework-common-helpers: specifiers: - '@boostercloud/eslint-config': workspace:^2.18.1 - '@boostercloud/framework-types': workspace:^2.18.1 + '@boostercloud/eslint-config': workspace:^2.18.2 + '@boostercloud/framework-types': workspace:^2.18.2 '@effect-ts/core': ^0.60.4 '@types/chai': 4.2.18 '@types/chai-as-promised': 7.1.4 @@ -258,10 +258,10 @@ importers: ../../packages/framework-core: specifiers: - '@boostercloud/eslint-config': workspace:^2.18.1 - '@boostercloud/framework-common-helpers': workspace:^2.18.1 - '@boostercloud/framework-types': workspace:^2.18.1 - '@boostercloud/metadata-booster': workspace:^2.18.1 + '@boostercloud/eslint-config': workspace:^2.18.2 + '@boostercloud/framework-common-helpers': workspace:^2.18.2 + '@boostercloud/framework-types': workspace:^2.18.2 + '@boostercloud/metadata-booster': workspace:^2.18.2 '@effect/cli': 0.35.26 '@effect/platform': 0.48.24 '@effect/platform-node': 0.45.26 @@ -378,19 +378,19 @@ importers: ../../packages/framework-integration-tests: specifiers: '@apollo/client': 3.7.13 - '@boostercloud/application-tester': workspace:^2.18.1 - '@boostercloud/cli': workspace:^2.18.1 - '@boostercloud/eslint-config': workspace:^2.18.1 - '@boostercloud/framework-common-helpers': workspace:^2.18.1 - '@boostercloud/framework-core': workspace:^2.18.1 - '@boostercloud/framework-provider-aws': workspace:^2.18.1 - '@boostercloud/framework-provider-aws-infrastructure': workspace:^2.18.1 - '@boostercloud/framework-provider-azure': workspace:^2.18.1 - '@boostercloud/framework-provider-azure-infrastructure': workspace:^2.18.1 - '@boostercloud/framework-provider-local': workspace:^2.18.1 - '@boostercloud/framework-provider-local-infrastructure': workspace:^2.18.1 - '@boostercloud/framework-types': workspace:^2.18.1 - '@boostercloud/metadata-booster': workspace:^2.18.1 + '@boostercloud/application-tester': workspace:^2.18.2 + '@boostercloud/cli': workspace:^2.18.2 + '@boostercloud/eslint-config': workspace:^2.18.2 + '@boostercloud/framework-common-helpers': workspace:^2.18.2 + '@boostercloud/framework-core': workspace:^2.18.2 + '@boostercloud/framework-provider-aws': workspace:^2.18.2 + '@boostercloud/framework-provider-aws-infrastructure': workspace:^2.18.2 + '@boostercloud/framework-provider-azure': workspace:^2.18.2 + '@boostercloud/framework-provider-azure-infrastructure': workspace:^2.18.2 + '@boostercloud/framework-provider-local': workspace:^2.18.2 + '@boostercloud/framework-provider-local-infrastructure': workspace:^2.18.2 + '@boostercloud/framework-types': workspace:^2.18.2 + '@boostercloud/metadata-booster': workspace:^2.18.2 '@effect-ts/core': ^0.60.4 '@effect/cli': 0.35.26 '@effect/platform': 0.48.24 @@ -536,9 +536,9 @@ importers: ../../packages/framework-provider-aws: specifiers: - '@boostercloud/eslint-config': workspace:^2.18.1 - '@boostercloud/framework-common-helpers': workspace:^2.18.1 - '@boostercloud/framework-types': workspace:^2.18.1 + '@boostercloud/eslint-config': workspace:^2.18.2 + '@boostercloud/framework-common-helpers': workspace:^2.18.2 + '@boostercloud/framework-types': workspace:^2.18.2 '@effect-ts/core': ^0.60.4 '@types/aws-lambda': 8.10.48 '@types/chai': 4.2.18 @@ -632,10 +632,10 @@ importers: '@aws-cdk/core': ^1.170.0 '@aws-cdk/custom-resources': ^1.170.0 '@aws-cdk/cx-api': ^1.170.0 - '@boostercloud/eslint-config': workspace:^2.18.1 - '@boostercloud/framework-common-helpers': workspace:^2.18.1 - '@boostercloud/framework-provider-aws': workspace:^2.18.1 - '@boostercloud/framework-types': workspace:^2.18.1 + '@boostercloud/eslint-config': workspace:^2.18.2 + '@boostercloud/framework-common-helpers': workspace:^2.18.2 + '@boostercloud/framework-provider-aws': workspace:^2.18.2 + '@boostercloud/framework-types': workspace:^2.18.2 '@effect-ts/core': ^0.60.4 '@types/archiver': 5.1.0 '@types/aws-lambda': 8.10.48 @@ -749,9 +749,9 @@ importers: '@azure/functions': ^1.2.2 '@azure/identity': ~2.1.0 '@azure/web-pubsub': ~1.1.0 - '@boostercloud/eslint-config': workspace:^2.18.1 - '@boostercloud/framework-common-helpers': workspace:^2.18.1 - '@boostercloud/framework-types': workspace:^2.18.1 + '@boostercloud/eslint-config': workspace:^2.18.2 + '@boostercloud/framework-common-helpers': workspace:^2.18.2 + '@boostercloud/framework-types': workspace:^2.18.2 '@cdktf/provider-azurerm': 13.3.0 '@effect-ts/core': ^0.60.4 '@types/chai': 4.2.18 @@ -825,11 +825,11 @@ importers: '@azure/arm-resources': ^5.0.1 '@azure/cosmos': ^4.0.0 '@azure/identity': ~2.1.0 - '@boostercloud/eslint-config': workspace:^2.18.1 - '@boostercloud/framework-common-helpers': workspace:^2.18.1 - '@boostercloud/framework-core': workspace:^2.18.1 - '@boostercloud/framework-provider-azure': workspace:^2.18.1 - '@boostercloud/framework-types': workspace:^2.18.1 + '@boostercloud/eslint-config': workspace:^2.18.2 + '@boostercloud/framework-common-helpers': workspace:^2.18.2 + '@boostercloud/framework-core': workspace:^2.18.2 + '@boostercloud/framework-provider-azure': workspace:^2.18.2 + '@boostercloud/framework-types': workspace:^2.18.2 '@cdktf/provider-azurerm': 13.3.0 '@cdktf/provider-time': 9.0.2 '@effect-ts/core': ^0.60.4 @@ -936,9 +936,9 @@ importers: ../../packages/framework-provider-local: specifiers: - '@boostercloud/eslint-config': workspace:^2.18.1 - '@boostercloud/framework-common-helpers': workspace:^2.18.1 - '@boostercloud/framework-types': workspace:^2.18.1 + '@boostercloud/eslint-config': workspace:^2.18.2 + '@boostercloud/framework-common-helpers': workspace:^2.18.2 + '@boostercloud/framework-types': workspace:^2.18.2 '@effect-ts/core': ^0.60.4 '@seald-io/nedb': 4.0.2 '@types/chai': 4.2.18 @@ -1015,10 +1015,10 @@ importers: ../../packages/framework-provider-local-infrastructure: specifiers: - '@boostercloud/eslint-config': workspace:^2.18.1 - '@boostercloud/framework-common-helpers': workspace:^2.18.1 - '@boostercloud/framework-provider-local': workspace:^2.18.1 - '@boostercloud/framework-types': workspace:^2.18.1 + '@boostercloud/eslint-config': workspace:^2.18.2 + '@boostercloud/framework-common-helpers': workspace:^2.18.2 + '@boostercloud/framework-provider-local': workspace:^2.18.2 + '@boostercloud/framework-types': workspace:^2.18.2 '@effect-ts/core': ^0.60.4 '@types/chai': 4.2.18 '@types/chai-as-promised': 7.1.4 @@ -1098,8 +1098,8 @@ importers: ../../packages/framework-types: specifiers: - '@boostercloud/eslint-config': workspace:^2.18.1 - '@boostercloud/metadata-booster': workspace:^2.18.1 + '@boostercloud/eslint-config': workspace:^2.18.2 + '@boostercloud/metadata-booster': workspace:^2.18.2 '@effect-ts/core': ^0.60.4 '@effect-ts/node': ~0.39.0 '@effect/cli': 0.35.26 @@ -1183,7 +1183,7 @@ importers: ../../packages/metadata-booster: specifiers: - '@boostercloud/eslint-config': workspace:^2.18.1 + '@boostercloud/eslint-config': workspace:^2.18.2 '@effect-ts/core': ^0.60.4 '@types/node': ^18.18.2 '@typescript-eslint/eslint-plugin': ^5.0.0 @@ -6419,7 +6419,7 @@ packages: dependencies: semver: 7.6.3 shelljs: 0.8.5 - typescript: 5.7.0-dev.20240925 + typescript: 5.7.0-dev.20240927 /duration/0.2.2: resolution: {integrity: sha512-06kgtea+bGreF5eKYgI/36A6pLXggY7oR4p1pq4SmdFBn1ReOL5D8RhG64VrqfTTKNucqqtBAwEj8aB88mcqrg==} @@ -11566,8 +11566,8 @@ packages: engines: {node: '>=14.17'} hasBin: true - /typescript/5.7.0-dev.20240925: - resolution: {integrity: sha512-k43Yzt0H2VwIvVJzSKaMUKSnAOWkAitSB+Zioc340LCsh4kSwCbyHCJSAJmAdFH0QuNIL3AuSybTfwOys8tqZw==} + /typescript/5.7.0-dev.20240927: + resolution: {integrity: sha512-IWKZHTHAlS8BglLp8iM4rUHhy0h79B9r9vj6b6zpa8U38ofctFS1fLiKY7okZ3JYeG15kUHuOwsLwOmvc5+e1Q==} engines: {node: '>=14.17'} hasBin: true diff --git a/packages/framework-provider-local/src/services/read-model-registry.ts b/packages/framework-provider-local/src/services/read-model-registry.ts index c2e578497..68d9ff8dd 100644 --- a/packages/framework-provider-local/src/services/read-model-registry.ts +++ b/packages/framework-provider-local/src/services/read-model-registry.ts @@ -7,10 +7,6 @@ interface LocalSortedFor { [key: string]: number } -interface LocalSelectFor { - [key: string]: number -} - export type NedbError = Error & { [key: string | number | symbol]: unknown } export const UNIQUE_VIOLATED_ERROR_TYPE = 'uniqueViolated' @@ -39,7 +35,7 @@ export class ReadModelRegistry { select?: ProjectionFor ): Promise> { await this.loadDatabaseIfNeeded() - let cursor = this.readModels.find(query, this.toLocalSelectFor(select)) + let cursor = this.readModels.find(query) const sortByList = this.toLocalSortFor(sortBy) if (sortByList) { cursor = cursor.sort(sortByList) @@ -51,45 +47,15 @@ export class ReadModelRegistry { cursor = cursor.limit(limit) } - const arrayFields: { [key: string]: any } = {} - select?.forEach((field: string) => { - const parts = field.split('.') - let currentLevel = arrayFields - let isArrayField = false - for (let i = 0; i < parts.length; ++i) { - const part = parts[i] - if (part.endsWith('[]')) { - const arrayField = part.slice(0, -2) - if (!Object.prototype.hasOwnProperty.call(currentLevel, arrayField)) { - currentLevel[arrayField] = {} - } - currentLevel = currentLevel[arrayField] - isArrayField = true - } else { - if (i === parts.length - 1) { - if (isArrayField) { - if (!currentLevel['__fields']) { - currentLevel['__fields'] = [] - } - currentLevel['__fields'].push(part) - } - } else { - if (!Object.prototype.hasOwnProperty.call(currentLevel, part)) { - currentLevel[part] = {} - } - currentLevel = currentLevel[part] - } - } - } - }) - // Fetch results from the cursor const results = await cursor.execAsync() // Process each result to filter the array fields - results.forEach((result: any) => { - result.value = this.filterObjectByArrayFields(result.value, arrayFields) - }) + if (select && select.length > 0) { + results.forEach((result: any) => { + result.value = this.filterFields(result.value, select) + }) + } return results } @@ -139,7 +105,7 @@ export class ReadModelRegistry { toLocalSortFor( sortBy?: SortFor, parentKey = '', - sortedList: LocalSortedFor = {} + sortedList: LocalSortedFor = Object.create(null) ): undefined | LocalSortedFor { if (!sortBy || Object.keys(sortBy).length === 0) return const elements = sortBy! @@ -153,76 +119,59 @@ export class ReadModelRegistry { return sortedList } - toLocalSelectFor(select?: ProjectionFor): LocalSelectFor { - if (!select || select.length === 0) return {} + filterFields(obj: any, select: string[]): any { + const result: any = Object.create(null) - const result: LocalSelectFor = {} - const seenFields = new Set() - - select.forEach((field: string) => { + select.forEach((field) => { const parts = field.split('.') - const topLevelField = parts[0] - - if (topLevelField.endsWith('[]')) { - const arrayField = `value.${topLevelField.slice(0, -2)}` - if (!seenFields.has(arrayField)) { - seenFields.add(arrayField) - result[arrayField] = 1 - } - } else { - if (parts.some((part) => part.endsWith('[]'))) { - const arrayIndex = parts.findIndex((part) => part.endsWith('[]')) - const arrayField = `value.${parts - .slice(0, arrayIndex + 1) - .join('.') - .slice(0, -2)}` - if (!seenFields.has(arrayField)) { - seenFields.add(arrayField) - result[arrayField] = 1 - } - } else { - const fullPath = `value.${field}` - if (!seenFields.has(fullPath)) { - seenFields.add(fullPath) - result[fullPath] = 1 - } - } - } + this.setNestedValue(result, obj, parts) }) return result } - filterArrayFields(item: any, fields: { [key: string]: any; __fields?: string[] }): any { - const filteredItem: { [key: string]: any } = {} - if (fields.__fields) { - fields.__fields.forEach((field) => { - if (field in item) { - filteredItem[field] = item[field] - } - }) - } - Object.keys(fields).forEach((key) => { - if (key !== '__fields' && item[key] && Array.isArray(item[key])) { - filteredItem[key] = item[key].map((subItem: any) => this.filterArrayFields(subItem, fields[key])) - } - }) - return filteredItem - } + setNestedValue(result: any, source: any, parts: string[]): void { + let currentResult = result + let currentSource = source + + for (let i = 0; i < parts.length; i++) { + const part = parts[i] + const isLast = i === parts.length - 1 - filterObjectByArrayFields(obj: any, arrayFields: { [key: string]: any; __fields?: string[] }): any { - const filteredObj: { [key: string]: any } = {} - Object.keys(obj).forEach((key) => { - if (key in arrayFields) { - if (Array.isArray(obj[key])) { - filteredObj[key] = obj[key].map((item: any) => this.filterArrayFields(item, arrayFields[key])) + if (part.endsWith('[]')) { + const arrayField = part.slice(0, -2) + if (!Array.isArray(currentSource[arrayField])) { + return + } + if (!currentResult[arrayField]) { + currentResult[arrayField] = [] + } + if (isLast) { + currentResult[arrayField] = currentSource[arrayField] } else { - filteredObj[key] = this.filterObjectByArrayFields(obj[key], arrayFields[key]) + currentSource[arrayField].forEach((item: any, index: number) => { + if (!currentResult[arrayField][index]) { + currentResult[arrayField][index] = Object.create(null) + } + this.setNestedValue(currentResult[arrayField][index], item, parts.slice(i + 1)) + }) } } else { - filteredObj[key] = obj[key] + if (isLast) { + if (currentSource[part] !== undefined) { + currentResult[part] = currentSource[part] + } + } else { + if (!currentSource[part]) { + return + } + if (!currentResult[part]) { + currentResult[part] = Array.isArray(currentSource[part]) ? [] : Object.create(null) + } + currentResult = currentResult[part] + currentSource = currentSource[part] + } } - }) - return filteredObj + } } } diff --git a/packages/framework-provider-local/test/services/read-model-registry.test.ts b/packages/framework-provider-local/test/services/read-model-registry.test.ts index dbdadfda1..0c73e9500 100644 --- a/packages/framework-provider-local/test/services/read-model-registry.test.ts +++ b/packages/framework-provider-local/test/services/read-model-registry.test.ts @@ -225,6 +225,74 @@ describe('the read model registry', () => { } expect(result[0]).to.deep.include(expectedReadModel) }) + + it('should return only projected fields for complex read models', async () => { + const complexReadModel: ReadModelEnvelope = { + typeName: random.word(), + value: { + id: random.uuid(), + x: { + arr: [{ y: random.word(), z: random.number() }], + }, + foo: { + bar: { + items: [{ id: random.uuid(), name: random.word() }], + baz: { items: [{ id: random.uuid(), name: random.word() }] }, + }, + }, + arr: [ + { + id: random.uuid(), + subArr: [{ id: random.uuid(), name: random.word() }], + }, + ], + boosterMetadata: { + version: 1, + schemaVersion: 1, + }, + }, + } + + await readModelRegistry.store(complexReadModel, 1) + + const result = await readModelRegistry.query( + { + value: complexReadModel.value, + typeName: complexReadModel.typeName, + }, + undefined, + undefined, + undefined, + [ + 'id', + 'x.arr[].z', + 'foo.bar.items[].id', + 'foo.bar.baz.items[].id', + 'arr[].subArr[].id', + 'arr[].id', + ] as ProjectionFor + ) + + expect(result.length).to.be.equal(1) + const expectedReadModel = { + value: { + id: complexReadModel.value.id, + x: { + arr: complexReadModel.value.x.arr.map((item: any) => ({ z: item.z })), + }, + foo: { + bar: { + items: complexReadModel.value.foo.bar.items.map((item: any) => ({ id: item.id })), + baz: { items: complexReadModel.value.foo.bar.baz.items.map((item: any) => ({ id: item.id })) }, + }, + }, + arr: complexReadModel.value.arr.map((item: any) => { + return { id: item.id, subArr: item.subArr.map((subItem: any) => ({ id: subItem.id })) } + }), + }, + } + expect(result[0]).to.deep.include(expectedReadModel) + }) }) describe('delete by id', () => {