From c0ef3dbfbe3536d583d695813edabc272659dbcb Mon Sep 17 00:00:00 2001 From: Daneryl Date: Thu, 22 Aug 2024 17:02:37 +0200 Subject: [PATCH 01/26] WIP, fixtures rework for ix filters and aggregations --- .../suggestions/specs/customFilters.spec.ts | 188 +++++++++++++++++- app/api/utils/fixturesFactory.ts | 20 +- 2 files changed, 201 insertions(+), 7 deletions(-) diff --git a/app/api/suggestions/specs/customFilters.spec.ts b/app/api/suggestions/specs/customFilters.spec.ts index bd8fc83228..4b168afcf5 100644 --- a/app/api/suggestions/specs/customFilters.spec.ts +++ b/app/api/suggestions/specs/customFilters.spec.ts @@ -1,7 +1,8 @@ -import db from 'api/utils/testing_db'; -import { SuggestionCustomFilter } from 'shared/types/suggestionType'; +import db, { DBFixture } from 'api/utils/testing_db'; +import { IXSuggestionType, SuggestionCustomFilter } from 'shared/types/suggestionType'; import { factory, stateFilterFixtures } from './fixtures'; import { Suggestions } from '../suggestions'; +import { getIdMapper } from 'api/utils/fixturesFactory'; const blankCustomFilter: SuggestionCustomFilter = { labeled: { @@ -241,7 +242,190 @@ describe('suggestions with CustomFilters', () => { }); describe('aggreagate()', () => { + const suggestion = (fileId: string, suggestedValue: string) => { + const file = stateFilterFixtures.files.find( + f => f._id?.toString() === factory.idString(fileId) + ); + const entity = stateFilterFixtures.entities.find(e => { + return e.sharedId === file?.entity; + }); + return { + _id: factory.id(`suggestion_for_${fileId}`), + entityId: file.entity, + status: 'ready' as const, + entityTemplate: entity?.template, + language: file?.language, + fileId: factory.id(fileId), + propertyName: Object.keys(entity.metadata)[0], + extractorId: factory.id('test_extractor'), + error: '', + segment: '', + suggestedValue: suggestedValue, + date: 1001, + state: { + labeled: false, + withValue: true, + withSuggestion: false, + match: false, + hasContext: false, + obsolete: false, + processing: false, + error: false, + }, + } as IXSuggestionType; + }; + + // fit('should return count of labeled and non labeled suggestions', async () => { + // stateFilterFixtures.entities = [ + // ...factory.entityInMultipleLanguages(['es', 'en'], 'labeled-match', 'template1', { + // testprop: [{ value: 'test-labeled-match' }], + // }), + // ...factory.entityInMultipleLanguages(['es', 'en'], 'unlabeled-no-suggestion', 'template1', { + // testprop: [{ value: 'test-unlabeled-no-suggestion' }], + // }), + // ]; + // stateFilterFixtures.files = [ + // factory.document('labeled-match', { + // extractedMetadata: [factory.fileExtractedMetadata('testprop', 'labeled-value')], + // }), + // factory.document('unlabeled-no-suggestion'), + // ]; + // await db.setupFixturesAndContext({ + // ...stateFilterFixtures, + // ixsuggestions: [ + // suggestion('document_for_labeled-match', 'test-labeled-match'), + // suggestion('document_for_unlabeled-no-suggestion', 'test-labeled-match'), + // ], + // }); + // + // await Suggestions.updateStates({}); + // + // const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); + // expect(result).toMatchObject({ + // labeled: { + // _count: 1, + // }, + // nonLabeled: { + // _count: 1, + // }, + // }); + // }); + + const nFactory = (dbFixtures: DBFixture) => { + return { + suggestion(props: Partial): IXSuggestionType { + return { + + } + }, + }; + }; + + fit('should return count of match and missmatch', async () => { + //in this scenario only match should matter for the aggregation, nothing else + await db.setupFixturesAndContext({ + ixsuggestions: [ + { extractorId: factory.id('test_extractor'), state: { match: true } }, + { extractorId: factory.id('test_extractor'), state: { match: false } }, + ], + }); + + const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); + expect(result).toMatchObject({ + labeled: { + match: 1, + mismatch: 1, + }, + }); + }); + + fit('should return count of labeled and non labeled suggestions', async () => { + //in this scenario match should not be necessary to pass the test + await db.setupFixturesAndContext({ + ixsuggestions: [ + { extractorId: factory.id('test_extractor'), state: { match: false, labeled: true } }, + { extractorId: factory.id('test_extractor'), state: { labeled: false } }, + ], + }); + + const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); + expect(result).toMatchObject({ + labeled: { + _count: 1, + }, + nonLabeled: { + _count: 1, + }, + }); + }); + + fit('should return count of obsolete suggestions', async () => { + //in this scenatio we need to only filter by obsolete and not by labeled + await db.setupFixturesAndContext({ + ixsuggestions: [ + { extractorId: factory.id('test_extractor'), state: { labeled: false, obsolete: true } }, + { extractorId: factory.id('test_extractor'), state: { labeled: false } }, + ], + }); + + const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); + expect(result).toMatchObject({ + nonLabeled: { + obsolete: 1, + }, + }); + }); + it('should return correct aggregation', async () => { + // const newFactory = {}; + // newFactory.suggestion( + // newFactory.file(newFactory.entity('entity_id', { prop: { value: 'test_value' } })), + // 'suggested_value' + // ); + // + await db.setupFixturesAndContext({ + ...stateFilterFixtures, + ixsuggestions: [ + suggestion('label-match-file-en', 'test-labeled-match'), + suggestion({ file: 'label-match-file-en', suggestionValue: 'test-labeled-match' }), + suggestion('label-match-file-es', 'test-labeled-match'), + suggestion('label-mismatch-file-en', 'test-labeled-mismatch-mismatch'), + suggestion('label-mismatch-file-es', 'test-labeled-mismatch-mismatch'), + suggestion('unlabeled-no-suggestion-file-en', ''), + suggestion('unlabeled-no-suggestion-file-es', ''), + suggestion('unlabeled-no-context-file-en', 'test-unlabeled-no-context'), + suggestion('unlabeled-no-context-file-es', 'test-unlabeled-no-context'), + { + ...suggestion('unlabeled-obsolete-file-en', 'test-unlabeled-obsolete'), + segment: 'test-unlabeled-obsolete', // no context aggregation + date: 0, // obsolete aggregation + }, + { + ...suggestion('unlabeled-obsolete-file-es', 'test-unlabeled-obsolete'), + segment: 'test-unlabeled-obsolete', // no context aggregation + date: 0, // obsolete aggregation + }, + { + ...suggestion('unlabeled-processing-file-en', 'test-unlabeled-processing'), + status: 'processing', // withSuggestion aggregation should not have processing ones + }, + { + ...suggestion('unlabeled-processing-file-es', 'test-unlabeled-processing'), + status: 'processing', // withSuggestion aggregation should not have processing ones + }, + { + ...suggestion('unlabeled-error-file-en', 'test-unlabeled-error'), + segment: 'test-unlabeled-error', // no context aggregation + status: 'failed', + }, + { + ...suggestion('unlabeled-error-file-es', 'test-unlabeled-error'), + segment: 'test-unlabeled-error', // no context aggregation + status: 'failed', + }, + ], + }); + await Suggestions.updateStates({}); const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); expect(result).toEqual({ total: 12, diff --git a/app/api/utils/fixturesFactory.ts b/app/api/utils/fixturesFactory.ts index 8bd83fcc83..15b01aa27e 100644 --- a/app/api/utils/fixturesFactory.ts +++ b/app/api/utils/fixturesFactory.ts @@ -129,8 +129,8 @@ function getFixturesFactory() { defaultProps: EntitySchema = {}, propsPerLanguage: | { - [key: string]: EntitySchema; - } + [key: string]: EntitySchema; + } | undefined = undefined ): EntitySchema[] { return languages.map(language => { @@ -161,6 +161,16 @@ function getFixturesFactory() { }, }), + document: (entity: string, props: Partial> = {}): WithId => ({ + _id: idMapper(`document_for_${entity}`), + entity, + language: 'en', + type: 'document', + filename: `${entity}_document.pdf`, + originalname: `${entity}_document.pdf`, + ...props, + }), + file: ( id: string, entity: string | undefined, @@ -170,7 +180,7 @@ function getFixturesFactory() { originalname: string | undefined = undefined, extractedMetadata: ExtractedMetadataSchema[] = [] ): WithId => ({ - _id: idMapper(`${id}`), + _id: idMapper(id), entity, language, type, @@ -235,8 +245,8 @@ function getFixturesFactory() { typeof item === 'string' ? [{ _id: idMapper(item), id: item, label: item }] : Object.entries(item).map(([rootValue, children]) => - thesaurusNestedValues(rootValue, children, idMapper) - ); + thesaurusNestedValues(rootValue, children, idMapper) + ); return [...accumulator, ...nestedItems]; }, [] From a60b08ecbccd7fab8f5a33936fbb0bf7350628d6 Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Fri, 23 Aug 2024 08:58:53 -0500 Subject: [PATCH 02/26] Begun flattening aggregations response --- .../suggestions/specs/customFilters.spec.ts | 117 ++++++++---------- app/api/suggestions/suggestions.ts | 57 ++++----- 2 files changed, 76 insertions(+), 98 deletions(-) diff --git a/app/api/suggestions/specs/customFilters.spec.ts b/app/api/suggestions/specs/customFilters.spec.ts index 4b168afcf5..3bf8cb4bf2 100644 --- a/app/api/suggestions/specs/customFilters.spec.ts +++ b/app/api/suggestions/specs/customFilters.spec.ts @@ -1,8 +1,8 @@ -import db, { DBFixture } from 'api/utils/testing_db'; +import db, { DBFixture, testingDB } from 'api/utils/testing_db'; import { IXSuggestionType, SuggestionCustomFilter } from 'shared/types/suggestionType'; +import { getIdMapper } from 'api/utils/fixturesFactory'; import { factory, stateFilterFixtures } from './fixtures'; import { Suggestions } from '../suggestions'; -import { getIdMapper } from 'api/utils/fixturesFactory'; const blankCustomFilter: SuggestionCustomFilter = { labeled: { @@ -246,9 +246,7 @@ describe('suggestions with CustomFilters', () => { const file = stateFilterFixtures.files.find( f => f._id?.toString() === factory.idString(fileId) ); - const entity = stateFilterFixtures.entities.find(e => { - return e.sharedId === file?.entity; - }); + const entity = stateFilterFixtures.entities.find(e => e.sharedId === file?.entity); return { _id: factory.id(`suggestion_for_${fileId}`), entityId: file.entity, @@ -260,7 +258,7 @@ describe('suggestions with CustomFilters', () => { extractorId: factory.id('test_extractor'), error: '', segment: '', - suggestedValue: suggestedValue, + suggestedValue, date: 1001, state: { labeled: false, @@ -275,91 +273,82 @@ describe('suggestions with CustomFilters', () => { } as IXSuggestionType; }; - // fit('should return count of labeled and non labeled suggestions', async () => { - // stateFilterFixtures.entities = [ - // ...factory.entityInMultipleLanguages(['es', 'en'], 'labeled-match', 'template1', { - // testprop: [{ value: 'test-labeled-match' }], - // }), - // ...factory.entityInMultipleLanguages(['es', 'en'], 'unlabeled-no-suggestion', 'template1', { - // testprop: [{ value: 'test-unlabeled-no-suggestion' }], - // }), - // ]; - // stateFilterFixtures.files = [ - // factory.document('labeled-match', { - // extractedMetadata: [factory.fileExtractedMetadata('testprop', 'labeled-value')], - // }), - // factory.document('unlabeled-no-suggestion'), - // ]; - // await db.setupFixturesAndContext({ - // ...stateFilterFixtures, - // ixsuggestions: [ - // suggestion('document_for_labeled-match', 'test-labeled-match'), - // suggestion('document_for_unlabeled-no-suggestion', 'test-labeled-match'), - // ], - // }); - // - // await Suggestions.updateStates({}); - // - // const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); - // expect(result).toMatchObject({ - // labeled: { - // _count: 1, - // }, - // nonLabeled: { - // _count: 1, - // }, - // }); - // }); + type PartialSuggestion = Partial> & { + state?: Partial; + }; - const nFactory = (dbFixtures: DBFixture) => { - return { - suggestion(props: Partial): IXSuggestionType { - return { + const nFactory = () => ({ + suggestion(props: PartialSuggestion): IXSuggestionType { + const { state, ...otherProps } = props; - } - }, - }; - }; + return { + _id: testingDB.id(), + entityId: testingDB.id().toString(), + status: 'ready' as const, + entityTemplate: testingDB.id().toString(), + language: 'en', + fileId: testingDB.id().toString(), + propertyName: 'propertyName', + extractorId: testingDB.id(), + error: '', + segment: '', + suggestedValue: '', + date: 1001, + state: { + labeled: false, + withValue: true, + withSuggestion: false, + match: false, + hasContext: false, + obsolete: false, + processing: false, + error: false, + ...state, + }, + ...otherProps, + }; + }, + }); fit('should return count of match and missmatch', async () => { + const f = nFactory(); //in this scenario only match should matter for the aggregation, nothing else await db.setupFixturesAndContext({ ixsuggestions: [ - { extractorId: factory.id('test_extractor'), state: { match: true } }, - { extractorId: factory.id('test_extractor'), state: { match: false } }, + f.suggestion({ extractorId: factory.id('test_extractor'), state: { match: true } }), + f.suggestion({ extractorId: factory.id('another_extractor'), state: { match: true } }), + f.suggestion({ extractorId: factory.id('test_extractor'), state: { match: true } }), + f.suggestion({ extractorId: factory.id('test_extractor'), state: { match: false } }), ], }); const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); expect(result).toMatchObject({ - labeled: { - match: 1, - mismatch: 1, - }, + match: 2, + mismatch: 1, }); }); fit('should return count of labeled and non labeled suggestions', async () => { + const f = nFactory(); //in this scenario match should not be necessary to pass the test await db.setupFixturesAndContext({ ixsuggestions: [ - { extractorId: factory.id('test_extractor'), state: { match: false, labeled: true } }, - { extractorId: factory.id('test_extractor'), state: { labeled: false } }, + f.suggestion({ extractorId: factory.id('test_extractor'), state: { labeled: true } }), + f.suggestion({ extractorId: factory.id('another_extractor'), state: { labeled: true } }), + f.suggestion({ extractorId: factory.id('test_extractor'), state: { labeled: false } }), + f.suggestion({ extractorId: factory.id('test_extractor'), state: { labeled: false } }), ], }); const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); expect(result).toMatchObject({ - labeled: { - _count: 1, - }, - nonLabeled: { - _count: 1, - }, + labeled: 1, + nonLabeled: 2, }); }); - fit('should return count of obsolete suggestions', async () => { + it('should return count of obsolete suggestions', async () => { //in this scenatio we need to only filter by obsolete and not by labeled await db.setupFixturesAndContext({ ixsuggestions: [ diff --git a/app/api/suggestions/suggestions.ts b/app/api/suggestions/suggestions.ts index 05db7b7ab8..cb5e193532 100644 --- a/app/api/suggestions/suggestions.ts +++ b/app/api/suggestions/suggestions.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-lines */ import { ObjectId } from 'mongodb'; import { files } from 'api/files/files'; @@ -137,23 +138,27 @@ const buildListQuery = ( }; async function getLabeledCounts(extractorId: ObjectId) { - const labeledAggregationQuery = [ + const labeledQuery = { extractorId, 'state.labeled': true }; + const labeledCount = await IXSuggestionsModel.count(labeledQuery); + return { labeled: labeledCount }; +} + +async function getAggregationCounts(extractorId: ObjectId) { + const aggregationQuery = [ { $match: { ...baseQueryFragment(extractorId), - ...filterFragments.labeled._fragment, }, }, ...groupByAndCount('$state.match'), ]; - const labeledAggregation: { _id: boolean; count: number }[] = - await IXSuggestionsModel.db.aggregate(labeledAggregationQuery); - const matchCount = - labeledAggregation.find((aggregation: any) => aggregation._id === true)?.count || 0; - const mismatchCount = - labeledAggregation.find((aggregation: any) => aggregation._id === false)?.count || 0; - const labeledCount = matchCount + mismatchCount; - return { labeledCount, matchCount, mismatchCount }; + const matchedMismatchedAggregation: { _id: boolean; count: number }[] = + await IXSuggestionsModel.db.aggregate(aggregationQuery); + const match = + matchedMismatchedAggregation.find((aggregation: any) => aggregation._id === true)?.count || 0; + const mismatch = + matchedMismatchedAggregation.find((aggregation: any) => aggregation._id === false)?.count || 0; + return { match, mismatch }; } const getNonLabeledCounts = async (_extractorId: ObjectId) => { @@ -268,33 +273,17 @@ const Suggestions = { }; }, - aggregate: async (_extractorId: ObjectIdSchema): Promise => { + aggregate: async (_extractorId: ObjectIdSchema) => { const extractorId = new ObjectId(_extractorId); - const { labeledCount, matchCount, mismatchCount } = await getLabeledCounts(extractorId); - const { - nonLabeledCount, - noContextCount, - withSuggestionCount, - noSuggestionCount, - obsoleteCount, - othersCount, - } = await getNonLabeledCounts(extractorId); - const totalCount = labeledCount + nonLabeledCount; + const aggregations = await getAggregationCounts(extractorId); + const { labeled } = await getLabeledCounts(extractorId); + const { nonLabeledCount } = await getNonLabeledCounts(extractorId); + const totalCount = labeled + nonLabeledCount; return { total: totalCount, - labeled: { - _count: labeledCount, - match: matchCount, - mismatch: mismatchCount, - }, - nonLabeled: { - _count: nonLabeledCount, - noContext: noContextCount, - withSuggestion: withSuggestionCount, - noSuggestion: noSuggestionCount, - obsolete: obsoleteCount, - others: othersCount, - }, + ...aggregations, + labeled, + nonLabeled: nonLabeledCount, }; }, From 467f36660c5a7e7634a411383b922f7b81cbcf1a Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Tue, 3 Sep 2024 18:23:33 -0500 Subject: [PATCH 03/26] Flattened the Aggregations return --- .../suggestions/specs/customFilters.spec.ts | 101 ++++------------ app/api/suggestions/suggestions.ts | 111 ++++++++---------- app/shared/types/suggestionSchema.ts | 36 ++---- app/shared/types/suggestionType.d.ts | 19 +-- 4 files changed, 89 insertions(+), 178 deletions(-) diff --git a/app/api/suggestions/specs/customFilters.spec.ts b/app/api/suggestions/specs/customFilters.spec.ts index 3bf8cb4bf2..407c3c31d0 100644 --- a/app/api/suggestions/specs/customFilters.spec.ts +++ b/app/api/suggestions/specs/customFilters.spec.ts @@ -243,18 +243,19 @@ describe('suggestions with CustomFilters', () => { describe('aggreagate()', () => { const suggestion = (fileId: string, suggestedValue: string) => { - const file = stateFilterFixtures.files.find( + const file = stateFilterFixtures.files?.find( f => f._id?.toString() === factory.idString(fileId) ); - const entity = stateFilterFixtures.entities.find(e => e.sharedId === file?.entity); + const entity = stateFilterFixtures.entities?.find(e => e.sharedId === file?.entity); + return { _id: factory.id(`suggestion_for_${fileId}`), - entityId: file.entity, + entityId: file?.entity, status: 'ready' as const, entityTemplate: entity?.template, language: file?.language, fileId: factory.id(fileId), - propertyName: Object.keys(entity.metadata)[0], + propertyName: Object.keys(entity?.metadata || {})[0], extractorId: factory.id('test_extractor'), error: '', segment: '', @@ -310,9 +311,9 @@ describe('suggestions with CustomFilters', () => { }, }); - fit('should return count of match and missmatch', async () => { + it('should return count of match and missmatch', async () => { const f = nFactory(); - //in this scenario only match should matter for the aggregation, nothing else + await db.setupFixturesAndContext({ ixsuggestions: [ f.suggestion({ extractorId: factory.id('test_extractor'), state: { match: true } }), @@ -329,9 +330,9 @@ describe('suggestions with CustomFilters', () => { }); }); - fit('should return count of labeled and non labeled suggestions', async () => { + it('should return count of labeled and non labeled suggestions', async () => { const f = nFactory(); - //in this scenario match should not be necessary to pass the test + await db.setupFixturesAndContext({ ixsuggestions: [ f.suggestion({ extractorId: factory.id('test_extractor'), state: { labeled: true } }), @@ -349,88 +350,38 @@ describe('suggestions with CustomFilters', () => { }); it('should return count of obsolete suggestions', async () => { - //in this scenatio we need to only filter by obsolete and not by labeled + const f = nFactory(); + await db.setupFixturesAndContext({ ixsuggestions: [ - { extractorId: factory.id('test_extractor'), state: { labeled: false, obsolete: true } }, - { extractorId: factory.id('test_extractor'), state: { labeled: false } }, + f.suggestion({ extractorId: factory.id('test_extractor'), state: { obsolete: true } }), + f.suggestion({ extractorId: factory.id('test_extractor'), state: { obsolete: true } }), + f.suggestion({ extractorId: factory.id('another_extractor'), state: { obsolete: true } }), + f.suggestion({ extractorId: factory.id('test_extractor'), state: { obsolete: false } }), + f.suggestion({ extractorId: factory.id('test_extractor'), state: {} }), ], }); const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); expect(result).toMatchObject({ - nonLabeled: { - obsolete: 1, - }, + obsolete: 2, }); }); - it('should return correct aggregation', async () => { - // const newFactory = {}; - // newFactory.suggestion( - // newFactory.file(newFactory.entity('entity_id', { prop: { value: 'test_value' } })), - // 'suggested_value' - // ); - // + it('should return count of errors in suggestions', async () => { + const f = nFactory(); + await db.setupFixturesAndContext({ - ...stateFilterFixtures, ixsuggestions: [ - suggestion('label-match-file-en', 'test-labeled-match'), - suggestion({ file: 'label-match-file-en', suggestionValue: 'test-labeled-match' }), - suggestion('label-match-file-es', 'test-labeled-match'), - suggestion('label-mismatch-file-en', 'test-labeled-mismatch-mismatch'), - suggestion('label-mismatch-file-es', 'test-labeled-mismatch-mismatch'), - suggestion('unlabeled-no-suggestion-file-en', ''), - suggestion('unlabeled-no-suggestion-file-es', ''), - suggestion('unlabeled-no-context-file-en', 'test-unlabeled-no-context'), - suggestion('unlabeled-no-context-file-es', 'test-unlabeled-no-context'), - { - ...suggestion('unlabeled-obsolete-file-en', 'test-unlabeled-obsolete'), - segment: 'test-unlabeled-obsolete', // no context aggregation - date: 0, // obsolete aggregation - }, - { - ...suggestion('unlabeled-obsolete-file-es', 'test-unlabeled-obsolete'), - segment: 'test-unlabeled-obsolete', // no context aggregation - date: 0, // obsolete aggregation - }, - { - ...suggestion('unlabeled-processing-file-en', 'test-unlabeled-processing'), - status: 'processing', // withSuggestion aggregation should not have processing ones - }, - { - ...suggestion('unlabeled-processing-file-es', 'test-unlabeled-processing'), - status: 'processing', // withSuggestion aggregation should not have processing ones - }, - { - ...suggestion('unlabeled-error-file-en', 'test-unlabeled-error'), - segment: 'test-unlabeled-error', // no context aggregation - status: 'failed', - }, - { - ...suggestion('unlabeled-error-file-es', 'test-unlabeled-error'), - segment: 'test-unlabeled-error', // no context aggregation - status: 'failed', - }, + f.suggestion({ extractorId: factory.id('test_extractor'), state: { error: true } }), + f.suggestion({ extractorId: factory.id('another_extractor'), state: { error: true } }), + f.suggestion({ extractorId: factory.id('test_extractor'), state: {} }), ], }); - await Suggestions.updateStates({}); + const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); - expect(result).toEqual({ - total: 12, - labeled: { - _count: 4, - match: 2, - mismatch: 2, - }, - nonLabeled: { - _count: 8, - withSuggestion: 6, - noSuggestion: 2, - noContext: 4, - obsolete: 2, - others: 2, - }, + expect(result).toMatchObject({ + error: 1, }); }); }); diff --git a/app/api/suggestions/suggestions.ts b/app/api/suggestions/suggestions.ts index cb5e193532..302f672e1f 100644 --- a/app/api/suggestions/suggestions.ts +++ b/app/api/suggestions/suggestions.ts @@ -30,7 +30,6 @@ import { Extractors } from 'api/services/informationextraction/ixextractors'; import { registerEventListeners } from './eventListeners'; import { baseQueryFragment, - filterFragments, getCurrentValueStage, getEntityStage, getFileStage, @@ -137,67 +136,44 @@ const buildListQuery = ( return pipeline; }; -async function getLabeledCounts(extractorId: ObjectId) { - const labeledQuery = { extractorId, 'state.labeled': true }; - const labeledCount = await IXSuggestionsModel.count(labeledQuery); - return { labeled: labeledCount }; -} +type CountAggregation = { _id: boolean; count: number }[]; -async function getAggregationCounts(extractorId: ObjectId) { - const aggregationQuery = [ - { - $match: { - ...baseQueryFragment(extractorId), - }, - }, +const getStatePropertyCount = async ( + extractorId: ObjectId, + property: string, + value: boolean = true +) => { + const query = { extractorId, [`state.${property}`]: value }; + return IXSuggestionsModel.count(query); +}; + +const getLabeledCount = async (extractorId: ObjectId) => ({ + labeled: await getStatePropertyCount(extractorId, 'labeled'), +}); + +const getNonLabeledCount = async (extractorId: ObjectId) => ({ + nonLabeled: await getStatePropertyCount(extractorId, 'labeled', false), +}); + +const getMatchedCount = async (extractorId: ObjectId) => { + const query = [ + { $match: { ...baseQueryFragment(extractorId) } }, ...groupByAndCount('$state.match'), ]; - const matchedMismatchedAggregation: { _id: boolean; count: number }[] = - await IXSuggestionsModel.db.aggregate(aggregationQuery); - const match = - matchedMismatchedAggregation.find((aggregation: any) => aggregation._id === true)?.count || 0; - const mismatch = - matchedMismatchedAggregation.find((aggregation: any) => aggregation._id === false)?.count || 0; - return { match, mismatch }; -} - -const getNonLabeledCounts = async (_extractorId: ObjectId) => { - const extractorId = new ObjectId(_extractorId); - const unlabeledMatch = { - ...baseQueryFragment(extractorId), - ...filterFragments.nonLabeled._fragment, - }; - const nonLabeledCount = await IXSuggestionsModel.count(unlabeledMatch); - const noContextCount = await IXSuggestionsModel.count({ - ...unlabeledMatch, - ...filterFragments.nonLabeled.noContext, - }); - const withSuggestionCount = await IXSuggestionsModel.count({ - ...unlabeledMatch, - ...filterFragments.nonLabeled.withSuggestion, - }); - const noSuggestionCount = await IXSuggestionsModel.count({ - ...unlabeledMatch, - ...filterFragments.nonLabeled.noSuggestion, - }); - const obsoleteCount = await IXSuggestionsModel.count({ - ...unlabeledMatch, - ...filterFragments.nonLabeled.obsolete, - }); - const othersCount = await IXSuggestionsModel.count({ - ...unlabeledMatch, - ...filterFragments.nonLabeled.others, - }); - return { - nonLabeledCount, - noContextCount, - withSuggestionCount, - noSuggestionCount, - obsoleteCount, - othersCount, - }; + const aggregation: CountAggregation = await IXSuggestionsModel.db.aggregate(query); + const matchCount = aggregation.find((a: any) => a._id === true)?.count || 0; + const mismatchCount = aggregation.find((a: any) => a._id === false)?.count || 0; + return { match: matchCount, mismatch: mismatchCount }; }; +const getObsoleteCount = async (extractorId: ObjectId) => ({ + obsolete: await getStatePropertyCount(extractorId, 'obsolete'), +}); + +const getErrorCount = async (extractorId: ObjectId) => ({ + error: await getStatePropertyCount(extractorId, 'error'), +}); + const readFilter = (filter: IXSuggestionsFilter) => { const { customFilter, extractorId: _extractorId } = filter; const extractorId = new ObjectId(_extractorId); @@ -273,17 +249,24 @@ const Suggestions = { }; }, - aggregate: async (_extractorId: ObjectIdSchema) => { + aggregate: async (_extractorId: ObjectIdSchema): Promise => { const extractorId = new ObjectId(_extractorId); - const aggregations = await getAggregationCounts(extractorId); - const { labeled } = await getLabeledCounts(extractorId); - const { nonLabeledCount } = await getNonLabeledCounts(extractorId); - const totalCount = labeled + nonLabeledCount; + const { labeled } = await getLabeledCount(extractorId); + const { nonLabeled } = await getNonLabeledCount(extractorId); + const { match, mismatch } = await getMatchedCount(extractorId); + const { obsolete } = await getObsoleteCount(extractorId); + const { error } = await getErrorCount(extractorId); + + const totalCount = labeled + nonLabeled; + return { total: totalCount, - ...aggregations, labeled, - nonLabeled: nonLabeledCount, + nonLabeled, + match, + mismatch, + obsolete, + error, }; }, diff --git a/app/shared/types/suggestionSchema.ts b/app/shared/types/suggestionSchema.ts index 8a0f80f393..a019afdd11 100644 --- a/app/shared/types/suggestionSchema.ts +++ b/app/shared/types/suggestionSchema.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-lines */ import { objectIdSchema, propertyValueSchema, @@ -5,14 +6,14 @@ import { } from 'shared/types/commonSchemas'; import { propertyTypes } from 'shared/propertyTypes'; -export const emitSchemaTypes = true; - const commonSuggestionMessageProperties = { tenant: { type: 'string', minLength: 1 }, id: { type: 'string', minLength: 1 }, xml_file_name: { type: 'string', minLength: 1 }, }; +export const emitSchemaTypes = true; + export const CommonSuggestionSchema = { type: 'object', title: 'CommonSuggestion', @@ -285,31 +286,14 @@ export const IXSuggestionAggregationSchema = { type: 'object', title: 'IXSuggestionAggregation', additionalProperties: false, - required: ['labeled', 'nonLabeled', 'total'], + required: ['total', 'labeled', 'nonLabeled', 'match', 'mismatch', 'obsolete', 'error'], properties: { total: { type: 'number' }, - labeled: { - type: 'object', - additionalProperties: false, - required: ['_count', 'match', 'mismatch'], - properties: { - _count: { type: 'number' }, - match: { type: 'number' }, - mismatch: { type: 'number' }, - }, - }, - nonLabeled: { - type: 'object', - additionalProperties: false, - required: ['_count', 'withSuggestion', 'noSuggestion', 'noContext', 'obsolete', 'others'], - properties: { - _count: { type: 'number' }, - withSuggestion: { type: 'number' }, - noSuggestion: { type: 'number' }, - noContext: { type: 'number' }, - obsolete: { type: 'number' }, - others: { type: 'number' }, - }, - }, + labeled: { type: 'number' }, + nonLabeled: { type: 'number' }, + match: { type: 'number' }, + mismatch: { type: 'number' }, + obsolete: { type: 'number' }, + error: { type: 'number' }, }, }; diff --git a/app/shared/types/suggestionType.d.ts b/app/shared/types/suggestionType.d.ts index 504cb24983..152511f554 100644 --- a/app/shared/types/suggestionType.d.ts +++ b/app/shared/types/suggestionType.d.ts @@ -47,19 +47,12 @@ export interface IXAggregationQuery { export interface IXSuggestionAggregation { total: number; - labeled: { - _count: number; - match: number; - mismatch: number; - }; - nonLabeled: { - _count: number; - withSuggestion: number; - noSuggestion: number; - noContext: number; - obsolete: number; - others: number; - }; + labeled: number; + nonLabeled: number; + match: number; + mismatch: number; + obsolete: number; + error: number; } export interface IXSuggestionType { From 72a81e38648de324d45fbe823bc0eebea4b0d4f3 Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Tue, 3 Sep 2024 19:11:41 -0500 Subject: [PATCH 04/26] Complete refactor to aggregation results --- .../suggestions/specs/customFilters.spec.ts | 49 +++++++---- app/api/suggestions/suggestions.ts | 83 +++++++------------ 2 files changed, 65 insertions(+), 67 deletions(-) diff --git a/app/api/suggestions/specs/customFilters.spec.ts b/app/api/suggestions/specs/customFilters.spec.ts index 407c3c31d0..ff0b1ee42a 100644 --- a/app/api/suggestions/specs/customFilters.spec.ts +++ b/app/api/suggestions/specs/customFilters.spec.ts @@ -311,41 +311,43 @@ describe('suggestions with CustomFilters', () => { }, }); - it('should return count of match and missmatch', async () => { + it('should return count of labeled and non labeled suggestions', async () => { const f = nFactory(); await db.setupFixturesAndContext({ ixsuggestions: [ - f.suggestion({ extractorId: factory.id('test_extractor'), state: { match: true } }), - f.suggestion({ extractorId: factory.id('another_extractor'), state: { match: true } }), - f.suggestion({ extractorId: factory.id('test_extractor'), state: { match: true } }), - f.suggestion({ extractorId: factory.id('test_extractor'), state: { match: false } }), + f.suggestion({ extractorId: factory.id('test_extractor'), state: { labeled: true } }), + f.suggestion({ extractorId: factory.id('another_extractor'), state: { labeled: true } }), + f.suggestion({ extractorId: factory.id('test_extractor'), state: { labeled: false } }), + f.suggestion({ extractorId: factory.id('test_extractor'), state: { labeled: false } }), ], }); const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); expect(result).toMatchObject({ - match: 2, - mismatch: 1, + total: 3, + labeled: 1, + nonLabeled: 2, }); }); - it('should return count of labeled and non labeled suggestions', async () => { + it('should return count of match and missmatch', async () => { const f = nFactory(); await db.setupFixturesAndContext({ ixsuggestions: [ - f.suggestion({ extractorId: factory.id('test_extractor'), state: { labeled: true } }), - f.suggestion({ extractorId: factory.id('another_extractor'), state: { labeled: true } }), - f.suggestion({ extractorId: factory.id('test_extractor'), state: { labeled: false } }), - f.suggestion({ extractorId: factory.id('test_extractor'), state: { labeled: false } }), + f.suggestion({ extractorId: factory.id('test_extractor'), state: { match: true } }), + f.suggestion({ extractorId: factory.id('another_extractor'), state: { match: true } }), + f.suggestion({ extractorId: factory.id('test_extractor'), state: { match: true } }), + f.suggestion({ extractorId: factory.id('test_extractor'), state: { match: false } }), ], }); const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); expect(result).toMatchObject({ - labeled: 1, - nonLabeled: 2, + total: 3, + match: 2, + mismatch: 1, }); }); @@ -364,6 +366,7 @@ describe('suggestions with CustomFilters', () => { const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); expect(result).toMatchObject({ + total: 4, obsolete: 2, }); }); @@ -381,8 +384,26 @@ describe('suggestions with CustomFilters', () => { const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); expect(result).toMatchObject({ + total: 2, error: 1, }); }); + + it('should correctly return all zeroes if no suggestions found', async () => { + await db.setupFixturesAndContext({ + ixsuggestions: [], + }); + + const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); + expect(result).toMatchObject({ + total: 0, + labeled: 0, + nonLabeled: 0, + match: 0, + mismatch: 0, + obsolete: 0, + error: 0, + }); + }); }); }); diff --git a/app/api/suggestions/suggestions.ts b/app/api/suggestions/suggestions.ts index 302f672e1f..82897cc6d3 100644 --- a/app/api/suggestions/suggestions.ts +++ b/app/api/suggestions/suggestions.ts @@ -136,44 +136,6 @@ const buildListQuery = ( return pipeline; }; -type CountAggregation = { _id: boolean; count: number }[]; - -const getStatePropertyCount = async ( - extractorId: ObjectId, - property: string, - value: boolean = true -) => { - const query = { extractorId, [`state.${property}`]: value }; - return IXSuggestionsModel.count(query); -}; - -const getLabeledCount = async (extractorId: ObjectId) => ({ - labeled: await getStatePropertyCount(extractorId, 'labeled'), -}); - -const getNonLabeledCount = async (extractorId: ObjectId) => ({ - nonLabeled: await getStatePropertyCount(extractorId, 'labeled', false), -}); - -const getMatchedCount = async (extractorId: ObjectId) => { - const query = [ - { $match: { ...baseQueryFragment(extractorId) } }, - ...groupByAndCount('$state.match'), - ]; - const aggregation: CountAggregation = await IXSuggestionsModel.db.aggregate(query); - const matchCount = aggregation.find((a: any) => a._id === true)?.count || 0; - const mismatchCount = aggregation.find((a: any) => a._id === false)?.count || 0; - return { match: matchCount, mismatch: mismatchCount }; -}; - -const getObsoleteCount = async (extractorId: ObjectId) => ({ - obsolete: await getStatePropertyCount(extractorId, 'obsolete'), -}); - -const getErrorCount = async (extractorId: ObjectId) => ({ - error: await getStatePropertyCount(extractorId, 'error'), -}); - const readFilter = (filter: IXSuggestionsFilter) => { const { customFilter, extractorId: _extractorId } = filter; const extractorId = new ObjectId(_extractorId); @@ -251,23 +213,38 @@ const Suggestions = { aggregate: async (_extractorId: ObjectIdSchema): Promise => { const extractorId = new ObjectId(_extractorId); - const { labeled } = await getLabeledCount(extractorId); - const { nonLabeled } = await getNonLabeledCount(extractorId); - const { match, mismatch } = await getMatchedCount(extractorId); - const { obsolete } = await getObsoleteCount(extractorId); - const { error } = await getErrorCount(extractorId); - - const totalCount = labeled + nonLabeled; - return { - total: totalCount, - labeled, - nonLabeled, - match, - mismatch, - obsolete, - error, + const aggregations: (IXSuggestionAggregation & { _id: ObjectId })[] = + await IXSuggestionsModel.db.aggregate([ + { + $match: { extractorId }, + }, + { + $group: { + _id: null, + total: { $sum: 1 }, + labeled: { $sum: { $cond: ['$state.labeled', 1, 0] } }, + nonLabeled: { $sum: { $cond: [{ $not: '$state.labeled' }, 1, 0] } }, + match: { $sum: { $cond: ['$state.match', 1, 0] } }, + mismatch: { $sum: { $cond: [{ $not: '$state.match' }, 1, 0] } }, + obsolete: { $sum: { $cond: ['$state.obsolete', 1, 0] } }, + error: { $sum: { $cond: ['$state.error', 1, 0] } }, + }, + }, + ]); + + const { _id, ...results } = aggregations[0] || { + _id: null, + total: 0, + labeled: 0, + nonLabeled: 0, + match: 0, + mismatch: 0, + obsolete: 0, + error: 0, }; + + return results; }, updateStates, From 57b66531cd8e9ec4697648c25c04ff6222ea5148 Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Wed, 4 Sep 2024 14:55:46 -0500 Subject: [PATCH 05/26] Removed unused variables --- app/api/suggestions/suggestions.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/api/suggestions/suggestions.ts b/app/api/suggestions/suggestions.ts index 82897cc6d3..41d5d6924f 100644 --- a/app/api/suggestions/suggestions.ts +++ b/app/api/suggestions/suggestions.ts @@ -29,13 +29,11 @@ import { import { Extractors } from 'api/services/informationextraction/ixextractors'; import { registerEventListeners } from './eventListeners'; import { - baseQueryFragment, getCurrentValueStage, getEntityStage, getFileStage, getLabeledValueStage, getMatchStage, - groupByAndCount, } from './pipelineStages'; import { postProcessCurrentValues, updateStates } from './updateState'; import { From 9182249ad7428024da8a52fd0769f6f6391363a4 Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Wed, 4 Sep 2024 14:58:54 -0500 Subject: [PATCH 06/26] Removed unnecessary code --- .../suggestions/specs/customFilters.spec.ts | 35 +------------------ 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/app/api/suggestions/specs/customFilters.spec.ts b/app/api/suggestions/specs/customFilters.spec.ts index ff0b1ee42a..4974b03ae4 100644 --- a/app/api/suggestions/specs/customFilters.spec.ts +++ b/app/api/suggestions/specs/customFilters.spec.ts @@ -1,6 +1,5 @@ -import db, { DBFixture, testingDB } from 'api/utils/testing_db'; +import db, { testingDB } from 'api/utils/testing_db'; import { IXSuggestionType, SuggestionCustomFilter } from 'shared/types/suggestionType'; -import { getIdMapper } from 'api/utils/fixturesFactory'; import { factory, stateFilterFixtures } from './fixtures'; import { Suggestions } from '../suggestions'; @@ -242,38 +241,6 @@ describe('suggestions with CustomFilters', () => { }); describe('aggreagate()', () => { - const suggestion = (fileId: string, suggestedValue: string) => { - const file = stateFilterFixtures.files?.find( - f => f._id?.toString() === factory.idString(fileId) - ); - const entity = stateFilterFixtures.entities?.find(e => e.sharedId === file?.entity); - - return { - _id: factory.id(`suggestion_for_${fileId}`), - entityId: file?.entity, - status: 'ready' as const, - entityTemplate: entity?.template, - language: file?.language, - fileId: factory.id(fileId), - propertyName: Object.keys(entity?.metadata || {})[0], - extractorId: factory.id('test_extractor'), - error: '', - segment: '', - suggestedValue, - date: 1001, - state: { - labeled: false, - withValue: true, - withSuggestion: false, - match: false, - hasContext: false, - obsolete: false, - processing: false, - error: false, - }, - } as IXSuggestionType; - }; - type PartialSuggestion = Partial> & { state?: Partial; }; From 31f90de8d0a9d1abdc409a532afeef77f523c6d2 Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Wed, 4 Sep 2024 14:59:19 -0500 Subject: [PATCH 07/26] Renamed as _deprecated previous factory implementation --- .../specs/ixextractors.spec.ts | 10 +++---- .../suggestions/specs/eventListeners.spec.ts | 12 ++++---- app/api/suggestions/specs/fixtures.ts | 30 +++++++++---------- app/api/utils/fixturesFactory.ts | 11 +++---- 4 files changed, 32 insertions(+), 31 deletions(-) diff --git a/app/api/services/informationextraction/specs/ixextractors.spec.ts b/app/api/services/informationextraction/specs/ixextractors.spec.ts index f4bcff15c5..c905ce1572 100644 --- a/app/api/services/informationextraction/specs/ixextractors.spec.ts +++ b/app/api/services/informationextraction/specs/ixextractors.spec.ts @@ -93,7 +93,7 @@ const fixtures: DBFixture = { fixtureFactory.ixExtractor('fungusKindExtractor', 'kind', ['fungusTemplate']), ], ixsuggestions: [ - fixtureFactory.ixSuggestion( + fixtureFactory.ixSuggestion_deprecated( 'sh1_en', 'existingExtractor', 'shared1', @@ -101,7 +101,7 @@ const fixtures: DBFixture = { 'F3', 'kind' ), - fixtureFactory.ixSuggestion( + fixtureFactory.ixSuggestion_deprecated( 'sh1_es', 'existingExtractor', 'shared1', @@ -110,7 +110,7 @@ const fixtures: DBFixture = { 'kind', { language: 'es' } ), - fixtureFactory.ixSuggestion( + fixtureFactory.ixSuggestion_deprecated( 'sh3_en', 'existingExtractor', 'shared3', @@ -118,7 +118,7 @@ const fixtures: DBFixture = { 'F5', 'kind' ), - fixtureFactory.ixSuggestion( + fixtureFactory.ixSuggestion_deprecated( 'sh4_en', 'fungusKindExtractor', 'shared4', @@ -126,7 +126,7 @@ const fixtures: DBFixture = { 'F7', 'kind' ), - fixtureFactory.ixSuggestion( + fixtureFactory.ixSuggestion_deprecated( 'sh4_es', 'fungusKindExtractor', 'shared4', diff --git a/app/api/suggestions/specs/eventListeners.spec.ts b/app/api/suggestions/specs/eventListeners.spec.ts index 8eecda80a8..2260996907 100644 --- a/app/api/suggestions/specs/eventListeners.spec.ts +++ b/app/api/suggestions/specs/eventListeners.spec.ts @@ -99,7 +99,7 @@ const fixtures: DBFixture = { fixturesFactory.ixExtractor('extractor8', 'relationship_property', [extractedTemplateName]), ], ixsuggestions: [ - fixturesFactory.ixSuggestion( + fixturesFactory.ixSuggestion_deprecated( 'new_prop_1_suggestion', 'extractor1', 'entity for new file', @@ -107,7 +107,7 @@ const fixtures: DBFixture = { 'entfile', 'extracted_property_1' ), - fixturesFactory.ixSuggestion( + fixturesFactory.ixSuggestion_deprecated( 'new_prop_2_suggestion', 'extractor2', 'entity for new file', @@ -115,7 +115,7 @@ const fixtures: DBFixture = { 'entfile', 'extracted_property_2' ), - fixturesFactory.ixSuggestion( + fixturesFactory.ixSuggestion_deprecated( 'new_title_suggestion', 'title_extractor', 'entity for new file', @@ -123,7 +123,7 @@ const fixtures: DBFixture = { 'entfile', 'title' ), - fixturesFactory.ixSuggestion( + fixturesFactory.ixSuggestion_deprecated( 'new_select_suggestion', 'extractor6', 'entity for new file', @@ -131,7 +131,7 @@ const fixtures: DBFixture = { 'entfile', 'select_property' ), - fixturesFactory.ixSuggestion( + fixturesFactory.ixSuggestion_deprecated( 'new_multiselect_suggestion', 'extractor7', 'entity for new file', @@ -139,7 +139,7 @@ const fixtures: DBFixture = { 'entfile', 'multiselect_property' ), - fixturesFactory.ixSuggestion( + fixturesFactory.ixSuggestion_deprecated( 'new_relationship_suggestion', 'extractor8', 'entity for new file', diff --git a/app/api/suggestions/specs/fixtures.ts b/app/api/suggestions/specs/fixtures.ts index 5b201df0ab..ae31c62e29 100644 --- a/app/api/suggestions/specs/fixtures.ts +++ b/app/api/suggestions/specs/fixtures.ts @@ -1090,7 +1090,7 @@ const stateFilterFixtures: DBFixture = { factory.ixExtractor('unused_extractor', 'unused_prop', ['template1']), ], ixsuggestions: [ - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'label-match-suggestion-en', 'test_extractor', 'labeled-match', @@ -1104,7 +1104,7 @@ const stateFilterFixtures: DBFixture = { suggestedValue: 'test-labeled-match', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'label-match-suggestion-es', 'test_extractor', 'labeled-match', @@ -1118,7 +1118,7 @@ const stateFilterFixtures: DBFixture = { suggestedValue: 'test-labeled-match', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'label-mismatch-suggestion-en', 'test_extractor', 'labeled-mismatch', @@ -1132,7 +1132,7 @@ const stateFilterFixtures: DBFixture = { suggestedValue: 'test-labeled-mismatch-mismatch', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'label-mismatch-suggestion-es', 'test_extractor', 'labeled-mismatch', @@ -1146,7 +1146,7 @@ const stateFilterFixtures: DBFixture = { suggestedValue: 'test-labeled-mismatch-mismatch', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unlabeled-no-suggestion-suggestion-en', 'test_extractor', 'unlabeled-no-suggestion', @@ -1160,7 +1160,7 @@ const stateFilterFixtures: DBFixture = { suggestedValue: '', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unlabeled-no-suggestion-suggestion-es', 'test_extractor', 'unlabeled-no-suggestion', @@ -1174,7 +1174,7 @@ const stateFilterFixtures: DBFixture = { suggestedValue: '', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unlabeled-no-context-suggestion-en', 'test_extractor', 'unlabeled-no-context', @@ -1188,7 +1188,7 @@ const stateFilterFixtures: DBFixture = { suggestedValue: 'test-unlabeled-no-context', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unlabeled-no-context-suggestion-es', 'test_extractor', 'unlabeled-no-context', @@ -1202,7 +1202,7 @@ const stateFilterFixtures: DBFixture = { suggestedValue: 'test-unlabeled-no-context', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unlabeled-obsolete-suggestion-en', 'test_extractor', 'unlabeled-obsolete', @@ -1217,7 +1217,7 @@ const stateFilterFixtures: DBFixture = { segment: 'test-unlabeled-obsolete', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unlabeled-obsolete-suggestion-es', 'test_extractor', 'unlabeled-obsolete', @@ -1232,7 +1232,7 @@ const stateFilterFixtures: DBFixture = { segment: 'test-unlabeled-obsolete', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unlabeled-processing-suggestion-en', 'test_extractor', 'unlabeled-processing', @@ -1247,7 +1247,7 @@ const stateFilterFixtures: DBFixture = { segment: 'test-unlabeled-processing', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unlabeled-processing-suggestion-es', 'test_extractor', 'unlabeled-processing', @@ -1262,7 +1262,7 @@ const stateFilterFixtures: DBFixture = { segment: 'test-unlabeled-processing', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unlabeled-error-suggestion-en', 'test_extractor', 'unlabeled-error', @@ -1278,7 +1278,7 @@ const stateFilterFixtures: DBFixture = { error: 'some error happened', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unlabeled-error-suggestion-es', 'test_extractor', 'unlabeled-error', @@ -1294,7 +1294,7 @@ const stateFilterFixtures: DBFixture = { error: 'some error happened', } ), - factory.ixSuggestion( + factory.ixSuggestion_deprecated( 'unusedsuggestion', 'unused_extractor', 'unused', diff --git a/app/api/utils/fixturesFactory.ts b/app/api/utils/fixturesFactory.ts index 15b01aa27e..0641bf5643 100644 --- a/app/api/utils/fixturesFactory.ts +++ b/app/api/utils/fixturesFactory.ts @@ -129,8 +129,8 @@ function getFixturesFactory() { defaultProps: EntitySchema = {}, propsPerLanguage: | { - [key: string]: EntitySchema; - } + [key: string]: EntitySchema; + } | undefined = undefined ): EntitySchema[] { return languages.map(language => { @@ -245,8 +245,8 @@ function getFixturesFactory() { typeof item === 'string' ? [{ _id: idMapper(item), id: item, label: item }] : Object.entries(item).map(([rootValue, children]) => - thesaurusNestedValues(rootValue, children, idMapper) - ); + thesaurusNestedValues(rootValue, children, idMapper) + ); return [...accumulator, ...nestedItems]; }, [] @@ -298,7 +298,8 @@ function getFixturesFactory() { extractorId: idMapper(extractor), }), - ixSuggestion: ( + // eslint-disable-next-line max-params + ixSuggestion_deprecated: ( suggestionId: string, extractor: string, entity: string, From f6fdc0dd870a55f9e5364f77a7ad15239d11d257 Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Wed, 4 Sep 2024 15:34:13 -0500 Subject: [PATCH 08/26] Moved ixSuggestions factory to Factory --- .../suggestions/specs/customFilters.spec.ts | 119 ++++++------------ app/api/utils/fixturesFactory.ts | 39 +++++- 2 files changed, 77 insertions(+), 81 deletions(-) diff --git a/app/api/suggestions/specs/customFilters.spec.ts b/app/api/suggestions/specs/customFilters.spec.ts index 4974b03ae4..e1844dc09c 100644 --- a/app/api/suggestions/specs/customFilters.spec.ts +++ b/app/api/suggestions/specs/customFilters.spec.ts @@ -1,6 +1,6 @@ -import db, { testingDB } from 'api/utils/testing_db'; -import { IXSuggestionType, SuggestionCustomFilter } from 'shared/types/suggestionType'; -import { factory, stateFilterFixtures } from './fixtures'; +import { testingDB } from 'api/utils/testing_db'; +import { SuggestionCustomFilter } from 'shared/types/suggestionType'; +import { factory as f, stateFilterFixtures } from './fixtures'; import { Suggestions } from '../suggestions'; const blankCustomFilter: SuggestionCustomFilter = { @@ -18,18 +18,18 @@ const blankCustomFilter: SuggestionCustomFilter = { }; beforeAll(async () => { - await db.setupFixturesAndContext(stateFilterFixtures); + await testingDB.setupFixturesAndContext(stateFilterFixtures); await Suggestions.updateStates({}); }); -afterAll(async () => db.disconnect()); +afterAll(async () => testingDB.disconnect()); describe('suggestions with CustomFilters', () => { describe('get()', () => { it('should return all suggestions (except processing) when no custom filter is provided', async () => { const result = await Suggestions.get( { - extractorId: factory.id('test_extractor').toString(), + extractorId: f.id('test_extractor').toString(), }, {} ); @@ -52,7 +52,7 @@ describe('suggestions with CustomFilters', () => { it('should be able to paginate', async () => { const result = await Suggestions.get( { - extractorId: factory.id('test_extractor').toString(), + extractorId: f.id('test_extractor').toString(), }, { page: { number: 3, size: 2 } } ); @@ -230,7 +230,7 @@ describe('suggestions with CustomFilters', () => { async ({ customFilter, expectedSuggestions }) => { const result = await Suggestions.get( { - extractorId: factory.id('test_extractor').toString(), + extractorId: f.id('test_extractor').toString(), customFilter, }, {} @@ -241,56 +241,20 @@ describe('suggestions with CustomFilters', () => { }); describe('aggreagate()', () => { - type PartialSuggestion = Partial> & { - state?: Partial; - }; - - const nFactory = () => ({ - suggestion(props: PartialSuggestion): IXSuggestionType { - const { state, ...otherProps } = props; - - return { - _id: testingDB.id(), - entityId: testingDB.id().toString(), - status: 'ready' as const, - entityTemplate: testingDB.id().toString(), - language: 'en', - fileId: testingDB.id().toString(), - propertyName: 'propertyName', - extractorId: testingDB.id(), - error: '', - segment: '', - suggestedValue: '', - date: 1001, - state: { - labeled: false, - withValue: true, - withSuggestion: false, - match: false, - hasContext: false, - obsolete: false, - processing: false, - error: false, - ...state, - }, - ...otherProps, - }; - }, - }); - it('should return count of labeled and non labeled suggestions', async () => { - const f = nFactory(); - - await db.setupFixturesAndContext({ + await testingDB.setupFixturesAndContext({ ixsuggestions: [ - f.suggestion({ extractorId: factory.id('test_extractor'), state: { labeled: true } }), - f.suggestion({ extractorId: factory.id('another_extractor'), state: { labeled: true } }), - f.suggestion({ extractorId: factory.id('test_extractor'), state: { labeled: false } }), - f.suggestion({ extractorId: factory.id('test_extractor'), state: { labeled: false } }), + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: { labeled: true } }), + f.ixSuggestion({ + extractorId: f.id('another_extractor'), + state: { labeled: true }, + }), + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: { labeled: false } }), + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: { labeled: false } }), ], }); - const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); + const result = await Suggestions.aggregate(f.id('test_extractor').toString()); expect(result).toMatchObject({ total: 3, labeled: 1, @@ -299,18 +263,16 @@ describe('suggestions with CustomFilters', () => { }); it('should return count of match and missmatch', async () => { - const f = nFactory(); - - await db.setupFixturesAndContext({ + await testingDB.setupFixturesAndContext({ ixsuggestions: [ - f.suggestion({ extractorId: factory.id('test_extractor'), state: { match: true } }), - f.suggestion({ extractorId: factory.id('another_extractor'), state: { match: true } }), - f.suggestion({ extractorId: factory.id('test_extractor'), state: { match: true } }), - f.suggestion({ extractorId: factory.id('test_extractor'), state: { match: false } }), + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: { match: true } }), + f.ixSuggestion({ extractorId: f.id('another_extractor'), state: { match: true } }), + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: { match: true } }), + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: { match: false } }), ], }); - const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); + const result = await Suggestions.aggregate(f.id('test_extractor').toString()); expect(result).toMatchObject({ total: 3, match: 2, @@ -319,19 +281,20 @@ describe('suggestions with CustomFilters', () => { }); it('should return count of obsolete suggestions', async () => { - const f = nFactory(); - - await db.setupFixturesAndContext({ + await testingDB.setupFixturesAndContext({ ixsuggestions: [ - f.suggestion({ extractorId: factory.id('test_extractor'), state: { obsolete: true } }), - f.suggestion({ extractorId: factory.id('test_extractor'), state: { obsolete: true } }), - f.suggestion({ extractorId: factory.id('another_extractor'), state: { obsolete: true } }), - f.suggestion({ extractorId: factory.id('test_extractor'), state: { obsolete: false } }), - f.suggestion({ extractorId: factory.id('test_extractor'), state: {} }), + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: { obsolete: true } }), + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: { obsolete: true } }), + f.ixSuggestion({ + extractorId: f.id('another_extractor'), + state: { obsolete: true }, + }), + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: { obsolete: false } }), + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: {} }), ], }); - const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); + const result = await Suggestions.aggregate(f.id('test_extractor').toString()); expect(result).toMatchObject({ total: 4, obsolete: 2, @@ -339,17 +302,15 @@ describe('suggestions with CustomFilters', () => { }); it('should return count of errors in suggestions', async () => { - const f = nFactory(); - - await db.setupFixturesAndContext({ + await testingDB.setupFixturesAndContext({ ixsuggestions: [ - f.suggestion({ extractorId: factory.id('test_extractor'), state: { error: true } }), - f.suggestion({ extractorId: factory.id('another_extractor'), state: { error: true } }), - f.suggestion({ extractorId: factory.id('test_extractor'), state: {} }), + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: { error: true } }), + f.ixSuggestion({ extractorId: f.id('another_extractor'), state: { error: true } }), + f.ixSuggestion({ extractorId: f.id('test_extractor'), state: {} }), ], }); - const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); + const result = await Suggestions.aggregate(f.id('test_extractor').toString()); expect(result).toMatchObject({ total: 2, error: 1, @@ -357,11 +318,11 @@ describe('suggestions with CustomFilters', () => { }); it('should correctly return all zeroes if no suggestions found', async () => { - await db.setupFixturesAndContext({ + await testingDB.setupFixturesAndContext({ ixsuggestions: [], }); - const result = await Suggestions.aggregate(factory.id('test_extractor').toString()); + const result = await Suggestions.aggregate(f.id('test_extractor').toString()); expect(result).toMatchObject({ total: 0, labeled: 0, diff --git a/app/api/utils/fixturesFactory.ts b/app/api/utils/fixturesFactory.ts index 0641bf5643..879a89c1bc 100644 --- a/app/api/utils/fixturesFactory.ts +++ b/app/api/utils/fixturesFactory.ts @@ -2,7 +2,7 @@ import _ from 'lodash'; import { ObjectId } from 'mongodb'; -import db from 'api/utils/testing_db'; +import { testingDB } from 'api/utils/testing_db'; import { EntitySchema } from 'shared/types/entityType'; import { FileType } from 'shared/types/fileType'; import { UserRole } from 'shared/types/userSchema'; @@ -24,11 +24,15 @@ import { getV2FixturesFactoryElements } from 'api/common.v2/testing/fixturesFact import { IXModelType } from 'shared/types/IXModelType'; import { PermissionSchema } from 'shared/types/permissionType'; +type PartialSuggestion = Partial> & { + state?: Partial; +}; + function getIdMapper() { const map = new Map(); return function setAndGet(key: string) { - if (!map.has(key)) map.set(key, db.id() as ObjectId); + if (!map.has(key)) map.set(key, testingDB.id() as ObjectId); return map.get(key)!; }; @@ -298,6 +302,37 @@ function getFixturesFactory() { extractorId: idMapper(extractor), }), + ixSuggestion(props: PartialSuggestion): IXSuggestionType { + const { state, ...otherProps } = props; + + return { + _id: testingDB.id(), + entityId: testingDB.id().toString(), + status: 'ready' as const, + entityTemplate: testingDB.id().toString(), + language: 'en', + fileId: testingDB.id().toString(), + propertyName: 'propertyName', + extractorId: testingDB.id(), + error: '', + segment: '', + suggestedValue: '', + date: 1001, + state: { + labeled: false, + withValue: true, + withSuggestion: false, + match: false, + hasContext: false, + obsolete: false, + processing: false, + error: false, + ...state, + }, + ...otherProps, + }; + }, + // eslint-disable-next-line max-params ixSuggestion_deprecated: ( suggestionId: string, From 78e0afc4e8a1a55c18c2ccbd9472f4a3120432cb Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Wed, 4 Sep 2024 17:06:19 -0500 Subject: [PATCH 09/26] Changed filtering logic. --- app/api/suggestions/pipelineStages.ts | 47 ++--- .../suggestions/specs/customFilters.spec.ts | 180 +++++++----------- app/api/suggestions/specs/fixtures.ts | 9 + app/api/suggestions/suggestions.ts | 17 +- app/shared/types/suggestionSchema.ts | 29 +-- app/shared/types/suggestionType.d.ts | 17 +- 6 files changed, 110 insertions(+), 189 deletions(-) diff --git a/app/api/suggestions/pipelineStages.ts b/app/api/suggestions/pipelineStages.ts index c222e9c315..26702c2e61 100644 --- a/app/api/suggestions/pipelineStages.ts +++ b/app/api/suggestions/pipelineStages.ts @@ -14,46 +14,23 @@ export const baseQueryFragment = (extractorId: ObjectId, ignoreProcessing = true }; export const filterFragments = { - labeled: { - _fragment: { 'state.labeled': true }, - match: { 'state.labeled': true, 'state.match': true }, - mismatch: { 'state.labeled': true, 'state.match': false }, - }, - nonLabeled: { - _fragment: { 'state.labeled': false }, - withSuggestion: { 'state.labeled': false, 'state.withSuggestion': true }, - noSuggestion: { 'state.labeled': false, 'state.withSuggestion': false }, - noContext: { 'state.labeled': false, 'state.hasContext': false }, - obsolete: { 'state.labeled': false, 'state.obsolete': true }, - others: { 'state.labeled': false, 'state.error': true }, - }, + labeled: { 'state.labeled': true }, + nonLabeled: { 'state.labeled': false }, + match: { 'state.match': true }, + mismatch: { 'state.match': false }, + obsolete: { 'state.obsolete': true }, + error: { 'state.error': true }, }; export const translateCustomFilter = (customFilter: SuggestionCustomFilter) => { const orFilters = []; - if (customFilter.labeled.match) { - orFilters.push(filterFragments.labeled.match); - } - if (customFilter.labeled.mismatch) { - orFilters.push(filterFragments.labeled.mismatch); - } - - if (customFilter.nonLabeled.withSuggestion) { - orFilters.push(filterFragments.nonLabeled.withSuggestion); - } + if (customFilter.labeled) orFilters.push(filterFragments.labeled); + if (customFilter.nonLabeled) orFilters.push(filterFragments.nonLabeled); + if (customFilter.match) orFilters.push(filterFragments.match); + if (customFilter.mismatch) orFilters.push(filterFragments.mismatch); + if (customFilter.obsolete) orFilters.push(filterFragments.obsolete); + if (customFilter.error) orFilters.push(filterFragments.error); - if (customFilter.nonLabeled.noSuggestion) { - orFilters.push(filterFragments.nonLabeled.noSuggestion); - } - if (customFilter.nonLabeled.noContext) { - orFilters.push(filterFragments.nonLabeled.noContext); - } - if (customFilter.nonLabeled.obsolete) { - orFilters.push(filterFragments.nonLabeled.obsolete); - } - if (customFilter.nonLabeled.others) { - orFilters.push(filterFragments.nonLabeled.others); - } return orFilters; }; diff --git a/app/api/suggestions/specs/customFilters.spec.ts b/app/api/suggestions/specs/customFilters.spec.ts index e1844dc09c..f07c5fd1ac 100644 --- a/app/api/suggestions/specs/customFilters.spec.ts +++ b/app/api/suggestions/specs/customFilters.spec.ts @@ -4,17 +4,12 @@ import { factory as f, stateFilterFixtures } from './fixtures'; import { Suggestions } from '../suggestions'; const blankCustomFilter: SuggestionCustomFilter = { - labeled: { - match: false, - mismatch: false, - }, - nonLabeled: { - withSuggestion: false, - noSuggestion: false, - noContext: false, - obsolete: false, - others: false, - }, + labeled: false, + nonLabeled: false, + match: false, + mismatch: false, + obsolete: false, + error: false, }; beforeAll(async () => { @@ -33,6 +28,7 @@ describe('suggestions with CustomFilters', () => { }, {} ); + expect(result.suggestions.length).toBe(12); expect(result.suggestions).toMatchObject([ { sharedId: 'unlabeled-obsolete', language: 'en' }, { sharedId: 'unlabeled-obsolete', language: 'es' }, @@ -64,42 +60,18 @@ describe('suggestions with CustomFilters', () => { it.each([ { - description: 'filtering for labeled - match', - customFilter: { - ...blankCustomFilter, - labeled: { - match: true, - mismatch: false, - }, - }, + description: 'filtering for labeled', + customFilter: { ...blankCustomFilter, labeled: true }, expectedSuggestions: [ { sharedId: 'labeled-match', language: 'en' }, { sharedId: 'labeled-match', language: 'es' }, - ], - }, - { - description: 'filtering for labeled - mismatch', - customFilter: { - ...blankCustomFilter, - labeled: { - match: false, - mismatch: true, - }, - }, - expectedSuggestions: [ { sharedId: 'labeled-mismatch', language: 'en' }, { sharedId: 'labeled-mismatch', language: 'es' }, ], }, { - description: 'filtering for nonLabeled - withSuggestion', - customFilter: { - ...blankCustomFilter, - nonLabeled: { - ...blankCustomFilter.nonLabeled, - withSuggestion: true, - }, - }, + description: 'filtering for nonLabeled', + customFilter: { ...blankCustomFilter, nonLabeled: true }, expectedSuggestions: [ { sharedId: 'unlabeled-obsolete', language: 'en' }, { sharedId: 'unlabeled-obsolete', language: 'es' }, @@ -107,118 +79,101 @@ describe('suggestions with CustomFilters', () => { { sharedId: 'unlabeled-error', language: 'es' }, { sharedId: 'unlabeled-no-context', language: 'en' }, { sharedId: 'unlabeled-no-context', language: 'es' }, - ], - }, - { - description: 'filtering for nonLabeled - noSuggestion', - customFilter: { - ...blankCustomFilter, - nonLabeled: { - ...blankCustomFilter.nonLabeled, - noSuggestion: true, - }, - }, - expectedSuggestions: [ { sharedId: 'unlabeled-no-suggestion', language: 'en' }, { sharedId: 'unlabeled-no-suggestion', language: 'es' }, ], }, { - description: 'filtering for nonLabeled - withSuggestions and noSuggestion', - customFilter: { - ...blankCustomFilter, - nonLabeled: { - ...blankCustomFilter.nonLabeled, - withSuggestion: true, - noSuggestion: true, - }, - }, + description: 'filtering for match', + customFilter: { ...blankCustomFilter, match: true }, expectedSuggestions: [ { sharedId: 'unlabeled-obsolete', language: 'en' }, { sharedId: 'unlabeled-obsolete', language: 'es' }, + { sharedId: 'labeled-match', language: 'en' }, + { sharedId: 'labeled-match', language: 'es' }, { sharedId: 'unlabeled-error', language: 'en' }, { sharedId: 'unlabeled-error', language: 'es' }, { sharedId: 'unlabeled-no-context', language: 'en' }, { sharedId: 'unlabeled-no-context', language: 'es' }, - { sharedId: 'unlabeled-no-suggestion', language: 'en' }, - { sharedId: 'unlabeled-no-suggestion', language: 'es' }, ], }, { - description: 'filtering for nonLabeled - noContext', - customFilter: { - ...blankCustomFilter, - nonLabeled: { - ...blankCustomFilter.nonLabeled, - noContext: true, - }, - }, + description: 'filtering for mismatch', + customFilter: { ...blankCustomFilter, mismatch: true }, expectedSuggestions: [ - { sharedId: 'unlabeled-no-context', language: 'en' }, - { sharedId: 'unlabeled-no-context', language: 'es' }, + { sharedId: 'labeled-mismatch', language: 'en' }, + { sharedId: 'labeled-mismatch', language: 'es' }, { sharedId: 'unlabeled-no-suggestion', language: 'en' }, { sharedId: 'unlabeled-no-suggestion', language: 'es' }, ], }, { - description: 'filtering for nonLabeled - obsolete', - customFilter: { - ...blankCustomFilter, - nonLabeled: { - ...blankCustomFilter.nonLabeled, - obsolete: true, - }, - }, + description: 'filtering for obsolete', + customFilter: { ...blankCustomFilter, obsolete: true }, expectedSuggestions: [ { sharedId: 'unlabeled-obsolete', language: 'en' }, { sharedId: 'unlabeled-obsolete', language: 'es' }, ], }, { - description: 'filtering for nonLabeled - others', - customFilter: { - ...blankCustomFilter, - nonLabeled: { - ...blankCustomFilter.nonLabeled, - others: true, - }, - }, + description: 'filtering for error', + customFilter: { ...blankCustomFilter, error: true }, expectedSuggestions: [ { sharedId: 'unlabeled-error', language: 'en' }, { sharedId: 'unlabeled-error', language: 'es' }, ], }, { - description: 'filtering for labeled - match and nonLabeled - obsolete', - customFilter: { - ...blankCustomFilter, - labeled: { - match: true, - mismatch: false, - }, - nonLabeled: { - ...blankCustomFilter.nonLabeled, - obsolete: true, - }, - }, + description: 'filtering for OR combinations like: error OR obsolete', + customFilter: { ...blankCustomFilter, error: true, obsolete: true }, + expectedSuggestions: [ + { sharedId: 'unlabeled-obsolete', language: 'en' }, + { sharedId: 'unlabeled-obsolete', language: 'es' }, + { sharedId: 'unlabeled-error', language: 'en' }, + { sharedId: 'unlabeled-error', language: 'es' }, + ], + }, + { + description: 'filtering for OR combinations like: mismatch OR error', + customFilter: { ...blankCustomFilter, mismatch: true, error: true }, + expectedSuggestions: [ + { sharedId: 'labeled-mismatch', language: 'en' }, + { sharedId: 'labeled-mismatch', language: 'es' }, + { sharedId: 'unlabeled-error', language: 'en' }, + { sharedId: 'unlabeled-error', language: 'es' }, + { sharedId: 'unlabeled-no-suggestion', language: 'en' }, + { sharedId: 'unlabeled-no-suggestion', language: 'es' }, + ], + }, + { + description: 'filtering for OR combinations like: labeled OR match', + customFilter: { ...blankCustomFilter, labeled: true, match: true }, expectedSuggestions: [ { sharedId: 'unlabeled-obsolete', language: 'en' }, { sharedId: 'unlabeled-obsolete', language: 'es' }, { sharedId: 'labeled-match', language: 'en' }, { sharedId: 'labeled-match', language: 'es' }, + { sharedId: 'labeled-mismatch', language: 'en' }, + { sharedId: 'labeled-mismatch', language: 'es' }, + { sharedId: 'unlabeled-error', language: 'en' }, + { sharedId: 'unlabeled-error', language: 'es' }, + { sharedId: 'unlabeled-no-context', language: 'en' }, + { sharedId: 'unlabeled-no-context', language: 'es' }, ], }, { - description: 'filtering for nonLabeled - noSuggestion and nonLabeled - noContext', - customFilter: { - ...blankCustomFilter, - nonLabeled: { - ...blankCustomFilter.nonLabeled, - noSuggestion: true, - noContext: true, - }, - }, + description: + 'filtering for OR combinations of complimentary filters like: labeled OR nonLabeled, which would result in all suggestions', + customFilter: { ...blankCustomFilter, labeled: true, nonLabeled: true }, expectedSuggestions: [ + { sharedId: 'unlabeled-obsolete', language: 'en' }, + { sharedId: 'unlabeled-obsolete', language: 'es' }, + { sharedId: 'labeled-match', language: 'en' }, + { sharedId: 'labeled-match', language: 'es' }, + { sharedId: 'labeled-mismatch', language: 'en' }, + { sharedId: 'labeled-mismatch', language: 'es' }, + { sharedId: 'unlabeled-error', language: 'en' }, + { sharedId: 'unlabeled-error', language: 'es' }, { sharedId: 'unlabeled-no-context', language: 'en' }, { sharedId: 'unlabeled-no-context', language: 'es' }, { sharedId: 'unlabeled-no-suggestion', language: 'en' }, @@ -229,12 +184,11 @@ describe('suggestions with CustomFilters', () => { 'should use the custom filter properly when $description', async ({ customFilter, expectedSuggestions }) => { const result = await Suggestions.get( - { - extractorId: f.id('test_extractor').toString(), - customFilter, - }, + { extractorId: f.id('test_extractor').toString(), customFilter }, {} ); + + expect(result.suggestions.length).toBe(expectedSuggestions.length); expect(result.suggestions).toMatchObject(expectedSuggestions); } ); diff --git a/app/api/suggestions/specs/fixtures.ts b/app/api/suggestions/specs/fixtures.ts index ae31c62e29..82b041e53b 100644 --- a/app/api/suggestions/specs/fixtures.ts +++ b/app/api/suggestions/specs/fixtures.ts @@ -1308,6 +1308,15 @@ const stateFilterFixtures: DBFixture = { suggestedValue: 'test-unused', } ), + factory.ixSuggestion({ + extractorId: factory.id('unused_extractor'), + state: { + labeled: true, + match: true, + obsolete: true, + error: true, + }, + }), ], }; diff --git a/app/api/suggestions/suggestions.ts b/app/api/suggestions/suggestions.ts index 41d5d6924f..2793beeb3b 100644 --- a/app/api/suggestions/suggestions.ts +++ b/app/api/suggestions/suggestions.ts @@ -42,6 +42,8 @@ import { updateEntitiesWithSuggestion, } from './updateEntities'; +const DEFAULT_LIMIT = 30; + const updateExtractedMetadata = async ( suggestions: IXSuggestionType[], property: PropertySchema @@ -88,10 +90,12 @@ const buildListQuery = ( extractorId: ObjectId, customFilter: SuggestionCustomFilter | undefined, setLanguages: LanguagesListSchema | undefined, - offset: number, - limit: number, - sort?: IXSuggestionsQuery['sort'] + options: { page?: IXSuggestionsQuery['page']; sort?: IXSuggestionsQuery['sort'] } ) => { + const offset = options && options.page ? options.page.size * (options.page.number - 1) : 0; + const limit = options.page?.size || DEFAULT_LIMIT; + const { sort } = options; + const sortOrder = sort?.order === 'desc' ? -1 : 1; const sorting = sort?.property ? { [sort.property]: sortOrder } : { date: 1, state: -1 }; @@ -188,9 +192,6 @@ const Suggestions = { sort?: IXSuggestionsQuery['sort']; } ) => { - const offset = options && options.page ? options.page.size * (options.page.number - 1) : 0; - const DEFAULT_LIMIT = 30; - const limit = options.page?.size || DEFAULT_LIMIT; const { languages: setLanguages } = await settings.get(); const { customFilter, extractorId } = readFilter(filter); @@ -199,13 +200,13 @@ const Suggestions = { .then(result => (result?.length ? result[0].count : 0)); let suggestions = await IXSuggestionsModel.db.aggregate( - buildListQuery(extractorId, customFilter, setLanguages, offset, limit, options.sort) + buildListQuery(extractorId, customFilter, setLanguages, options) ); suggestions = await postProcessSuggestions(suggestions, extractorId); return { suggestions, - totalPages: Math.ceil(count / limit), + totalPages: Math.ceil(count / (options.page?.size || DEFAULT_LIMIT)), }; }, diff --git a/app/shared/types/suggestionSchema.ts b/app/shared/types/suggestionSchema.ts index a019afdd11..51e871742e 100644 --- a/app/shared/types/suggestionSchema.ts +++ b/app/shared/types/suggestionSchema.ts @@ -205,29 +205,14 @@ export const SuggestionCustomFilterSchema = { title: 'SuggestionCustomFilter', additionalProperties: false, properties: { - labeled: { - type: 'object', - properties: { - match: { type: 'boolean' }, - mismatch: { type: 'boolean' }, - }, - additionalProperties: false, - required: ['match', 'mismatch'], - }, - nonLabeled: { - type: 'object', - properties: { - withSuggestion: { type: 'boolean' }, - noSuggestion: { type: 'boolean' }, - noContext: { type: 'boolean' }, - obsolete: { type: 'boolean' }, - others: { type: 'boolean' }, - }, - additionalProperties: false, - required: ['withSuggestion', 'noSuggestion', 'noContext', 'obsolete', 'others'], - }, + labeled: { type: 'boolean' }, + match: { type: 'boolean' }, + mismatch: { type: 'boolean' }, + nonLabeled: { type: 'boolean' }, + obsolete: { type: 'boolean' }, + error: { type: 'boolean' }, }, - required: ['labeled', 'nonLabeled'], + required: ['labeled', 'nonLabeled', 'match', 'mismatch', 'obsolete', 'error'], }; export const SuggestionsQueryFilterSchema = { diff --git a/app/shared/types/suggestionType.d.ts b/app/shared/types/suggestionType.d.ts index 152511f554..93469addbe 100644 --- a/app/shared/types/suggestionType.d.ts +++ b/app/shared/types/suggestionType.d.ts @@ -105,17 +105,12 @@ export interface IXSuggestionsQuery { } export interface SuggestionCustomFilter { - labeled: { - match: boolean; - mismatch: boolean; - }; - nonLabeled: { - withSuggestion: boolean; - noSuggestion: boolean; - noContext: boolean; - obsolete: boolean; - others: boolean; - }; + labeled: boolean; + match: boolean; + mismatch: boolean; + nonLabeled: boolean; + obsolete: boolean; + error: boolean; } export interface IXSuggestionsFilter { From 25255c07035ce156bf93996a70f7f0777decef23 Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Wed, 4 Sep 2024 17:29:21 -0500 Subject: [PATCH 10/26] Fixed Route validation and tests --- app/api/suggestions/specs/routes.spec.ts | 57 +++++++------------ app/api/suggestions/specs/suggestions.spec.ts | 3 +- 2 files changed, 21 insertions(+), 39 deletions(-) diff --git a/app/api/suggestions/specs/routes.spec.ts b/app/api/suggestions/specs/routes.spec.ts index 68b97b1756..6a36545b84 100644 --- a/app/api/suggestions/specs/routes.spec.ts +++ b/app/api/suggestions/specs/routes.spec.ts @@ -208,36 +208,24 @@ describe('suggestions routes', () => { filter: { extractorId: factory.id('enemy_extractor').toString(), customFilter: { - labeled: { - match: false, - mismatch: false, - }, - nonLabeled: { - withSuggestion: false, - noSuggestion: true, - noContext: false, - obsolete: false, - others: false, - }, + labeled: true, + nonLabeled: false, + match: false, + mismatch: false, + obsolete: false, + error: false, }, }, }) .expect(200); expect(response.body.suggestions).toEqual([ expect.objectContaining({ - entityTitle: 'Catwoman', - state: { - labeled: false, - withValue: false, - withSuggestion: false, - match: false, - hasContext: true, - obsolete: false, - processing: false, - error: false, - }, - suggestedValue: '', - currentValue: '', + entityTitle: 'The Penguin', + language: 'en', + }), + expect.objectContaining({ + entityTitle: 'The Penguin', + language: 'es', }), ]); }); @@ -428,20 +416,13 @@ describe('aggregation routes', () => { }) .expect(200); expect(response.body).toEqual({ - total: 12, - labeled: { - _count: 4, - match: 2, - mismatch: 2, - }, - nonLabeled: { - _count: 8, - withSuggestion: 6, - noSuggestion: 2, - noContext: 4, - obsolete: 2, - others: 2, - }, + total: 14, + labeled: 4, + nonLabeled: 10, + match: 10, + mismatch: 4, + obsolete: 2, + error: 2, }); }); }); diff --git a/app/api/suggestions/specs/suggestions.spec.ts b/app/api/suggestions/specs/suggestions.spec.ts index cd54b6124d..09ac72ccce 100644 --- a/app/api/suggestions/specs/suggestions.spec.ts +++ b/app/api/suggestions/specs/suggestions.spec.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-statements */ import db from 'api/utils/testing_db'; import { @@ -6,6 +7,7 @@ import { IXSuggestionType, IXSuggestionsFilter, } from 'shared/types/suggestionType'; +import { ObjectId } from 'mongodb'; import { Suggestions } from '../suggestions'; import { factory, @@ -20,7 +22,6 @@ import { selectAcceptanceFixtureBase, relationshipAcceptanceFixtureBase, } from './fixtures'; -import { ObjectId } from 'mongodb'; const getSuggestions = async (filter: IXSuggestionsFilter, size = 5) => Suggestions.get(filter, { page: { size, number: 1 } }); From ebaa8c004544a1f47af37d232ff78d471a4b66b1 Mon Sep 17 00:00:00 2001 From: A happy cat Date: Wed, 11 Sep 2024 14:56:16 +0200 Subject: [PATCH 11/26] Using updated component --- .../V2/Routes/Settings/IX/IXSuggestions.tsx | 18 +- .../IX/components/FiltersSidepanel.old.tsx | 231 ------------------ 2 files changed, 11 insertions(+), 238 deletions(-) delete mode 100644 app/react/V2/Routes/Settings/IX/components/FiltersSidepanel.old.tsx diff --git a/app/react/V2/Routes/Settings/IX/IXSuggestions.tsx b/app/react/V2/Routes/Settings/IX/IXSuggestions.tsx index 6a2132f193..07f49007a2 100644 --- a/app/react/V2/Routes/Settings/IX/IXSuggestions.tsx +++ b/app/react/V2/Routes/Settings/IX/IXSuggestions.tsx @@ -24,7 +24,7 @@ import { ClientPropertySchema, ClientTemplateSchema } from 'app/istore'; import { notificationAtom } from 'app/V2/atoms'; import { socket } from 'app/socket'; import { SuggestionsTitle } from './components/SuggestionsTitle'; -import { FiltersSidepanel } from './components/FiltersSidepanel.old'; +import { FiltersSidepanel } from './components/FiltersSidepanel'; import { suggestionsTableColumnsBuilder } from './components/TableElements'; import { PDFSidepanel } from './components/PDFSidepanel'; import { @@ -112,6 +112,7 @@ const IXSuggestions = () => { const [sidepanelSuggestion, setSidepanelSuggestion] = useState(); const [selected, setSelected] = useState([]); const [sorting, setSorting] = useState([]); + const [aggregations, setAggregations] = useState(aggregation); const { revalidate } = useRevalidator(); const setNotifications = useSetAtom(notificationAtom); const [status, setStatus] = useState<{ @@ -139,6 +140,10 @@ const IXSuggestions = () => { }; }, [extractor._id, revalidate]); + useEffect(() => { + setAggregations(aggregation); + }, [aggregation]); + useEffect(() => { if (searchParams.has('sort') && !sorting.length) { navigate(location.pathname, { replace: true }); @@ -188,6 +193,8 @@ const IXSuggestions = () => { await suggestionsAPI.accept(preparedSuggestions); setCurrentSuggestions(updateSuggestions(currentSuggestions, acceptedSuggestions)); + const newAggregations = await suggestionsAPI.aggregation(extractor._id); + setAggregations(newAggregations); setNotifications({ type: 'success', text: Suggestion accepted., @@ -272,7 +279,7 @@ const IXSuggestions = () => {
@@ -353,7 +360,7 @@ const IXSuggestions = () => { >; - aggregation: any; -} - -const Header = ({ label, total }: { label: string; total: number }) => ( -
-
{label}
-
-
{total}
-
-); - -const getPercentage = (match: number, total: number): string => { - const percentage = (match / total) * 100; - - if (Number.isNaN(percentage)) { - return '-'; - } - - return `${Math.round(percentage)}%`; -}; - -const FiltersSidepanel = ({ - showSidepanel, - setShowSidepanel, - aggregation, -}: FiltersSidepanelProps) => { - const [searchParams, setSearchParams] = useSearchParams(); - - const defaultFilter: SuggestionCustomFilter = { - labeled: { - match: false, - mismatch: false, - }, - nonLabeled: { - noContext: false, - withSuggestion: false, - noSuggestion: false, - obsolete: false, - others: false, - }, - }; - - let initialFilters: SuggestionCustomFilter = defaultFilter; - - try { - if (searchParams.has('filter')) { - initialFilters = JSON.parse(searchParams.get('filter')!); - } - } catch (e) {} - - const { register, handleSubmit, reset, setValue } = useForm({ - values: initialFilters, - defaultValues: defaultFilter, - }); - - const submitFilters = async (filters: SuggestionCustomFilter) => { - setSearchParams((prev: URLSearchParams) => { - prev.set('page', '1'); - prev.set('filter', JSON.stringify(filters)); - return prev; - }); - setShowSidepanel(false); - }; - - const checkOption = (e: any, optionName: any) => { - const { checked } = e.target; - setValue(optionName, checked); - }; - - const clearFilters = () => { - setSearchParams(prev => { - prev.delete('filter'); - return prev; - }); - setShowSidepanel(false); - reset(); - }; - - return ( - setShowSidepanel(false)} - title={ - - Stats & Filters - - } - > -
- - } - > -
-
-
- Accuracy -
-
-
- {getPercentage(aggregation.labeled.match, aggregation.labeled._count)} -
-
-
- { - checkOption(e, 'labeled.match'); - }} - /> -
-
{aggregation.labeled.match}
-
-
- { - checkOption(e, 'labeled.mismatch'); - }} - /> -
-
{aggregation.labeled.mismatch}
-
-
- - - } - > -
-
-
- Pending -
-
-
- {getPercentage(aggregation.nonLabeled._count, aggregation.total)} -
-
-
- With suggestion} - {...register('nonLabeled.withSuggestion')} - onChange={e => { - checkOption(e, 'nonLabeled.withSuggestion'); - }} - /> -
-
{aggregation.nonLabeled.withSuggestion}
-
-
- No suggestion} - {...register('nonLabeled.noSuggestion')} - onChange={e => { - checkOption(e, 'nonLabeled.noSuggestion'); - }} - /> -
-
{aggregation.nonLabeled.noSuggestion}
-
-
- No context} - {...register('nonLabeled.noContext')} - onChange={e => { - checkOption(e, 'nonLabeled.noContext'); - }} - /> -
-
{aggregation.nonLabeled.noContext}
-
-
- Obsolete} - {...register('nonLabeled.obsolete')} - onChange={e => { - checkOption(e, 'nonLabeled.obsolete'); - }} - /> -
-
{aggregation.nonLabeled.obsolete}
-
-
- Others} - {...register('nonLabeled.others')} - onChange={e => { - checkOption(e, 'nonLabeled.others'); - }} - /> -
-
{aggregation.nonLabeled.others}
-
-
- - - -
- - -
- - - - ); -}; - -export { FiltersSidepanel }; From b58cec791623c243cb913de47242e5354fe8e09c Mon Sep 17 00:00:00 2001 From: A happy cat Date: Wed, 11 Sep 2024 17:15:18 +0200 Subject: [PATCH 12/26] removed accuracy from the filters untill the backend returns it --- .../Settings/IX/components/FiltersSidepanel.tsx | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/app/react/V2/Routes/Settings/IX/components/FiltersSidepanel.tsx b/app/react/V2/Routes/Settings/IX/components/FiltersSidepanel.tsx index f03ad2d8e5..b8b9d79c5a 100644 --- a/app/react/V2/Routes/Settings/IX/components/FiltersSidepanel.tsx +++ b/app/react/V2/Routes/Settings/IX/components/FiltersSidepanel.tsx @@ -89,13 +89,6 @@ const FiltersSidepanel = ({
-
-
- Accuracy (labeled data) -
-
-
{aggregation.accuracy}%
-
Labeled} @@ -118,8 +111,6 @@ const FiltersSidepanel = ({
{aggregation.nonLabeled}
- -
Match} @@ -142,8 +133,6 @@ const FiltersSidepanel = ({
{aggregation.mismatch}
- -
Obsolete} From 677c442d6b7e3e40b951075af275f798b47fb77a Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Tue, 17 Sep 2024 13:06:23 -0500 Subject: [PATCH 13/26] Changed labeled and match to be optional --- app/shared/types/suggestionSchema.ts | 11 +---------- app/shared/types/suggestionType.d.ts | 4 ++-- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/app/shared/types/suggestionSchema.ts b/app/shared/types/suggestionSchema.ts index 51e871742e..7d3568d8fa 100644 --- a/app/shared/types/suggestionSchema.ts +++ b/app/shared/types/suggestionSchema.ts @@ -85,16 +85,7 @@ export const IXSuggestionStateSchema = { processing: { type: 'boolean' }, error: { type: 'boolean' }, }, - required: [ - 'labeled', - 'withValue', - 'withSuggestion', - 'match', - 'hasContext', - 'obsolete', - 'processing', - 'error', - ], + required: ['withValue', 'withSuggestion', 'hasContext', 'processing', 'obsolete', 'error'], }; export const IXSuggestionSchema = { diff --git a/app/shared/types/suggestionType.d.ts b/app/shared/types/suggestionType.d.ts index 93469addbe..a42a738dfa 100644 --- a/app/shared/types/suggestionType.d.ts +++ b/app/shared/types/suggestionType.d.ts @@ -81,10 +81,10 @@ export interface IXSuggestionType { } export interface IXSuggestionStateType { - labeled: boolean; + labeled?: boolean; withValue: boolean; withSuggestion: boolean; - match: boolean; + match?: boolean; hasContext: boolean; obsolete: boolean; processing: boolean; From f7e71b47a32bff7ffeb7c94959b48d6bff774929 Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Tue, 17 Sep 2024 13:07:08 -0500 Subject: [PATCH 14/26] Removed optional parameters from factory --- app/api/utils/fixturesFactory.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/app/api/utils/fixturesFactory.ts b/app/api/utils/fixturesFactory.ts index 879a89c1bc..16234bf0be 100644 --- a/app/api/utils/fixturesFactory.ts +++ b/app/api/utils/fixturesFactory.ts @@ -319,13 +319,11 @@ function getFixturesFactory() { suggestedValue: '', date: 1001, state: { - labeled: false, withValue: true, withSuggestion: false, - match: false, hasContext: false, - obsolete: false, processing: false, + obsolete: false, error: false, ...state, }, @@ -356,13 +354,11 @@ function getFixturesFactory() { suggestedValue: '', date: 1, state: { - labeled: false, withValue: true, withSuggestion: false, - match: false, hasContext: false, - obsolete: false, processing: false, + obsolete: false, error: false, }, ...otherProps, From d4ced636a2be6b849a92c4dd5fe125068114d490 Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Tue, 17 Sep 2024 13:07:39 -0500 Subject: [PATCH 15/26] Changed logic to remove labeled and match if error or obsolete. --- app/shared/getIXSuggestionState.ts | 19 ++++++++++++++----- app/shared/specs/getIXSuggestionState.spec.ts | 6 +----- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/app/shared/getIXSuggestionState.ts b/app/shared/getIXSuggestionState.ts index bac8f4ff5c..7986d85b28 100644 --- a/app/shared/getIXSuggestionState.ts +++ b/app/shared/getIXSuggestionState.ts @@ -40,13 +40,13 @@ const equalsForType = (type: PropertySchema['type']) => (first: any, second: any EQUALITIES[type] ? EQUALITIES[type](first, second) : first === second; class IXSuggestionState implements IXSuggestionStateType { - labeled = false; + labeled: boolean | undefined; withValue = false; withSuggestion = false; - match = false; + match: boolean | undefined; hasContext = false; @@ -71,6 +71,7 @@ class IXSuggestionState implements IXSuggestionStateType { { labeledValue, currentValue }: SuggestionValues, propertyType: PropertySchema['type'] ) { + this.labeled = false; if ( labeledValue || (propertyIsSelect(propertyType) && currentValue) || @@ -104,9 +105,13 @@ class IXSuggestionState implements IXSuggestionStateType { ) { const equals = equalsForType(propertyType); - if (suggestedValue === '' || (Array.isArray(suggestedValue) && suggestedValue.length === 0)) { - this.match = false; - } else if (equals(suggestedValue, currentValue)) { + this.match = false; + + if ( + suggestedValue !== '' && + (!Array.isArray(suggestedValue) || suggestedValue.length !== 0) && + equals(suggestedValue, currentValue) + ) { this.match = true; } } @@ -124,6 +129,8 @@ class IXSuggestionState implements IXSuggestionStateType { setObsolete({ modelCreationDate, date }: SuggestionValues) { if (date < modelCreationDate) { this.obsolete = true; + this.labeled = undefined; + this.match = undefined; } } @@ -136,6 +143,8 @@ class IXSuggestionState implements IXSuggestionStateType { setError({ error, status }: SuggestionValues) { if ((error && error !== '') || (status && status === 'failed')) { this.error = true; + this.labeled = undefined; + this.match = undefined; } } } diff --git a/app/shared/specs/getIXSuggestionState.spec.ts b/app/shared/specs/getIXSuggestionState.spec.ts index d0b73915be..6bc10b84d8 100644 --- a/app/shared/specs/getIXSuggestionState.spec.ts +++ b/app/shared/specs/getIXSuggestionState.spec.ts @@ -83,10 +83,8 @@ describe('getIXSuggestionState', () => { const state = getSuggestionState(values, 'text'); expect(state).toEqual({ - labeled: false, withValue: true, withSuggestion: false, - match: false, hasContext: false, obsolete: false, processing: false, @@ -140,7 +138,7 @@ describe('getIXSuggestionState', () => { }); }); - it('should mark when currentValue != suggestedValue, labeledValue are empty', () => { + it('should mark when currentValue != suggestedValue, labeledValue is empty', () => { const values = { currentValue: 'some other value', date: 1234, @@ -221,10 +219,8 @@ describe('getIXSuggestionState', () => { const state = getSuggestionState(values, 'text'); expect(state).toEqual({ - labeled: false, withValue: false, withSuggestion: true, - match: false, hasContext: false, obsolete: true, processing: false, From 6f54baca581ef65ca0888fce9e285c9852d26681 Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Tue, 17 Sep 2024 13:08:01 -0500 Subject: [PATCH 16/26] Disabled eslint to be able to look at code --- app/api/suggestions/updateState.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/app/api/suggestions/updateState.ts b/app/api/suggestions/updateState.ts index a352033ed0..f38f6886a6 100644 --- a/app/api/suggestions/updateState.ts +++ b/app/api/suggestions/updateState.ts @@ -109,6 +109,7 @@ export const postProcessCurrentValues = ( propertyType: PropertyTypeSchema ) => suggestions.map(s => postProcessCurrentValue(s, propertyType)); +// eslint-disable-next-line max-statements export const updateStates = async (query: any) => { const { languages } = await settings.get(); const propertyTypes = objectIndex( From 4ccba415a09a799018d0657907c081cb73b7af41 Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Tue, 17 Sep 2024 13:08:39 -0500 Subject: [PATCH 17/26] Changed logic to ensure undefined values are not counted towards limits. --- .../suggestions/specs/customFilters.spec.ts | 24 ++--------- app/api/suggestions/specs/routes.spec.ts | 5 ++- app/api/suggestions/suggestions.ts | 40 ++++++++++++++++--- 3 files changed, 41 insertions(+), 28 deletions(-) diff --git a/app/api/suggestions/specs/customFilters.spec.ts b/app/api/suggestions/specs/customFilters.spec.ts index f07c5fd1ac..104b87bb34 100644 --- a/app/api/suggestions/specs/customFilters.spec.ts +++ b/app/api/suggestions/specs/customFilters.spec.ts @@ -36,12 +36,12 @@ describe('suggestions with CustomFilters', () => { { sharedId: 'labeled-match', language: 'es' }, { sharedId: 'labeled-mismatch', language: 'en' }, { sharedId: 'labeled-mismatch', language: 'es' }, - { sharedId: 'unlabeled-error', language: 'en' }, - { sharedId: 'unlabeled-error', language: 'es' }, { sharedId: 'unlabeled-no-context', language: 'en' }, { sharedId: 'unlabeled-no-context', language: 'es' }, { sharedId: 'unlabeled-no-suggestion', language: 'en' }, { sharedId: 'unlabeled-no-suggestion', language: 'es' }, + { sharedId: 'unlabeled-error', language: 'en' }, + { sharedId: 'unlabeled-error', language: 'es' }, ]); }); @@ -73,10 +73,6 @@ describe('suggestions with CustomFilters', () => { description: 'filtering for nonLabeled', customFilter: { ...blankCustomFilter, nonLabeled: true }, expectedSuggestions: [ - { sharedId: 'unlabeled-obsolete', language: 'en' }, - { sharedId: 'unlabeled-obsolete', language: 'es' }, - { sharedId: 'unlabeled-error', language: 'en' }, - { sharedId: 'unlabeled-error', language: 'es' }, { sharedId: 'unlabeled-no-context', language: 'en' }, { sharedId: 'unlabeled-no-context', language: 'es' }, { sharedId: 'unlabeled-no-suggestion', language: 'en' }, @@ -87,12 +83,8 @@ describe('suggestions with CustomFilters', () => { description: 'filtering for match', customFilter: { ...blankCustomFilter, match: true }, expectedSuggestions: [ - { sharedId: 'unlabeled-obsolete', language: 'en' }, - { sharedId: 'unlabeled-obsolete', language: 'es' }, { sharedId: 'labeled-match', language: 'en' }, { sharedId: 'labeled-match', language: 'es' }, - { sharedId: 'unlabeled-error', language: 'en' }, - { sharedId: 'unlabeled-error', language: 'es' }, { sharedId: 'unlabeled-no-context', language: 'en' }, { sharedId: 'unlabeled-no-context', language: 'es' }, ], @@ -139,24 +131,20 @@ describe('suggestions with CustomFilters', () => { expectedSuggestions: [ { sharedId: 'labeled-mismatch', language: 'en' }, { sharedId: 'labeled-mismatch', language: 'es' }, - { sharedId: 'unlabeled-error', language: 'en' }, - { sharedId: 'unlabeled-error', language: 'es' }, { sharedId: 'unlabeled-no-suggestion', language: 'en' }, { sharedId: 'unlabeled-no-suggestion', language: 'es' }, + { sharedId: 'unlabeled-error', language: 'en' }, + { sharedId: 'unlabeled-error', language: 'es' }, ], }, { description: 'filtering for OR combinations like: labeled OR match', customFilter: { ...blankCustomFilter, labeled: true, match: true }, expectedSuggestions: [ - { sharedId: 'unlabeled-obsolete', language: 'en' }, - { sharedId: 'unlabeled-obsolete', language: 'es' }, { sharedId: 'labeled-match', language: 'en' }, { sharedId: 'labeled-match', language: 'es' }, { sharedId: 'labeled-mismatch', language: 'en' }, { sharedId: 'labeled-mismatch', language: 'es' }, - { sharedId: 'unlabeled-error', language: 'en' }, - { sharedId: 'unlabeled-error', language: 'es' }, { sharedId: 'unlabeled-no-context', language: 'en' }, { sharedId: 'unlabeled-no-context', language: 'es' }, ], @@ -166,14 +154,10 @@ describe('suggestions with CustomFilters', () => { 'filtering for OR combinations of complimentary filters like: labeled OR nonLabeled, which would result in all suggestions', customFilter: { ...blankCustomFilter, labeled: true, nonLabeled: true }, expectedSuggestions: [ - { sharedId: 'unlabeled-obsolete', language: 'en' }, - { sharedId: 'unlabeled-obsolete', language: 'es' }, { sharedId: 'labeled-match', language: 'en' }, { sharedId: 'labeled-match', language: 'es' }, { sharedId: 'labeled-mismatch', language: 'en' }, { sharedId: 'labeled-mismatch', language: 'es' }, - { sharedId: 'unlabeled-error', language: 'en' }, - { sharedId: 'unlabeled-error', language: 'es' }, { sharedId: 'unlabeled-no-context', language: 'en' }, { sharedId: 'unlabeled-no-context', language: 'es' }, { sharedId: 'unlabeled-no-suggestion', language: 'en' }, diff --git a/app/api/suggestions/specs/routes.spec.ts b/app/api/suggestions/specs/routes.spec.ts index 6a36545b84..e886729532 100644 --- a/app/api/suggestions/specs/routes.spec.ts +++ b/app/api/suggestions/specs/routes.spec.ts @@ -415,11 +415,12 @@ describe('aggregation routes', () => { extractorId: factory.id('test_extractor').toString(), }) .expect(200); + expect(response.body).toEqual({ total: 14, labeled: 4, - nonLabeled: 10, - match: 10, + nonLabeled: 6, + match: 6, mismatch: 4, obsolete: 2, error: 2, diff --git a/app/api/suggestions/suggestions.ts b/app/api/suggestions/suggestions.ts index 2793beeb3b..c545c38432 100644 --- a/app/api/suggestions/suggestions.ts +++ b/app/api/suggestions/suggestions.ts @@ -222,12 +222,40 @@ const Suggestions = { $group: { _id: null, total: { $sum: 1 }, - labeled: { $sum: { $cond: ['$state.labeled', 1, 0] } }, - nonLabeled: { $sum: { $cond: [{ $not: '$state.labeled' }, 1, 0] } }, - match: { $sum: { $cond: ['$state.match', 1, 0] } }, - mismatch: { $sum: { $cond: [{ $not: '$state.match' }, 1, 0] } }, - obsolete: { $sum: { $cond: ['$state.obsolete', 1, 0] } }, - error: { $sum: { $cond: ['$state.error', 1, 0] } }, + labeled: { + $sum: { + $cond: [{ $and: [{ $ne: ['$state.labeled', undefined] }, '$state.labeled'] }, 1, 0], + }, + }, + nonLabeled: { + $sum: { + $cond: [ + { $and: [{ $ne: ['$state.labeled', undefined] }, { $not: '$state.labeled' }] }, + 1, + 0, + ], + }, + }, + match: { + $sum: { + $cond: [{ $and: [{ $ne: ['$state.match', undefined] }, '$state.match'] }, 1, 0], + }, + }, + mismatch: { + $sum: { + $cond: [ + { $and: [{ $ne: ['$state.match', undefined] }, { $not: '$state.match' }] }, + 1, + 0, + ], + }, + }, + obsolete: { + $sum: { $cond: ['$state.obsolete', 1, 0] }, + }, + error: { + $sum: { $cond: ['$state.error', 1, 0] }, + }, }, }, ]); From 3d9887334b4d66b9dae723698f2e91e46464de64 Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Tue, 17 Sep 2024 13:13:37 -0500 Subject: [PATCH 18/26] Improved the conditions. --- app/api/suggestions/suggestions.ts | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/app/api/suggestions/suggestions.ts b/app/api/suggestions/suggestions.ts index c545c38432..0453fba71b 100644 --- a/app/api/suggestions/suggestions.ts +++ b/app/api/suggestions/suggestions.ts @@ -222,11 +222,7 @@ const Suggestions = { $group: { _id: null, total: { $sum: 1 }, - labeled: { - $sum: { - $cond: [{ $and: [{ $ne: ['$state.labeled', undefined] }, '$state.labeled'] }, 1, 0], - }, - }, + labeled: { $sum: { $cond: ['$state.labeled', 1, 0] } }, nonLabeled: { $sum: { $cond: [ @@ -236,11 +232,7 @@ const Suggestions = { ], }, }, - match: { - $sum: { - $cond: [{ $and: [{ $ne: ['$state.match', undefined] }, '$state.match'] }, 1, 0], - }, - }, + match: { $sum: { $cond: ['$state.match', 1, 0] } }, mismatch: { $sum: { $cond: [ @@ -250,12 +242,8 @@ const Suggestions = { ], }, }, - obsolete: { - $sum: { $cond: ['$state.obsolete', 1, 0] }, - }, - error: { - $sum: { $cond: ['$state.error', 1, 0] }, - }, + obsolete: { $sum: { $cond: ['$state.obsolete', 1, 0] } }, + error: { $sum: { $cond: ['$state.error', 1, 0] } }, }, }, ]); From 839202131509afe55704b30d466e3f98db673cae Mon Sep 17 00:00:00 2001 From: A happy cat Date: Wed, 18 Sep 2024 16:40:56 +0200 Subject: [PATCH 19/26] showing error and obsolete in the statuses --- app/react/App/styles/globals.css | 92 ++++++++----------- .../Settings/IX/components/TableElements.tsx | 80 ++++++++++------ 2 files changed, 90 insertions(+), 82 deletions(-) diff --git a/app/react/App/styles/globals.css b/app/react/App/styles/globals.css index dca95d058a..cdef6fd274 100644 --- a/app/react/App/styles/globals.css +++ b/app/react/App/styles/globals.css @@ -1,5 +1,5 @@ /* -! tailwindcss v3.4.1 | MIT License | https://tailwindcss.com +! tailwindcss v3.4.10 | MIT License | https://tailwindcss.com */ /* @@ -211,6 +211,8 @@ textarea { /* 1 */ line-height: inherit; /* 1 */ + letter-spacing: inherit; + /* 1 */ color: inherit; /* 1 */ margin: 0; @@ -234,9 +236,9 @@ select { */ button, -[type='button'], -[type='reset'], -[type='submit'] { +input:where([type='button']), +input:where([type='reset']), +input:where([type='submit']) { -webkit-appearance: button; /* 1 */ background-color: transparent; @@ -933,6 +935,10 @@ input[type="range"]::-ms-fill-lower { --tw-backdrop-opacity: ; --tw-backdrop-saturate: ; --tw-backdrop-sepia: ; + --tw-contain-size: ; + --tw-contain-layout: ; + --tw-contain-paint: ; + --tw-contain-style: ; } ::backdrop { @@ -983,6 +989,10 @@ input[type="range"]::-ms-fill-lower { --tw-backdrop-opacity: ; --tw-backdrop-saturate: ; --tw-backdrop-sepia: ; + --tw-contain-size: ; + --tw-contain-layout: ; + --tw-contain-paint: ; + --tw-contain-style: ; } .container { @@ -1730,11 +1740,6 @@ input[type="range"]::-ms-fill-lower { margin-right: 0.5rem; } -.mx-4 { - margin-left: 1rem; - margin-right: 1rem; -} - .mx-auto { margin-left: auto; margin-right: auto; @@ -1750,10 +1755,6 @@ input[type="range"]::-ms-fill-lower { margin-bottom: 1rem; } -.-ml-0 { - margin-left: -0px; -} - .-ml-0\.5 { margin-left: -0.125rem; } @@ -2402,12 +2403,6 @@ input[type="range"]::-ms-fill-lower { gap: 2rem; } -.space-x-0 > :not([hidden]) ~ :not([hidden]) { - --tw-space-x-reverse: 0; - margin-right: calc(0px * var(--tw-space-x-reverse)); - margin-left: calc(0px * calc(1 - var(--tw-space-x-reverse))); -} - .space-x-0\.5 > :not([hidden]) ~ :not([hidden]) { --tw-space-x-reverse: 0; margin-right: calc(0.125rem * var(--tw-space-x-reverse)); @@ -2432,12 +2427,6 @@ input[type="range"]::-ms-fill-lower { margin-left: calc(1rem * calc(1 - var(--tw-space-x-reverse))); } -.space-y-1 > :not([hidden]) ~ :not([hidden]) { - --tw-space-y-reverse: 0; - margin-top: calc(0.25rem * calc(1 - var(--tw-space-y-reverse))); - margin-bottom: calc(0.25rem * var(--tw-space-y-reverse)); -} - .space-y-3 > :not([hidden]) ~ :not([hidden]) { --tw-space-y-reverse: 0; margin-top: calc(0.75rem * calc(1 - var(--tw-space-y-reverse))); @@ -2844,11 +2833,6 @@ input[type="range"]::-ms-fill-lower { background-color: rgb(222 247 236 / var(--tw-bg-opacity)); } -.bg-green-200 { - --tw-bg-opacity: 1; - background-color: rgb(188 240 218 / var(--tw-bg-opacity)); -} - .bg-green-400 { --tw-bg-opacity: 1; background-color: rgb(49 196 141 / var(--tw-bg-opacity)); @@ -3317,11 +3301,6 @@ input[type="range"]::-ms-fill-lower { color: rgb(209 213 219 / var(--tw-text-opacity)) !important; } -.text-black { - --tw-text-opacity: 1; - color: rgb(0 0 0 / var(--tw-text-opacity)); -} - .text-blue-600 { --tw-text-opacity: 1; color: rgb(79 70 229 / var(--tw-text-opacity)); @@ -3332,6 +3311,11 @@ input[type="range"]::-ms-fill-lower { color: rgb(55 48 163 / var(--tw-text-opacity)); } +.text-error-500 { + --tw-text-opacity: 1; + color: rgb(236 72 153 / var(--tw-text-opacity)); +} + .text-error-600 { --tw-text-opacity: 1; color: rgb(219 39 119 / var(--tw-text-opacity)); @@ -4134,97 +4118,97 @@ input[type="range"]::-ms-fill-lower { display: none; } -:is(.dark .dark\:border-gray-600) { +.dark\:border-gray-600:is(.dark *) { --tw-border-opacity: 1; border-color: rgb(75 85 99 / var(--tw-border-opacity)); } -:is(.dark .dark\:\!bg-primary-600) { +.dark\:\!bg-primary-600:is(.dark *) { --tw-bg-opacity: 1 !important; background-color: rgb(79 70 229 / var(--tw-bg-opacity)) !important; } -:is(.dark .dark\:\!bg-primary-700) { +.dark\:\!bg-primary-700:is(.dark *) { --tw-bg-opacity: 1 !important; background-color: rgb(67 56 202 / var(--tw-bg-opacity)) !important; } -:is(.dark .dark\:bg-blue-600) { +.dark\:bg-blue-600:is(.dark *) { --tw-bg-opacity: 1; background-color: rgb(79 70 229 / var(--tw-bg-opacity)); } -:is(.dark .dark\:bg-gray-600) { +.dark\:bg-gray-600:is(.dark *) { --tw-bg-opacity: 1; background-color: rgb(75 85 99 / var(--tw-bg-opacity)); } -:is(.dark .dark\:bg-gray-700) { +.dark\:bg-gray-700:is(.dark *) { --tw-bg-opacity: 1; background-color: rgb(55 65 81 / var(--tw-bg-opacity)); } -:is(.dark .dark\:bg-gray-800) { +.dark\:bg-gray-800:is(.dark *) { --tw-bg-opacity: 1; background-color: rgb(31 41 55 / var(--tw-bg-opacity)); } -:is(.dark .dark\:text-error-400) { +.dark\:text-error-400:is(.dark *) { --tw-text-opacity: 1; color: rgb(244 114 182 / var(--tw-text-opacity)); } -:is(.dark .dark\:text-gray-400) { +.dark\:text-gray-400:is(.dark *) { --tw-text-opacity: 1; color: rgb(156 163 175 / var(--tw-text-opacity)); } -:is(.dark .dark\:text-gray-500) { +.dark\:text-gray-500:is(.dark *) { --tw-text-opacity: 1; color: rgb(107 114 128 / var(--tw-text-opacity)); } -:is(.dark .dark\:text-white) { +.dark\:text-white:is(.dark *) { --tw-text-opacity: 1; color: rgb(255 255 255 / var(--tw-text-opacity)); } -:is(.dark .dark\:placeholder-gray-400)::-moz-placeholder { +.dark\:placeholder-gray-400:is(.dark *)::-moz-placeholder { --tw-placeholder-opacity: 1; color: rgb(156 163 175 / var(--tw-placeholder-opacity)); } -:is(.dark .dark\:placeholder-gray-400)::placeholder { +.dark\:placeholder-gray-400:is(.dark *)::placeholder { --tw-placeholder-opacity: 1; color: rgb(156 163 175 / var(--tw-placeholder-opacity)); } -:is(.dark .dark\:hover\:\!bg-primary-700:hover) { +.dark\:hover\:\!bg-primary-700:hover:is(.dark *) { --tw-bg-opacity: 1 !important; background-color: rgb(67 56 202 / var(--tw-bg-opacity)) !important; } -:is(.dark .dark\:hover\:bg-blue-700:hover) { +.dark\:hover\:bg-blue-700:hover:is(.dark *) { --tw-bg-opacity: 1; background-color: rgb(67 56 202 / var(--tw-bg-opacity)); } -:is(.dark .dark\:hover\:bg-gray-600:hover) { +.dark\:hover\:bg-gray-600:hover:is(.dark *) { --tw-bg-opacity: 1; background-color: rgb(75 85 99 / var(--tw-bg-opacity)); } -:is(.dark .dark\:hover\:text-white:hover) { +.dark\:hover\:text-white:hover:is(.dark *) { --tw-text-opacity: 1; color: rgb(255 255 255 / var(--tw-text-opacity)); } -:is(.dark .dark\:focus\:border-blue-500:focus) { +.dark\:focus\:border-blue-500:focus:is(.dark *) { --tw-border-opacity: 1; border-color: rgb(99 102 241 / var(--tw-border-opacity)); } -:is(.dark .dark\:focus\:ring-blue-500:focus) { +.dark\:focus\:ring-blue-500:focus:is(.dark *) { --tw-ring-opacity: 1; --tw-ring-color: rgb(99 102 241 / var(--tw-ring-opacity)); } diff --git a/app/react/V2/Routes/Settings/IX/components/TableElements.tsx b/app/react/V2/Routes/Settings/IX/components/TableElements.tsx index c65c11786a..70fa46031b 100644 --- a/app/react/V2/Routes/Settings/IX/components/TableElements.tsx +++ b/app/react/V2/Routes/Settings/IX/components/TableElements.tsx @@ -77,6 +77,44 @@ const PropertyCell = ({ cell }: CellContext { + const suggestions = suggestion.subRows; + const ammountOfSuggestions = suggestions.length; + const amountOfValues = suggestions.filter(s => s.currentValue).length; + const amountOfMatches = suggestions.filter(s => s.currentValue === s.suggestedValue).length; + const amountOfMissmatches = ammountOfSuggestions - amountOfMatches; + + return ( +
+ + {amountOfValues} values + + | + + {ammountOfSuggestions} suggestions + + {amountOfMatches > 0 && ( + <> + | + + {amountOfMatches}{' '} + matching + + + )} + {amountOfMissmatches > 0 && ( + <> + | + + {amountOfMissmatches}{' '} + mismatching + + + )} +
+ ); +}; + const CurrentValueCell = ({ cell, allProperties, @@ -87,43 +125,29 @@ const CurrentValueCell = ({ >; allProperties: ClientPropertySchema[]; }) => { - if ('subRows' in cell.row.original) { - const suggestions = cell.row.original.subRows; - const ammountOfSuggestions = suggestions.length; - const amountOfValues = suggestions.filter(suggestion => suggestion.currentValue).length; - const amountOfMatches = suggestions.filter(s => s.currentValue === s.suggestedValue).length; - const amountOfMissmatches = ammountOfSuggestions - amountOfMatches; - + if (cell.row.original.state.obsolete) { return (
- {amountOfValues} values + Obsolete - | +
+ ); + } + + if (cell.row.original.state.error) { + return ( +
- {ammountOfSuggestions} suggestions + Error - {amountOfMatches > 0 && ( - <> - | - - {amountOfMatches}{' '} - matching - - - )} - {amountOfMissmatches > 0 && ( - <> - | - - {amountOfMissmatches}{' '} - mismatching - - - )}
); } + + if ('subRows' in cell.row.original) { + return ; + } return ( Date: Fri, 20 Sep 2024 13:07:15 +0200 Subject: [PATCH 20/26] changed filters structure --- .../V2/Routes/Settings/IX/components/FiltersSidepanel.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/react/V2/Routes/Settings/IX/components/FiltersSidepanel.tsx b/app/react/V2/Routes/Settings/IX/components/FiltersSidepanel.tsx index b8b9d79c5a..8f96df6399 100644 --- a/app/react/V2/Routes/Settings/IX/components/FiltersSidepanel.tsx +++ b/app/react/V2/Routes/Settings/IX/components/FiltersSidepanel.tsx @@ -111,6 +111,8 @@ const FiltersSidepanel = ({
{aggregation.nonLabeled}
+ +
Match} @@ -133,6 +135,8 @@ const FiltersSidepanel = ({
{aggregation.mismatch}
+ +
Obsolete} From 030dd2eeab52d9519c933efd76ec421b8d5dbe50 Mon Sep 17 00:00:00 2001 From: A happy cat Date: Fri, 20 Sep 2024 13:13:29 +0200 Subject: [PATCH 21/26] updated translations --- contents/ui-translations/ar.csv | 3 --- contents/ui-translations/en.csv | 3 --- contents/ui-translations/es.csv | 3 --- contents/ui-translations/fr.csv | 3 --- contents/ui-translations/ko.csv | 3 --- contents/ui-translations/my.csv | 3 --- contents/ui-translations/ru.csv | 3 --- contents/ui-translations/th.csv | 3 --- contents/ui-translations/tr.csv | 3 --- 9 files changed, 27 deletions(-) diff --git a/contents/ui-translations/ar.csv b/contents/ui-translations/ar.csv index 5c6ff8644d..f8c514d8dc 100644 --- a/contents/ui-translations/ar.csv +++ b/contents/ui-translations/ar.csv @@ -14,8 +14,6 @@ Account locked,Account locked Account locked. Check your email to unlock.,Account locked. Check your email to unlock. Account unlocked successfully,Account unlocked successfully Account updated,Account updated -Accuracy,Accuracy -Accuracy (labeled data),Accuracy (labeled data) Action,عمل/ إجراء Activate,Activate Activated,Activated @@ -943,7 +941,6 @@ Welcome to Uwazi,مرحباً بكم في أوازي will be updated with the same value.,سيحدث بالقيمة نفسها with,مع With great power comes great responsibility!,كلما ازدادت القوة، ازدادت معها المسؤولية! -With suggestion,With suggestion x less,X أقل x more,X أكثر Year,سنة diff --git a/contents/ui-translations/en.csv b/contents/ui-translations/en.csv index a88651bb77..d67ee5fcd5 100644 --- a/contents/ui-translations/en.csv +++ b/contents/ui-translations/en.csv @@ -14,8 +14,6 @@ Account locked,Account locked Account locked. Check your email to unlock.,Account locked. Check your email to unlock. Account unlocked successfully,Account unlocked successfully Account updated,Account updated -Accuracy,Accuracy -Accuracy (labeled data),Accuracy (labeled data) Action,Action Activate,Activate Activated,Activated @@ -946,7 +944,6 @@ Welcome to Uwazi,Welcome to Uwazi will be updated with the same value.,will be updated with the same value. with,with With great power comes great responsibility!,With great power comes great responsibility! -With suggestion,With suggestion x less,less x more,more Year,Year diff --git a/contents/ui-translations/es.csv b/contents/ui-translations/es.csv index fa6964b75a..2062a10f82 100644 --- a/contents/ui-translations/es.csv +++ b/contents/ui-translations/es.csv @@ -14,8 +14,6 @@ Account locked,Account locked Account locked. Check your email to unlock.,Cuenta bloqueada. Revisa tu correo electrónico para desbloquearla. Account unlocked successfully,Cuenta desbloqueada satisfactoriamente Account updated,Account updated -Accuracy,Precisión -Accuracy (labeled data),Accuracy (labeled data) Action,Acción Activate,Activate Activated,Activated @@ -941,7 +939,6 @@ Welcome to Uwazi,Bienvenido a Uwazi will be updated with the same value.,se actualizará con el mismo valor with,con With great power comes great responsibility!,"¡Con un gran poder, viene una gran responsabilidad!" -With suggestion,With suggestion x less,menos x more,más Year,Año diff --git a/contents/ui-translations/fr.csv b/contents/ui-translations/fr.csv index 179405940f..09b362f5ef 100644 --- a/contents/ui-translations/fr.csv +++ b/contents/ui-translations/fr.csv @@ -14,8 +14,6 @@ Account locked,Account locked Account locked. Check your email to unlock.,Account locked. Check your email to unlock. Account unlocked successfully,Account unlocked successfully Account updated,Account updated -Accuracy,Accuracy -Accuracy (labeled data),Accuracy (labeled data) Action,Action Activate,Activate Activated,Activated @@ -943,7 +941,6 @@ Welcome to Uwazi,Bienvenue à Uwazi will be updated with the same value.,sera mis à jour avec la même valeur. with,avec With great power comes great responsibility!,De grands pouvoirs impliquent de grandes responsabilités ! -With suggestion,With suggestion x less,x en moins x more,x en plus Year,Année diff --git a/contents/ui-translations/ko.csv b/contents/ui-translations/ko.csv index 4c88129ba4..bc925a456d 100644 --- a/contents/ui-translations/ko.csv +++ b/contents/ui-translations/ko.csv @@ -14,8 +14,6 @@ Account locked,Account locked Account locked. Check your email to unlock.,Account locked. Check your email to unlock. Account unlocked successfully,Account unlocked successfully Account updated,Account updated -Accuracy,Accuracy -Accuracy (labeled data),Accuracy (labeled data) Action,활동 Activate,Activate Activated,Activated @@ -944,7 +942,6 @@ Welcome to Uwazi,Uwazi 에 오신 것을 환영합니다. will be updated with the same value.,동일 값으로 업데이트됩니다. with,with With great power comes great responsibility!,큰 일에는 큰 책임이 따르는 법! -With suggestion,With suggestion x less,덜 보기 x more,더 보기 Year,연도 diff --git a/contents/ui-translations/my.csv b/contents/ui-translations/my.csv index 80024ff6a9..2cf8bd28b4 100644 --- a/contents/ui-translations/my.csv +++ b/contents/ui-translations/my.csv @@ -14,8 +14,6 @@ Account locked,Account locked Account locked. Check your email to unlock.,Account locked. Check your email to unlock. Account unlocked successfully,Account unlocked successfully Account updated,Account updated -Accuracy,Accuracy -Accuracy (labeled data),Accuracy (labeled data) Action,လုပ်ဆောင်ချက် Activate,Activate Activated,Activated @@ -944,7 +942,6 @@ Welcome to Uwazi,Uwazi မှ ကြိုဆိုပါသည် will be updated with the same value.,တူညီသော တန်ဖိုးဖြင့် အပ်ဒိတ်လုပ်ပါမည်။ with,ဖြင့် With great power comes great responsibility!,လုပ်ပိုင်ခွင့်ကြီးလျှင် တာဝန်ပိုကြီးသည်! -With suggestion,With suggestion x less,x လျော့ x more,x ပို Year,နှစ် diff --git a/contents/ui-translations/ru.csv b/contents/ui-translations/ru.csv index d762661471..06bd54b168 100644 --- a/contents/ui-translations/ru.csv +++ b/contents/ui-translations/ru.csv @@ -14,8 +14,6 @@ Account locked,Account locked Account locked. Check your email to unlock.,Account locked. Check your email to unlock. Account unlocked successfully,Account unlocked successfully Account updated,Account updated -Accuracy,Accuracy -Accuracy (labeled data),Accuracy (labeled data) Action,Действие Activate,Activate Activated,Activated @@ -941,7 +939,6 @@ Welcome to Uwazi,Добро пожаловать в Uwazi will be updated with the same value.,будет обновляться с тем же значением with,с With great power comes great responsibility!,С большой силой приходит большая ответственность! -With suggestion,With suggestion x less,x меньше x more,х больше Year,Год diff --git a/contents/ui-translations/th.csv b/contents/ui-translations/th.csv index 386ccdf887..df41947229 100644 --- a/contents/ui-translations/th.csv +++ b/contents/ui-translations/th.csv @@ -14,8 +14,6 @@ Account locked,Account locked Account locked. Check your email to unlock.,Account locked. Check your email to unlock. Account unlocked successfully,Account unlocked successfully Account updated,Account updated -Accuracy,Accuracy -Accuracy (labeled data),Accuracy (labeled data) Action,เริ่มทำ Activate,Activate Activated,Activated @@ -944,7 +942,6 @@ Welcome to Uwazi,ยินดีต้อนรับสู่ Uwazi will be updated with the same value.,จะถูกอัปเดทด้วยค่าเดียวกัน with,กับ With great power comes great responsibility!,พลังที่ยิ่งใหญ่มากับความรับผิดชอบที่ใหญ่ยิ่ง! -With suggestion,With suggestion x less,ลดขนาดลง x more,เพิ่มขนาดขึ้น Year,ปี diff --git a/contents/ui-translations/tr.csv b/contents/ui-translations/tr.csv index d6767488bf..dcf4bcff67 100644 --- a/contents/ui-translations/tr.csv +++ b/contents/ui-translations/tr.csv @@ -14,8 +14,6 @@ Account locked,Account locked Account locked. Check your email to unlock.,Account locked. Check your email to unlock. Account unlocked successfully,Account unlocked successfully Account updated,Account updated -Accuracy,Accuracy -Accuracy (labeled data),Accuracy (labeled data) Action,Eylem Activate,Activate Activated,Activated @@ -944,7 +942,6 @@ Welcome to Uwazi,Uwazi'ye hoş geldiniz will be updated with the same value.,aynı değerle güncellenecektir. with,ile With great power comes great responsibility!,Büyük güç büyük sorumluluk getirir! -With suggestion,With suggestion x less,daha az x more,daha az Year,Yıl From 2459660dede95a603896b78a8ca761e85285f236 Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Mon, 23 Sep 2024 15:21:07 -0500 Subject: [PATCH 22/26] Added logic for processing to refelct obsolete --- app/shared/getIXSuggestionState.ts | 7 ++--- app/shared/specs/getIXSuggestionState.spec.ts | 29 +++++++++++++++++++ app/shared/types/suggestionSchema.ts | 10 ++++++- app/shared/types/suggestionType.d.ts | 2 +- 4 files changed, 42 insertions(+), 6 deletions(-) diff --git a/app/shared/getIXSuggestionState.ts b/app/shared/getIXSuggestionState.ts index 7986d85b28..e45dfa2b34 100644 --- a/app/shared/getIXSuggestionState.ts +++ b/app/shared/getIXSuggestionState.ts @@ -40,7 +40,7 @@ const equalsForType = (type: PropertySchema['type']) => (first: any, second: any EQUALITIES[type] ? EQUALITIES[type](first, second) : first === second; class IXSuggestionState implements IXSuggestionStateType { - labeled: boolean | undefined; + labeled = false; withValue = false; @@ -71,7 +71,6 @@ class IXSuggestionState implements IXSuggestionStateType { { labeledValue, currentValue }: SuggestionValues, propertyType: PropertySchema['type'] ) { - this.labeled = false; if ( labeledValue || (propertyIsSelect(propertyType) && currentValue) || @@ -129,7 +128,6 @@ class IXSuggestionState implements IXSuggestionStateType { setObsolete({ modelCreationDate, date }: SuggestionValues) { if (date < modelCreationDate) { this.obsolete = true; - this.labeled = undefined; this.match = undefined; } } @@ -137,13 +135,14 @@ class IXSuggestionState implements IXSuggestionStateType { setProcessing({ status }: SuggestionValues) { if (status === 'processing') { this.processing = true; + this.obsolete = true; + this.match = undefined; } } setError({ error, status }: SuggestionValues) { if ((error && error !== '') || (status && status === 'failed')) { this.error = true; - this.labeled = undefined; this.match = undefined; } } diff --git a/app/shared/specs/getIXSuggestionState.spec.ts b/app/shared/specs/getIXSuggestionState.spec.ts index 6bc10b84d8..1cf06008bc 100644 --- a/app/shared/specs/getIXSuggestionState.spec.ts +++ b/app/shared/specs/getIXSuggestionState.spec.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-statements */ import { getSuggestionState, SuggestionValues } from '../getIXSuggestionState'; describe('getIXSuggestionState', () => { @@ -83,8 +84,10 @@ describe('getIXSuggestionState', () => { const state = getSuggestionState(values, 'text'); expect(state).toEqual({ + labeled: false, withValue: true, withSuggestion: false, + match: undefined, hasContext: false, obsolete: false, processing: false, @@ -219,13 +222,39 @@ describe('getIXSuggestionState', () => { const state = getSuggestionState(values, 'text'); expect(state).toEqual({ + labeled: false, withValue: false, withSuggestion: true, + match: undefined, hasContext: false, obsolete: true, processing: false, error: false, }); }); + + it('should mark processing when status is processing and set obsolete as true', () => { + const values = { + currentValue: '', + date: 1234, + labeledValue: '', + suggestedValue: 'some value', + modelCreationDate: 1, + status: 'processing', + }; + + const state = getSuggestionState(values, 'text'); + + expect(state).toEqual({ + labeled: false, + withValue: false, + withSuggestion: true, + match: undefined, + hasContext: false, + obsolete: true, + processing: true, + error: false, + }); + }); }); }); diff --git a/app/shared/types/suggestionSchema.ts b/app/shared/types/suggestionSchema.ts index 7d3568d8fa..e0ce40cf38 100644 --- a/app/shared/types/suggestionSchema.ts +++ b/app/shared/types/suggestionSchema.ts @@ -85,7 +85,15 @@ export const IXSuggestionStateSchema = { processing: { type: 'boolean' }, error: { type: 'boolean' }, }, - required: ['withValue', 'withSuggestion', 'hasContext', 'processing', 'obsolete', 'error'], + required: [ + 'labeled', + 'withValue', + 'withSuggestion', + 'hasContext', + 'processing', + 'obsolete', + 'error', + ], }; export const IXSuggestionSchema = { diff --git a/app/shared/types/suggestionType.d.ts b/app/shared/types/suggestionType.d.ts index a42a738dfa..be4626461a 100644 --- a/app/shared/types/suggestionType.d.ts +++ b/app/shared/types/suggestionType.d.ts @@ -81,7 +81,7 @@ export interface IXSuggestionType { } export interface IXSuggestionStateType { - labeled?: boolean; + labeled: boolean; withValue: boolean; withSuggestion: boolean; match?: boolean; From f585b5e2248ae9700f9bc3ed700347411af6a414 Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Fri, 18 Oct 2024 12:04:15 -0500 Subject: [PATCH 23/26] Replaced text to show new logic --- .../specs/InformationExtraction.spec.ts | 6 +++--- app/api/utils/fixturesFactory.ts | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/api/services/informationextraction/specs/InformationExtraction.spec.ts b/app/api/services/informationextraction/specs/InformationExtraction.spec.ts index 34230e8764..d68681a7e4 100644 --- a/app/api/services/informationextraction/specs/InformationExtraction.spec.ts +++ b/app/api/services/informationextraction/specs/InformationExtraction.spec.ts @@ -759,12 +759,12 @@ describe('InformationExtraction', () => { status: 'processing', state: { labeled: true, + match: null, withValue: true, withSuggestion: true, - match: false, hasContext: true, processing: true, - obsolete: false, + obsolete: true, error: false, }, }) @@ -1011,9 +1011,9 @@ describe('InformationExtraction', () => { error: 'Issue calculation suggestion', state: { labeled: true, + match: null, withValue: true, withSuggestion: false, - match: false, hasContext: false, processing: false, obsolete: false, diff --git a/app/api/utils/fixturesFactory.ts b/app/api/utils/fixturesFactory.ts index d322d7b68a..9a63e5eec8 100644 --- a/app/api/utils/fixturesFactory.ts +++ b/app/api/utils/fixturesFactory.ts @@ -341,6 +341,7 @@ function getFixturesFactory() { suggestedValue: '', date: 1001, state: { + labeled: false, withValue: true, withSuggestion: false, hasContext: false, @@ -376,6 +377,7 @@ function getFixturesFactory() { suggestedValue: '', date: 1, state: { + labeled: false, withValue: true, withSuggestion: false, hasContext: false, From 5521ccc07e8c4c126da7822f071b8f9ebc82545f Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Fri, 18 Oct 2024 12:20:46 -0500 Subject: [PATCH 24/26] Fixed tests to reflect new logic --- app/api/suggestions/specs/customFilters.spec.ts | 16 ++++++++++++---- app/api/suggestions/specs/routes.spec.ts | 6 +++--- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/app/api/suggestions/specs/customFilters.spec.ts b/app/api/suggestions/specs/customFilters.spec.ts index 104b87bb34..045d17352a 100644 --- a/app/api/suggestions/specs/customFilters.spec.ts +++ b/app/api/suggestions/specs/customFilters.spec.ts @@ -38,10 +38,10 @@ describe('suggestions with CustomFilters', () => { { sharedId: 'labeled-mismatch', language: 'es' }, { sharedId: 'unlabeled-no-context', language: 'en' }, { sharedId: 'unlabeled-no-context', language: 'es' }, - { sharedId: 'unlabeled-no-suggestion', language: 'en' }, - { sharedId: 'unlabeled-no-suggestion', language: 'es' }, { sharedId: 'unlabeled-error', language: 'en' }, { sharedId: 'unlabeled-error', language: 'es' }, + { sharedId: 'unlabeled-no-suggestion', language: 'en' }, + { sharedId: 'unlabeled-no-suggestion', language: 'es' }, ]); }); @@ -73,8 +73,12 @@ describe('suggestions with CustomFilters', () => { description: 'filtering for nonLabeled', customFilter: { ...blankCustomFilter, nonLabeled: true }, expectedSuggestions: [ + { sharedId: 'unlabeled-obsolete', language: 'en' }, + { sharedId: 'unlabeled-obsolete', language: 'es' }, { sharedId: 'unlabeled-no-context', language: 'en' }, { sharedId: 'unlabeled-no-context', language: 'es' }, + { sharedId: 'unlabeled-error', language: 'en' }, + { sharedId: 'unlabeled-error', language: 'es' }, { sharedId: 'unlabeled-no-suggestion', language: 'en' }, { sharedId: 'unlabeled-no-suggestion', language: 'es' }, ], @@ -131,10 +135,10 @@ describe('suggestions with CustomFilters', () => { expectedSuggestions: [ { sharedId: 'labeled-mismatch', language: 'en' }, { sharedId: 'labeled-mismatch', language: 'es' }, - { sharedId: 'unlabeled-no-suggestion', language: 'en' }, - { sharedId: 'unlabeled-no-suggestion', language: 'es' }, { sharedId: 'unlabeled-error', language: 'en' }, { sharedId: 'unlabeled-error', language: 'es' }, + { sharedId: 'unlabeled-no-suggestion', language: 'en' }, + { sharedId: 'unlabeled-no-suggestion', language: 'es' }, ], }, { @@ -154,12 +158,16 @@ describe('suggestions with CustomFilters', () => { 'filtering for OR combinations of complimentary filters like: labeled OR nonLabeled, which would result in all suggestions', customFilter: { ...blankCustomFilter, labeled: true, nonLabeled: true }, expectedSuggestions: [ + { sharedId: 'unlabeled-obsolete', language: 'en' }, + { sharedId: 'unlabeled-obsolete', language: 'es' }, { sharedId: 'labeled-match', language: 'en' }, { sharedId: 'labeled-match', language: 'es' }, { sharedId: 'labeled-mismatch', language: 'en' }, { sharedId: 'labeled-mismatch', language: 'es' }, { sharedId: 'unlabeled-no-context', language: 'en' }, { sharedId: 'unlabeled-no-context', language: 'es' }, + { sharedId: 'unlabeled-error', language: 'en' }, + { sharedId: 'unlabeled-error', language: 'es' }, { sharedId: 'unlabeled-no-suggestion', language: 'en' }, { sharedId: 'unlabeled-no-suggestion', language: 'es' }, ], diff --git a/app/api/suggestions/specs/routes.spec.ts b/app/api/suggestions/specs/routes.spec.ts index e886729532..e5d92fcc7c 100644 --- a/app/api/suggestions/specs/routes.spec.ts +++ b/app/api/suggestions/specs/routes.spec.ts @@ -419,10 +419,10 @@ describe('aggregation routes', () => { expect(response.body).toEqual({ total: 14, labeled: 4, - nonLabeled: 6, - match: 6, + nonLabeled: 10, + match: 4, mismatch: 4, - obsolete: 2, + obsolete: 4, error: 2, }); }); From ff97ec252122a743f1d2a21abfd25691ed4b61a8 Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Fri, 18 Oct 2024 12:29:23 -0500 Subject: [PATCH 25/26] Added condition for null to be excluded. --- app/api/suggestions/suggestions.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/app/api/suggestions/suggestions.ts b/app/api/suggestions/suggestions.ts index 0453fba71b..3f57f55fd7 100644 --- a/app/api/suggestions/suggestions.ts +++ b/app/api/suggestions/suggestions.ts @@ -226,7 +226,13 @@ const Suggestions = { nonLabeled: { $sum: { $cond: [ - { $and: [{ $ne: ['$state.labeled', undefined] }, { $not: '$state.labeled' }] }, + { + $and: [ + { $ne: ['$state.labeled', undefined] }, + { $ne: ['$state.labeled', null] }, + { $not: '$state.labeled' }, + ], + }, 1, 0, ], @@ -236,7 +242,13 @@ const Suggestions = { mismatch: { $sum: { $cond: [ - { $and: [{ $ne: ['$state.match', undefined] }, { $not: '$state.match' }] }, + { + $and: [ + { $ne: ['$state.match', undefined] }, + { $ne: ['$state.match', null] }, + { $not: '$state.match' }, + ], + }, 1, 0, ], From 6371aac6fa8d327c3affb3c89349f5562ca6f0c7 Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Fri, 18 Oct 2024 14:44:32 -0500 Subject: [PATCH 26/26] Reinstated document implementation --- app/api/utils/fixturesFactory.ts | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/app/api/utils/fixturesFactory.ts b/app/api/utils/fixturesFactory.ts index 9a63e5eec8..cb6e411490 100644 --- a/app/api/utils/fixturesFactory.ts +++ b/app/api/utils/fixturesFactory.ts @@ -166,20 +166,14 @@ function getFixturesFactory() { }, }), - document: (entity: string, props: Partial> = {}): WithId => ({ - _id: idMapper(`document_for_${entity}`), - entity, - language: 'en', - type: 'document', - filename: `${entity}_document.pdf`, - originalname: `${entity}_document.pdf`, - ...props, - }), - attachment(id: string, extra: Partial = {}): WithId { return this.file(id, { ...extra, type: 'attachment' }); }, + document(id: string, extra: Partial = {}): WithId { + return this.file(id, { ...extra, type: 'document' }); + }, + custom_upload(id: string, extra: Partial = {}): WithId { return this.file(id, { ...extra, type: 'custom' }); },