diff --git a/.github/workflows/ci_check_translations.yml b/.github/workflows/ci_check_translations.yml index 9f0a3cd2d3..6c0742501c 100644 --- a/.github/workflows/ci_check_translations.yml +++ b/.github/workflows/ci_check_translations.yml @@ -40,6 +40,14 @@ jobs: - name: install dependencies run: yarn install - run: yarn blank-state + id: blank-state-attempt1 + continue-on-error: true + env: + DBHOST: localhost:27017 + ELASTICSEARCH_URL: http://localhost:${{ job.services.elasticsearch.ports[9200] }} + - run: yarn blank-state --force + id: blank-state-attempt2 + if: steps.blank-state-attempt1.outcome == 'failure' env: DBHOST: localhost:27017 ELASTICSEARCH_URL: http://localhost:${{ job.services.elasticsearch.ports[9200] }} diff --git a/.github/workflows/ci_e2e_cypress.yml b/.github/workflows/ci_e2e_cypress.yml index 70f9caa62f..11d45e34bb 100644 --- a/.github/workflows/ci_e2e_cypress.yml +++ b/.github/workflows/ci_e2e_cypress.yml @@ -82,12 +82,23 @@ jobs: if: ${{ failure() }} run: cat dummy_extractor_services/logs.log - run: yarn blank-state + id: blank-state-attempt1 + continue-on-error: true env: DBHOST: localhost:27017 ELASTICSEARCH_URL: http://localhost:${{ job.services.elasticsearch.ports[9200] }} DATABASE_NAME: uwazi_e2e INDEX_NAME: uwazi_e2e TRANSPILED: true + - run: yarn blank-state --force + id: blank-state-attempt2 + if: steps.blank-state-attempt1.outcome == 'failure' + env: + DBHOST: localhost:27017 + ELASTICSEARCH_URL: http://localhost:${{ job.services.elasticsearch.ports[9200] }} + DATABASE_NAME: uwazi_e2e + INDEX_NAME: uwazi_e2e + TRANSPILED: true - run: yarn ix-config env: DBHOST: localhost:27017 diff --git a/README.md b/README.md index b0f8e2a6bd..145d034a0b 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Before anything else you will need to install the application dependencies: - **ElasticSearch 7.17.6** https://www.elastic.co/downloads/past-releases/elasticsearch-7-17-6 Please note that ElasticSearch requires Java. Follow the instructions to install the package manually, you also probably need to disable ml module in the ElasticSearch config file: `xpack.ml.enabled: false` - **ICU Analysis Plugin (recommended)** [installation](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-icu.html#analysis-icu) Adds support for number sorting in texts and solves other language sorting nuances. This option is activated by setting the env var USE_ELASTIC_ICU=true before running the server (defaults to false/unset). -- **MongoDB 5.0.27** https://docs.mongodb.com/v4.2/installation/ . If you have a previous version installed, please follow the instructions on how to [upgrade here](https://docs.mongodb.com/manual/release-notes/4.2-upgrade-standalone/). The new mongosh dependency needs to be added [installation](https://www.mongodb.com/docs/mongodb-shell/). +- **MongoDB 5.0.27** https://www.mongodb.com/docs/v5.0/installation/ . If you have a previous version installed, please follow the instructions on how to [upgrade here](https://www.mongodb.com/docs/manual/release-notes/5.0-upgrade-standalone/). The new mongosh dependency needs to be added [installation](https://www.mongodb.com/docs/mongodb-shell/). - **Yarn** https://yarnpkg.com/en/docs/install. - **pdftotext (Poppler)** tested to work on version 0.86 but it's recommended to use the latest available for your platform https://poppler.freedesktop.org/. Make sure to **install libjpeg-dev** if you build from source. diff --git a/app/api/auth/encryptPassword.js b/app/api/auth/encryptPassword.js index 6c1ce355c8..ced5cf7766 100644 --- a/app/api/auth/encryptPassword.js +++ b/app/api/auth/encryptPassword.js @@ -5,6 +5,4 @@ const saltRounds = 10; const encryptPassword = async plainPassword => bcrypt.hash(plainPassword, saltRounds); const comparePasswords = async (plain, hashed) => bcrypt.compare(plain, hashed); -export default encryptPassword; - -export { comparePasswords }; +export { comparePasswords, encryptPassword }; diff --git a/app/api/auth/index.js b/app/api/auth/index.js index 3e68603888..5814970a69 100644 --- a/app/api/auth/index.js +++ b/app/api/auth/index.js @@ -3,5 +3,5 @@ import captchaAuthorization from './captchaMiddleware'; import { CaptchaModel } from './CaptchaModel'; export { needsAuthorization, captchaAuthorization, CaptchaModel }; -export { default as encryptPassword } from './encryptPassword'; -export { comparePasswords } from './encryptPassword'; +export { comparePasswords, encryptPassword } from './encryptPassword'; +export { validatePasswordMiddleWare } from './validatePasswordMiddleWare'; diff --git a/app/api/auth/specs/validatePasswordMiddleWare.spec.ts b/app/api/auth/specs/validatePasswordMiddleWare.spec.ts new file mode 100644 index 0000000000..92d75fdf82 --- /dev/null +++ b/app/api/auth/specs/validatePasswordMiddleWare.spec.ts @@ -0,0 +1,122 @@ +import { Request, NextFunction, Response } from 'express'; +import { testingEnvironment } from 'api/utils/testingEnvironment'; +import { getFixturesFactory } from 'api/utils/fixturesFactory'; +import { UserSchema } from 'shared/types/userType'; +import { UserRole } from 'shared/types/userSchema'; +import { encryptPassword } from '../encryptPassword'; +import { validatePasswordMiddleWare } from '../validatePasswordMiddleWare'; + +const fixturesFactory = getFixturesFactory(); + +describe('validatePasswordMiddleWare', () => { + const mockResponse = (): Response => { + const res: Partial = {}; + res.status = jest.fn().mockReturnValue(res); + res.json = jest.fn().mockReturnValue(res); + return res as Response; + }; + + const res = mockResponse(); + const users: UserSchema[] = []; + const next: NextFunction = jest.fn(); + + const createRequest = (request: Partial): Request => + { + ...request, + user: request.user, + body: request.body, + headers: request.headers, + }; + + const gernerateFixtures = async () => { + users.push( + ...[ + { + ...fixturesFactory.user( + 'admin', + UserRole.ADMIN, + 'admin@test.com', + await encryptPassword('admin1234') + ), + }, + { + ...fixturesFactory.user( + 'editor', + UserRole.EDITOR, + 'editor@test.com', + await encryptPassword('editor1234') + ), + }, + ] + ); + + return { + users, + }; + }; + + beforeAll(async () => { + const fixtures = await gernerateFixtures(); + await testingEnvironment.setUp(fixtures); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + afterAll(async () => { + await testingEnvironment.tearDown(); + }); + + it('should reject when the password is incorrect', async () => { + const request = createRequest({ + user: { _id: users[0]._id, username: users[0].username, role: users[0].role }, + body: { username: 'a_new_user', role: 'collaborator', email: 'collaborator@huridocs.org' }, + headers: { authorization: 'Basic wrongPass' }, + }); + + await validatePasswordMiddleWare(request, res, next); + + expect(next).not.toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(403); + expect(res.json).toHaveBeenCalledWith({ message: 'Forbidden', error: 'Password error' }); + }); + + it('should reject when the password is empty', async () => { + const emptyPasswordRequest = createRequest({ + user: { _id: users[0]._id, username: users[0].username, role: users[0].role }, + body: { username: 'a_new_user', role: 'collaborator', email: 'collaborator@huridocs.org' }, + headers: { authorization: 'Basic ' }, + }); + + const noAuthHeaderRequest = createRequest({ + user: { _id: users[0]._id, username: users[0].username, role: users[0].role }, + body: { _id: users[0]._id, username: users[0].username, role: users[0].role }, + headers: { cookie: 'some cookie' }, + }); + + await validatePasswordMiddleWare(emptyPasswordRequest, res, next); + + expect(next).not.toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(403); + expect(res.json).toHaveBeenNthCalledWith(1, { message: 'Forbidden', error: 'Password error' }); + + await validatePasswordMiddleWare(noAuthHeaderRequest, res, next); + + expect(next).not.toHaveBeenCalled(); + expect(res.status).toHaveBeenCalledWith(403); + expect(res.json).toHaveBeenNthCalledWith(2, { message: 'Forbidden', error: 'Password error' }); + }); + + it('should succeed when the passwords match', async () => { + const request = createRequest({ + user: { _id: users[1]._id, username: users[1].username, role: users[1].role }, + body: { _id: users[1]._id, username: users[1].username, role: users[1].role }, + headers: { authorization: 'Basic editor1234' }, + }); + + await validatePasswordMiddleWare(request, res, next); + + expect(next).toHaveBeenCalled(); + }); +}); diff --git a/app/api/auth/validatePasswordMiddleWare.ts b/app/api/auth/validatePasswordMiddleWare.ts new file mode 100644 index 0000000000..73409ff59e --- /dev/null +++ b/app/api/auth/validatePasswordMiddleWare.ts @@ -0,0 +1,28 @@ +import { Request, Response, NextFunction } from 'express'; +import { User } from 'api/users/usersModel'; +import usersModel from '../users/users'; +import { comparePasswords } from './encryptPassword'; + +const validatePassword = async (submittedPassword: string, requestUser: User) => { + const user = await usersModel.getById(requestUser._id, '+password'); + const currentPassword = user.password; + return comparePasswords(submittedPassword, currentPassword); +}; + +const validatePasswordMiddleWare = async (req: Request, res: Response, next: NextFunction) => { + const { user, headers } = req; + const submittedPassword = headers?.authorization?.split('Basic ')[1]; + + if (submittedPassword) { + const validPassword = await validatePassword(submittedPassword, user); + + if (validPassword) { + return next(); + } + } + + res.status(403); + return res.json({ error: 'Password error', message: 'Forbidden' }); +}; + +export { validatePasswordMiddleWare }; diff --git a/app/api/auth2fa/routes.ts b/app/api/auth2fa/routes.ts index 1c1ffe20d8..acdb20faca 100644 --- a/app/api/auth2fa/routes.ts +++ b/app/api/auth2fa/routes.ts @@ -3,6 +3,7 @@ import needsAuthorization from 'api/auth/authMiddleware'; import * as usersUtils from 'api/auth2fa/usersUtils'; import { validation } from 'api/utils'; import { ObjectIdAsString } from 'api/utils/ajvSchemas'; +import { validatePasswordMiddleWare } from 'api/auth'; export default (app: Application) => { app.post( @@ -50,6 +51,7 @@ export default (app: Application) => { app.post( '/api/auth2fa-reset', needsAuthorization(['admin']), + validatePasswordMiddleWare, validation.validateRequest({ type: 'object', properties: { diff --git a/app/api/services/informationextraction/InformationExtraction.ts b/app/api/services/informationextraction/InformationExtraction.ts index b658c38830..b7bd04f7fa 100644 --- a/app/api/services/informationextraction/InformationExtraction.ts +++ b/app/api/services/informationextraction/InformationExtraction.ts @@ -29,6 +29,7 @@ import { FileWithAggregation, getFilesForTraining, getFilesForSuggestions, + propertyTypeIsWithoutExtractedMetadata, propertyTypeIsSelectOrMultiSelect, } from 'api/services/informationextraction/getFiles'; import { Suggestions } from 'api/suggestions/suggestions'; @@ -95,6 +96,16 @@ type MaterialsData = | TextSelectionMaterialsData | ValuesSelectionMaterialsData; +async function fetchCandidates(property: PropertySchema) { + const defaultLanguageKey = (await settings.getDefaultLanguage()).key; + const query: { template?: ObjectId; language: string } = { + language: defaultLanguageKey, + }; + if (property.content !== '') query.template = new ObjectId(property.content); + const candidates = await entities.getUnrestricted(query, ['title', 'sharedId']); + return candidates; +} + class InformationExtraction { static SERVICE_NAME = 'information_extraction'; @@ -142,9 +153,9 @@ class InformationExtraction { let data: MaterialsData = { ..._data, language_iso }; - const isSelect = propertyTypeIsSelectOrMultiSelect(propertyType); + const noExtractedData = propertyTypeIsWithoutExtractedMetadata(propertyType); - if (!isSelect && propertyLabeledData) { + if (!noExtractedData && propertyLabeledData) { data = { ...data, label_text: propertyValue || propertyLabeledData?.selection?.text, @@ -155,7 +166,7 @@ class InformationExtraction { }; } - if (isSelect) { + if (noExtractedData) { if (!Array.isArray(propertyValue)) { throw new Error('Property value should be an array'); } @@ -184,7 +195,7 @@ class InformationExtraction { ); const { propertyValue, propertyType } = file; - const missingData = propertyTypeIsSelectOrMultiSelect(propertyType) + const missingData = propertyTypeIsWithoutExtractedMetadata(propertyType) ? !propertyValue : type === 'labeled_data' && !propertyLabeledData; @@ -281,7 +292,10 @@ class InformationExtraction { templates, template => template.properties || [] ); - const property = allProps.find(p => p.name === extractor.property); + const property = + extractor.property === 'title' + ? { name: 'title' as 'title', type: 'title' as 'title' } + : allProps.find(p => p.name === extractor.property); const suggestion = await formatSuggestion( property, @@ -380,15 +394,22 @@ class InformationExtraction { const params: TaskParameters = { id: extractorId.toString(), - multi_value: property.type === 'multiselect', + multi_value: property.type === 'multiselect' || property.type === 'relationship', }; - if (property.type === 'select' || property.type === 'multiselect') { + if (propertyTypeIsSelectOrMultiSelect(property.type)) { const thesauri = await dictionatiesModel.getById(property.content); params.options = thesauri?.values?.map(value => ({ label: value.label, id: value.id as string })) || []; } + if (property.type === 'relationship') { + const candidates = await fetchCandidates(property); + params.options = candidates.map(candidate => ({ + label: candidate.title || '', + id: candidate.sharedId || '', + })); + } await this.taskManager.startTask({ task: 'create_model', diff --git a/app/api/services/informationextraction/getFiles.ts b/app/api/services/informationextraction/getFiles.ts index 8ddac8b566..9318260a7a 100644 --- a/app/api/services/informationextraction/getFiles.ts +++ b/app/api/services/informationextraction/getFiles.ts @@ -45,9 +45,16 @@ type FileEnforcedNotUndefined = { }; const selectProperties: Set = new Set([propertyTypes.select, propertyTypes.multiselect]); +const propertiesWithoutExtractedMetadata: Set = new Set([ + ...Array.from(selectProperties), + propertyTypes.relationship, +]); const propertyTypeIsSelectOrMultiSelect = (type: string) => selectProperties.has(type); +const propertyTypeIsWithoutExtractedMetadata = (type: string) => + propertiesWithoutExtractedMetadata.has(type); + async function getFilesWithAggregations(files: (FileType & FileEnforcedNotUndefined)[]) { const filesNames = files.filter(x => x.filename).map(x => x.filename); @@ -98,7 +105,7 @@ async function fileQuery( propertyType: string, entitiesFromTrainingTemplatesIds: string[] ) { - const needsExtractedMetadata = !propertyTypeIsSelectOrMultiSelect(propertyType); + const needsExtractedMetadata = !propertyTypeIsWithoutExtractedMetadata(propertyType); const query: { type: string; filename: { $exists: Boolean }; @@ -125,7 +132,7 @@ function entityForTrainingQuery( const query: { [key: string]: { $in?: ObjectIdSchema[]; $exists?: Boolean; $ne?: any[] }; } = { template: { $in: templates } }; - if (propertyTypeIsSelectOrMultiSelect(propertyType)) { + if (propertyTypeIsWithoutExtractedMetadata(propertyType)) { query[`metadata.${property}`] = { $exists: true, $ne: [] }; } return query; @@ -162,8 +169,8 @@ async function getFilesForTraining(templates: ObjectIdSchema[], property: string return { ...file, propertyType }; } - if (propertyTypeIsSelectOrMultiSelect(propertyType)) { - const propertyValue = (entity.metadata[property] || []).map(({ value, label }) => ({ + if (propertyTypeIsWithoutExtractedMetadata(propertyType)) { + const propertyValue = (entity.metadata?.[property] || []).map(({ value, label }) => ({ value: ensure(value), label: ensure(label), })); @@ -174,6 +181,8 @@ async function getFilesForTraining(templates: ObjectIdSchema[], property: string let stringValue: string; if (propertyType === propertyTypes.date) { stringValue = moment(value * 1000).format('YYYY-MM-DD'); + } else if (propertyType === propertyTypes.numeric) { + stringValue = value?.toString() || ''; } else { stringValue = value; } @@ -221,5 +230,6 @@ export { getFilesForSuggestions, getSegmentedFilesIds, propertyTypeIsSelectOrMultiSelect, + propertyTypeIsWithoutExtractedMetadata, }; export type { FileWithAggregation }; diff --git a/app/api/services/informationextraction/ixextractors.ts b/app/api/services/informationextraction/ixextractors.ts index 4817d6ef5d..1b6032e33f 100644 --- a/app/api/services/informationextraction/ixextractors.ts +++ b/app/api/services/informationextraction/ixextractors.ts @@ -8,17 +8,38 @@ import { createBlankSuggestionsForExtractor, createBlankSuggestionsForPartialExtractor, } from 'api/suggestions/blankSuggestions'; -import { propertyTypes } from 'shared/propertyTypes'; +import { Subset } from 'shared/tsUtils'; +import { PropertyTypeSchema } from 'shared/types/commonTypes'; import { IXExtractorModel as model } from './IXExtractorModel'; -const ALLOWED_PROPERTY_TYPES: (typeof propertyTypes)[keyof typeof propertyTypes][] = [ +type AllowedPropertyTypes = + | Subset< + PropertyTypeSchema, + 'text' | 'numeric' | 'date' | 'select' | 'multiselect' | 'relationship' + > + | 'title'; + +const ALLOWED_PROPERTY_TYPES: AllowedPropertyTypes[] = [ + 'title', 'text', 'numeric', 'date', 'select', 'multiselect', + 'relationship', ]; +const allowedTypeSet = new Set(ALLOWED_PROPERTY_TYPES); + +const typeIsAllowed = (type: string): type is AllowedPropertyTypes => allowedTypeSet.has(type); + +const checkTypeIsAllowed = (type: string) => { + if (!typeIsAllowed(type)) { + throw new Error('Property type not allowed.'); + } + return type; +}; + const templatePropertyExistenceCheck = async (propertyName: string, templateIds: string[]) => { const tArray = await templates.get({ _id: { $in: templateIds } }); const usedTemplates = objectIndex( @@ -43,9 +64,7 @@ const templatePropertyExistenceCheck = async (propertyName: string, templateIds: throw new Error('Missing property.'); } - if (!ALLOWED_PROPERTY_TYPES.includes(property.type)) { - throw new Error('Property type not allowed.'); - } + checkTypeIsAllowed(property.type); }); }; @@ -134,3 +153,6 @@ export const Extractors = { await Suggestions.delete({ entityTemplate: templateId, extractorId: { $in: extractorIds } }); }, }; + +export type { AllowedPropertyTypes }; +export { ALLOWED_PROPERTY_TYPES, typeIsAllowed, checkTypeIsAllowed }; diff --git a/app/api/services/informationextraction/specs/InformationExtraction.spec.ts b/app/api/services/informationextraction/specs/InformationExtraction.spec.ts index fd8de75bfe..513079e23c 100644 --- a/app/api/services/informationextraction/specs/InformationExtraction.spec.ts +++ b/app/api/services/informationextraction/specs/InformationExtraction.spec.ts @@ -188,6 +188,25 @@ describe('InformationExtraction', () => { ); }); + it('should send xmls (relationship)', async () => { + await informationExtraction.trainModel(factory.id('extractorWithRelationship')); + + const xmlK = await readDocument('K'); + const xmlL = await readDocument('L'); + + expect(IXExternalService.materialsFileParams).toEqual({ + 0: `/xml_to_train/tenant1/${factory.id('extractorWithRelationship')}`, + id: factory.id('extractorWithRelationship').toString(), + tenant: 'tenant1', + }); + + expect(IXExternalService.files.length).toBe(2); + expect(IXExternalService.files).toEqual(expect.arrayContaining([xmlK, xmlL])); + expect(IXExternalService.filesNames.sort()).toEqual( + ['documentK.xml', 'documentL.xml'].sort() + ); + }); + it('should send labeled data', async () => { await informationExtraction.trainModel(factory.id('prop1extractor')); @@ -244,6 +263,48 @@ describe('InformationExtraction', () => { }); }); + it('should send labeled data (relationship)', async () => { + await informationExtraction.trainModel(factory.id('extractorWithRelationship')); + + expect(IXExternalService.materials.length).toBe(2); + expect(IXExternalService.materials.find(m => m.xml_file_name === 'documentL.xml')).toEqual({ + xml_file_name: 'documentL.xml', + id: factory.id('extractorWithRelationship').toString(), + tenant: 'tenant1', + xml_segments_boxes: [ + { + left: 1, + top: 1, + width: 1, + height: 1, + page_number: 1, + text: 'P1', + }, + { + left: 1, + top: 1, + width: 1, + height: 1, + page_number: 1, + text: 'P2', + }, + ], + page_width: 13, + page_height: 13, + language_iso: 'en', + values: [ + { + id: 'P1sharedId', + label: 'P1', + }, + { + id: 'P3sharedId', + label: 'P3', + }, + ], + }); + }); + it('should sanitize dates before sending', async () => { await informationExtraction.trainModel(factory.id('prop2extractor')); @@ -277,7 +338,9 @@ describe('InformationExtraction', () => { tenant: 'tenant1', task: 'create_model', }); + }); + it('should start the task to train the model (multiselect)', async () => { await informationExtraction.trainModel(factory.id('extractorWithMultiselect')); expect(informationExtraction.taskManager?.startTask).toHaveBeenCalledWith({ @@ -304,10 +367,68 @@ describe('InformationExtraction', () => { }); }); + it('should start the task to train the model (relationship)', async () => { + await informationExtraction.trainModel(factory.id('extractorWithRelationship')); + + expect(informationExtraction.taskManager?.startTask).toHaveBeenCalledWith({ + params: { + id: factory.id('extractorWithRelationship').toString(), + multi_value: true, + options: [ + { + id: 'P1sharedId', + label: 'P1', + }, + { + id: 'P2sharedId', + label: 'P2', + }, + { + id: 'P3sharedId', + label: 'P3', + }, + ], + }, + tenant: 'tenant1', + task: 'create_model', + }); + }); + + it('should start the task to train the model (relationship to any template)', async () => { + await informationExtraction.trainModel(factory.id('extractorWithRelationshipToAny')); + + expect(informationExtraction.taskManager?.startTask).toHaveBeenCalledWith({ + params: { + id: factory.id('extractorWithRelationshipToAny').toString(), + multi_value: true, + options: [ + { + id: 'P1sharedId', + label: 'P1', + }, + { + id: 'P2sharedId', + label: 'P2', + }, + { + id: 'P3sharedId', + label: 'P3', + }, + ...Array.from({ length: 23 }, (_, i) => ({ + id: `A${i + 1}`, + label: `A${i + 1}`, + })), + ], + }, + tenant: 'tenant1', + task: 'create_model', + }); + }); + it('should return error status and stop finding suggestions, when there is no labaled data', async () => { const expectedError = { status: 'error', message: 'No labeled data' }; - const result = await informationExtraction.trainModel(factory.id('prop3extractor')); + const result = await informationExtraction.trainModel(factory.id('prop3extractor')); expect(result).toMatchObject(expectedError); const [model] = await IXModelsModel.get({ extractorId: factory.id('prop3extractor') }); expect(model.findingSuggestions).toBe(false); @@ -320,6 +441,15 @@ describe('InformationExtraction', () => { extractorId: factory.id('extractorWithMultiselectWithoutTrainingData'), }); expect(multiSelectModel.findingSuggestions).toBe(false); + + const relationshipResult = await informationExtraction.trainModel( + factory.id('extractorWithEmptyRelationship') + ); + expect(relationshipResult).toMatchObject(expectedError); + const [relationshipModel] = await IXModelsModel.get({ + extractorId: factory.id('extractorWithEmptyRelationship'), + }); + expect(relationshipModel.findingSuggestions).toBe(false); }); }); @@ -470,6 +600,90 @@ describe('InformationExtraction', () => { ]); }); + it('should send the materials for the suggestions (relationship)', async () => { + await informationExtraction.getSuggestions(factory.id('extractorWithRelationship')); + + const [xmlK, xmlL, xmlM] = await Promise.all(['K', 'L', 'M'].map(readDocument)); + + expect(IXExternalService.materialsFileParams).toEqual({ + 0: `/xml_to_predict/tenant1/${factory.id('extractorWithRelationship')}`, + id: factory.id('extractorWithRelationship').toString(), + tenant: 'tenant1', + }); + + expect(IXExternalService.filesNames.sort()).toEqual( + ['documentK.xml', 'documentL.xml', 'documentM.xml'].sort() + ); + expect(IXExternalService.files.length).toBe(3); + expect(IXExternalService.files).toEqual(expect.arrayContaining([xmlK, xmlL, xmlM])); + + expect(IXExternalService.materials.length).toBe(3); + const sortedMaterials = sortByStrings(IXExternalService.materials, [ + (m: any) => m.xml_file_name, + ]); + expect(sortedMaterials).toEqual([ + { + xml_file_name: 'documentK.xml', + id: factory.id('extractorWithRelationship').toString(), + tenant: 'tenant1', + page_height: 13, + page_width: 13, + xml_segments_boxes: [ + { + height: 1, + left: 1, + page_number: 1, + text: 'P1', + top: 1, + width: 1, + }, + ], + }, + { + xml_file_name: 'documentL.xml', + id: factory.id('extractorWithRelationship').toString(), + tenant: 'tenant1', + page_height: 13, + page_width: 13, + xml_segments_boxes: [ + { + height: 1, + left: 1, + page_number: 1, + text: 'P1', + top: 1, + width: 1, + }, + { + height: 1, + left: 1, + page_number: 1, + text: 'P2', + top: 1, + width: 1, + }, + ], + }, + { + xml_file_name: 'documentM.xml', + id: factory.id('extractorWithRelationship').toString(), + tenant: 'tenant1', + page_height: 13, + page_width: 13, + xml_segments_boxes: [ + { + height: 1, + left: 1, + page_number: 1, + text: 'P3', + top: 1, + width: 1, + }, + ], + }, + ]); + }); + it('should avoid sending materials for failed suggestions because no segmentation for instance', async () => { await informationExtraction.getSuggestions(factory.id('extractorWithOneFailedSegmentation')); @@ -610,7 +824,7 @@ describe('InformationExtraction', () => { IXExternalService.setResults([ { tenant: 'tenant1', - property_name: 'property1', + id: factory.id('prop1extractor').toString(), xml_file_name: 'documentA.xml', text: 'text_in_other_language', segment_text: 'segmented_text_in_other_language', @@ -626,7 +840,7 @@ describe('InformationExtraction', () => { }, { tenant: 'tenant1', - property_name: 'property1', + id: factory.id('prop1extractor').toString(), xml_file_name: 'documentD.xml', text: 'text_in_eng_language', segment_text: 'segmented_text_in_eng_language', @@ -813,7 +1027,7 @@ describe('InformationExtraction', () => { }, ]); - await saveSuggestionProcess('F1', 'A1', 'eng', 'prop4extractor'); + await saveSuggestionProcess('F1', 'A1', 'eng', 'prop1extractor'); await informationExtraction.processResults({ params: { id: factory.id('prop1extractor').toString() }, @@ -823,25 +1037,11 @@ describe('InformationExtraction', () => { data_url: 'http://localhost:1234/suggestions_results', }); - await informationExtraction.processResults({ - params: { id: factory.id('prop4extractor').toString() }, - tenant: 'tenant1', - task: 'suggestions', - success: true, - data_url: 'http://localhost:1234/suggestions_results', - }); - const suggestionsText = await IXSuggestionsModel.get({ status: 'ready', extractorId: factory.id('prop1extractor'), }); expect(suggestionsText.length).toBe(1); - - const suggestionsMarkdown = await IXSuggestionsModel.get({ - status: 'ready', - propertyName: 'property4', - }); - expect(suggestionsMarkdown.length).toBe(1); }); }); @@ -885,16 +1085,19 @@ describe('InformationExtraction', () => { id: factory.id('extractorWithSelect').toString(), xml_file_name: 'documentG.xml', values: [{ id: 'A', label: 'A' }], + segment_text: 'it is A', }, { id: factory.id('extractorWithSelect').toString(), xml_file_name: 'documentH.xml', values: [{ id: 'B', label: 'B' }], + segment_text: 'it is B', }, { id: factory.id('extractorWithSelect').toString(), xml_file_name: 'documentI.xml', values: [{ id: 'A', label: 'A' }], + segment_text: 'it is A', }, ], 'value' @@ -947,18 +1150,21 @@ describe('InformationExtraction', () => { fileId: factory.id('F17'), entityId: 'A17', suggestedValue: 'A', + segment: 'it is A', }, { ...expectedBase, fileId: factory.id('F18'), entityId: 'A18', suggestedValue: 'B', + segment: 'it is B', }, { ...expectedBase, fileId: factory.id('F19'), entityId: 'A19', suggestedValue: 'A', + segment: 'it is A', state: { ...expectedBase.state, withValue: false, @@ -976,6 +1182,7 @@ describe('InformationExtraction', () => { id: factory.id('extractorWithMultiselect').toString(), xml_file_name: 'documentG.xml', values: [{ id: 'A', label: 'A' }], + segment_text: 'it is A', }, { id: factory.id('extractorWithMultiselect').toString(), @@ -984,6 +1191,7 @@ describe('InformationExtraction', () => { { id: 'B', label: 'B' }, { id: 'C', label: 'C' }, ], + segment_text: 'it is B or C', }, { id: factory.id('extractorWithMultiselect').toString(), @@ -992,6 +1200,7 @@ describe('InformationExtraction', () => { { id: 'A', label: 'A' }, { id: 'C', label: 'C' }, ], + segment_text: 'it is A or C', }, ], 'value' @@ -1044,18 +1253,127 @@ describe('InformationExtraction', () => { fileId: factory.id('F17'), entityId: 'A17', suggestedValue: ['A'], + segment: 'it is A', }, { ...expectedBase, fileId: factory.id('F18'), entityId: 'A18', suggestedValue: ['B', 'C'], + segment: 'it is B or C', }, { ...expectedBase, fileId: factory.id('F19'), entityId: 'A19', suggestedValue: ['A', 'C'], + segment: 'it is A or C', + state: { + ...expectedBase.state, + withValue: false, + labeled: false, + match: false, + }, + }, + ]); + }); + }); + + describe('relationship', () => { + it('should request and store the suggestions (relationship)', async () => { + setIXServiceResults( + [ + { + id: factory.id('extractorWithRelationship').toString(), + xml_file_name: 'documentK.xml', + values: [{ id: 'P1sharedId', label: 'P1' }], + segment_text: 'it is P1', + }, + { + id: factory.id('extractorWithRelationship').toString(), + xml_file_name: 'documentL.xml', + values: [ + { id: 'P1sharedId', label: 'P1' }, + { id: 'P2sharedId', label: 'P2' }, + ], + segment_text: 'it is P1 or P2', + }, + { + id: factory.id('extractorWithRelationship').toString(), + xml_file_name: 'documentM.xml', + values: [{ id: 'P3sharedId', label: 'P3' }], + segment_text: 'it is P3', + }, + ], + 'value' + ); + + await saveSuggestionProcess('SUG21', 'A21', 'eng', 'extractorWithRelationship'); + await saveSuggestionProcess('SUG22', 'A22', 'eng', 'extractorWithRelationship'); + await saveSuggestionProcess('SUG23', 'A23', 'eng', 'extractorWithRelationship'); + + await informationExtraction.processResults({ + params: { id: factory.id('extractorWithRelationship').toString() }, + tenant: 'tenant1', + task: 'suggestions', + success: true, + data_url: 'http://localhost:1234/suggestions_results', + }); + + const suggestions = await IXSuggestionsModel.get({ + status: 'ready', + extractorId: factory.id('extractorWithRelationship'), + }); + + const sorted = sortByStrings(suggestions, [(s: any) => s.entityId]); + + const expectedBase = { + _id: expect.any(ObjectId), + entityTemplate: factory.id('templateToSegmentF').toString(), + language: 'en', + propertyName: 'property_relationship', + extractorId: factory.id('extractorWithRelationship'), + status: 'ready', + page: 1, + date: expect.any(Number), + error: '', + state: { + labeled: true, + withValue: true, + withSuggestion: true, + match: true, + hasContext: true, + processing: false, + obsolete: false, + error: false, + }, + }; + + expect(sorted).toEqual([ + { + ...expectedBase, + fileId: factory.id('F21'), + entityId: 'A21', + suggestedValue: ['P1sharedId'], + segment: 'it is P1', + }, + { + ...expectedBase, + fileId: factory.id('F22'), + entityId: 'A22', + suggestedValue: ['P1sharedId', 'P2sharedId'], + segment: 'it is P1 or P2', + state: { + ...expectedBase.state, + match: false, + }, + }, + { + ...expectedBase, + fileId: factory.id('F23'), + entityId: 'A23', + suggestedValue: ['P3sharedId'], + segment: 'it is P3', state: { ...expectedBase.state, withValue: false, diff --git a/app/api/services/informationextraction/specs/fixtures.ts b/app/api/services/informationextraction/specs/fixtures.ts index 9713847afc..fd87182dee 100644 --- a/app/api/services/informationextraction/specs/fixtures.ts +++ b/app/api/services/informationextraction/specs/fixtures.ts @@ -13,6 +13,9 @@ const fixturesPdfNameG = 'documentG.pdf'; const fixturesPdfNameH = 'documentH.pdf'; const fixturesPdfNameI = 'documentI.pdf'; const ficturesPdfNameJ = 'documentJ.pdf'; +const fixturesPdfNameK = 'documentK.pdf'; +const fixturesPdfNameL = 'documentL.pdf'; +const fixturesPdfNameM = 'documentM.pdf'; const fixtures: DBFixture = { settings: [ @@ -36,15 +39,26 @@ const fixtures: DBFixture = { ]), factory.ixExtractor('prop2extractor', 'property2', ['templateToSegmentA']), factory.ixExtractor('prop3extractor', 'property3', ['templateToSegmentA']), - factory.ixExtractor('prop4extractor', 'property4', ['templateToSegmentA']), factory.ixExtractor('extractorWithOneFailedSegmentation', 'property15', ['templateToSegmentC']), factory.ixExtractor('extractorWithSelect', 'property_select', ['templateToSegmentD']), factory.ixExtractor('extractorWithMultiselect', 'property_multiselect', ['templateToSegmentD']), factory.ixExtractor('extractorWithMultiselectWithoutTrainingData', 'property_multiselect', [ 'templateToSegmentE', ]), + factory.ixExtractor('extractorWithRelationship', 'property_relationship', [ + 'templateToSegmentF', + ]), + factory.ixExtractor('extractorWithEmptyRelationship', 'property_empty_relationship', [ + 'templateToSegmentF', + ]), + factory.ixExtractor('extractorWithRelationshipToAny', 'property_relationship_to_any', [ + 'templateToSegmentF', + ]), ], entities: [ + factory.entity('P1', 'relationshipPartnerTemplate', {}, { sharedId: 'P1sharedId' }), + factory.entity('P2', 'relationshipPartnerTemplate', {}, { sharedId: 'P2sharedId' }), + factory.entity('P3', 'relationshipPartnerTemplate', {}, { sharedId: 'P3sharedId' }), factory.entity( 'A1', 'templateToSegmentA', @@ -102,6 +116,27 @@ const fixtures: DBFixture = { factory.entity('A20', 'templateToSegmentE', { property_multiselect: [], }), + factory.entity('A21', 'templateToSegmentF', { + property_relationship: [{ value: 'P1sharedId', label: 'P1' }], + property_empty_relationship: [], + property_relationship_to_any: [{ value: 'P1sharedId', label: 'P1' }], + }), + factory.entity('A22', 'templateToSegmentF', { + property_relationship: [ + { value: 'P1sharedId', label: 'P1' }, + { value: 'P3sharedId', label: 'P3' }, + ], + property_empty_relationship: [], + property_relationship_to_any: [ + { value: 'P1', label: 'P1' }, + { value: 'A1', label: 'A1' }, + ], + }), + factory.entity('A23', 'templateToSegmentF', { + property_relationship: [], + property_empty_relationship: [], + property_relationship_to_any: [], + }), ], files: [ factory.file('F1', 'A1', 'document', fixturesPdfNameA, 'other', '', [ @@ -155,6 +190,9 @@ const fixtures: DBFixture = { factory.file('F18', 'A18', 'document', fixturesPdfNameH, 'eng'), factory.file('F19', 'A19', 'document', fixturesPdfNameI, 'eng'), factory.file('F20', 'A20', 'document', ficturesPdfNameJ, 'eng'), + factory.file('F21', 'A21', 'document', fixturesPdfNameK, 'eng'), + factory.file('F22', 'A22', 'document', fixturesPdfNameL, 'eng'), + factory.file('F23', 'A23', 'document', fixturesPdfNameM, 'eng'), ], segmentations: [ { @@ -291,6 +329,77 @@ const fixtures: DBFixture = { paragraphs: [], }, }, + { + _id: factory.id('S11'), + filename: fixturesPdfNameK, + xmlname: 'documentK.xml', + fileID: factory.id('F21'), + status: 'ready', + segmentation: { + page_height: 13, + page_width: 13, + paragraphs: [ + { + left: 1, + top: 1, + width: 1, + height: 1, + page_number: 1, + text: 'P1', + }, + ], + }, + }, + { + _id: factory.id('S12'), + filename: fixturesPdfNameL, + xmlname: 'documentL.xml', + fileID: factory.id('F22'), + status: 'ready', + segmentation: { + page_height: 13, + page_width: 13, + paragraphs: [ + { + left: 1, + top: 1, + width: 1, + height: 1, + page_number: 1, + text: 'P1', + }, + { + left: 1, + top: 1, + width: 1, + height: 1, + page_number: 1, + text: 'P2', + }, + ], + }, + }, + { + _id: factory.id('S13'), + filename: fixturesPdfNameM, + xmlname: 'documentM.xml', + fileID: factory.id('F23'), + status: 'ready', + segmentation: { + page_height: 13, + page_width: 13, + paragraphs: [ + { + left: 1, + top: 1, + width: 1, + height: 1, + page_number: 1, + text: 'P3', + }, + ], + }, + }, ], ixsuggestions: [ { @@ -506,6 +615,45 @@ const fixtures: DBFixture = { page: 1, date: 100, }, + { + _id: factory.id('SUG21'), + fileId: factory.id('F21'), + entityId: 'A21', + entityTemplate: factory.idString('templateToSegmentF'), + language: 'en', + propertyName: 'property_relationship', + extractorId: factory.id('extractorWithRelationship'), + suggestedValue: ['P1'], + status: 'ready', + page: 1, + date: 100, + }, + { + _id: factory.id('SUG22'), + fileId: factory.id('F22'), + entityId: 'A22', + entityTemplate: factory.idString('templateToSegmentF'), + language: 'en', + propertyName: 'property_relationship', + extractorId: factory.id('extractorWithRelationship'), + suggestedValue: ['P1', 'P2'], + status: 'ready', + page: 1, + date: 100, + }, + { + _id: factory.id('SUG23'), + fileId: factory.id('F23'), + entityId: 'A23', + entityTemplate: factory.idString('templateToSegmentF'), + language: 'en', + propertyName: 'property_relationship', + extractorId: factory.id('extractorWithRelationship'), + suggestedValue: [], + status: 'ready', + page: 1, + date: 100, + }, ], ixmodels: [ { @@ -556,13 +704,36 @@ const fixtures: DBFixture = { status: 'ready', findingSuggestions: false, }, + { + extractorId: factory.id('extractorWithRelationship'), + creationDate: 200, + status: 'ready', + findingSuggestions: true, + }, + { + extractorId: factory.id('extractorWithEmptyRelationship'), + creationDate: 200, + status: 'ready', + findingSuggestions: true, + }, + { + extractorId: factory.id('extractorWithRelationshipToAny'), + creationDate: 200, + status: 'ready', + findingSuggestions: true, + }, + ], + relationtypes: [ + factory.relationType('related'), + factory.relationType('emptyRelated'), + factory.relationType('relatedToAny'), ], templates: [ + factory.template('relationshipPartnerTemplate'), factory.template('templateToSegmentA', [ factory.property('property1', 'text'), factory.property('property2', 'date'), factory.property('property3', 'numeric'), - factory.property('property4', 'markdown'), ]), factory.template('templateToSegmentB', [factory.property('property1', 'text')]), factory.template('templateToSegmentC', [factory.property('property15', 'text')]), @@ -579,6 +750,20 @@ const fixtures: DBFixture = { content: factory.id('thesauri1').toString(), }), ]), + factory.template('templateToSegmentF', [ + factory.property('property_relationship', 'relationship', { + content: factory.idString('relationshipPartnerTemplate'), + relationType: factory.idString('related'), + }), + factory.property('property_empty_relationship', 'relationship', { + content: factory.idString('relationshipPartnerTemplate'), + relationType: factory.idString('emptyRelated'), + }), + factory.property('property_relationship_to_any', 'relationship', { + content: '', + relationType: factory.idString('relatedToAny'), + }), + ]), ], dictionaries: [factory.thesauri('thesauri1', ['A', 'B', 'C'])], }; diff --git a/app/api/services/informationextraction/specs/suggestionFormatting.spec.ts b/app/api/services/informationextraction/specs/suggestionFormatting.spec.ts new file mode 100644 index 0000000000..39f20fd93f --- /dev/null +++ b/app/api/services/informationextraction/specs/suggestionFormatting.spec.ts @@ -0,0 +1,557 @@ +import { getFixturesFactory } from 'api/utils/fixturesFactory'; +import { PropertySchema } from 'shared/types/commonTypes'; +import { EntitySchema } from 'shared/types/entityType'; +import { IXSuggestionType } from 'shared/types/suggestionType'; +import { formatSuggestion } from '../suggestionFormatting'; +import { InternalIXResultsMessage } from '../InformationExtraction'; + +const fixtureFactory = getFixturesFactory(); + +const successMessage: InternalIXResultsMessage = { + tenant: 'tenant', + task: 'suggestions', + params: { + id: fixtureFactory.id('extractor_id'), + }, + data_url: 'data_url', + file_url: 'file_url', + success: true, +}; + +const properties: Record = { + text: fixtureFactory.property('text_property', 'text'), + numeric: fixtureFactory.property('numeric_property', 'numeric'), + date: fixtureFactory.property('date_property', 'date'), + select: fixtureFactory.property('select_property', 'select'), + multiselect: fixtureFactory.property('multiselect_property', 'multiselect'), + relationship: fixtureFactory.property('relationship_property', 'relationship'), +}; + +const entities: Record = { + title: fixtureFactory.entity('entity_id', 'entity_template', {}), + text: fixtureFactory.entity('entity_id', 'entity_template', { + text_property: [{ value: 'previous_value' }], + }), + numeric: fixtureFactory.entity('entity_id', 'entity_template', { + numeric_property: [{ value: 0 }], + }), + date: fixtureFactory.entity('entity_id', 'entity_template', { + date_property: [{ value: 0 }], + }), + select: fixtureFactory.entity('entity_id', 'entity_template', { + select_property: [{ value: 'A_id', label: 'A' }], + }), + multiselect: fixtureFactory.entity('entity_id', 'entity_template', { + multiselect_property: [ + { value: 'A_id', label: 'A' }, + { value: 'B_id', label: 'B' }, + ], + }), + relationship: fixtureFactory.entity('entity_id', 'entity_template', { + relationship_property: [ + { value: 'related_1_id', label: 'related_1_title' }, + { value: 'related_2_id', label: 'related_2_title' }, + ], + }), +}; + +const currentSuggestionBase = { + _id: fixtureFactory.id('suggestion_id'), + entityId: 'entity_id', + extractorId: fixtureFactory.id('extractor_id'), + entityTemplate: 'entity_template', + fileId: fixtureFactory.id('file_id'), + segment: 'previous_context', + language: 'en', + page: 100, + date: 1, + status: 'ready' as 'ready', + error: '', + selectionRectangles: [ + { + top: 13, + left: 13, + width: 13, + height: 13, + page: '100', + }, + ], +}; + +const currentSuggestions: Record = { + title: { + ...currentSuggestionBase, + propertyName: 'title', + suggestedValue: 'previous_value', + }, + text: { + ...currentSuggestionBase, + propertyName: 'text_property', + suggestedValue: 'previous_value', + }, + numeric: { + ...currentSuggestionBase, + propertyName: 'numeric_property', + suggestedValue: 0, + }, + date: { + ...currentSuggestionBase, + propertyName: 'date_property', + suggestedValue: 0, + }, + select: { + ...currentSuggestionBase, + propertyName: 'select_property', + suggestedValue: 'A_id', + }, + multiselect: { + ...currentSuggestionBase, + propertyName: 'multiselect_property', + suggestedValue: ['A_id', 'B_id'], + }, + relationship: { + ...currentSuggestionBase, + propertyName: 'relationship_property', + suggestedValue: ['related_1_id', 'related_2_id'], + }, +}; + +const rawSuggestionBase = { + tenant: 'tenant', + id: 'extractor_id', + xml_file_name: 'file.xml', + segment_text: 'new context', +}; + +const suggestedDateTimeStamp = 1717743209000; +const suggestedDateText = new Date(suggestedDateTimeStamp).toISOString(); + +const validRawSuggestions = { + title: { + ...rawSuggestionBase, + text: 'recommended_value', + segments_boxes: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page_number: 1, + }, + ], + }, + text: { + ...rawSuggestionBase, + text: 'recommended_value', + segments_boxes: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page_number: 1, + }, + ], + }, + numeric: { + ...rawSuggestionBase, + text: '42', + segments_boxes: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page_number: 1, + }, + ], + }, + date: { + ...rawSuggestionBase, + text: suggestedDateText, + segments_boxes: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page_number: 1, + }, + ], + }, + select: { + ...rawSuggestionBase, + values: [{ id: 'B_id', label: 'B' }], + }, + multiselect: { + ...rawSuggestionBase, + values: [ + { id: 'C_id', label: 'C' }, + { id: 'D_id', label: 'D' }, + ], + }, + relationship: { + ...rawSuggestionBase, + values: [ + { id: 'related_1_id', label: 'related_1_title' }, + { id: 'related_3_id', label: 'related_3_title' }, + ], + }, +}; + +describe('formatSuggestion', () => { + it.each([ + { + case: 'missing properties', + property: properties.text, + rawSuggestion: { + ...validRawSuggestions.text, + tenant: undefined, + }, + currentSuggestion: currentSuggestions.text, + entity: entities.text, + expectedErrorMessage: ": must have required property 'tenant'", + }, + { + case: 'invalid tenant type', + property: properties.text, + rawSuggestion: { + ...validRawSuggestions.text, + tenant: 1, + }, + currentSuggestion: currentSuggestions.text, + entity: entities.text, + expectedErrorMessage: '/tenant: must be string', + }, + { + case: 'invalid id type', + property: properties.text, + rawSuggestion: { + ...validRawSuggestions.text, + id: 1, + }, + currentSuggestion: currentSuggestions.text, + entity: entities.text, + expectedErrorMessage: '/id: must be string', + }, + { + case: 'invalid xml_file_name type', + property: properties.text, + rawSuggestion: { + ...validRawSuggestions.text, + xml_file_name: 1, + }, + currentSuggestion: currentSuggestions.text, + entity: entities.text, + expectedErrorMessage: '/xml_file_name: must be string', + }, + { + case: 'invalid text type', + property: properties.text, + rawSuggestion: { + ...validRawSuggestions.text, + text: 1, + }, + currentSuggestion: currentSuggestions.text, + entity: entities.text, + expectedErrorMessage: '/text: must be string', + }, + { + case: 'invalid segment_text type', + property: properties.text, + rawSuggestion: { + ...validRawSuggestions.text, + segment_text: 1, + }, + currentSuggestion: currentSuggestions.text, + entity: entities.text, + expectedErrorMessage: '/segment_text: must be string', + }, + { + case: 'invalid segments_boxes subtype', + property: properties.text, + rawSuggestion: { + ...validRawSuggestions.text, + segments_boxes: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page_number: '1', + }, + ], + }, + currentSuggestion: currentSuggestions.text, + entity: entities.text, + expectedErrorMessage: '/segments_boxes/0/page_number: must be number', + }, + { + case: 'invalid select values type', + property: properties.select, + rawSuggestion: { + ...validRawSuggestions.select, + values: 1, + }, + currentSuggestion: currentSuggestions.select, + entity: entities.select, + expectedErrorMessage: '/values: must be array', + }, + { + case: 'invalid select values subtype', + property: properties.select, + rawSuggestion: { + ...validRawSuggestions.select, + values: [{ id: 1, label: 'value_label' }], + }, + currentSuggestion: currentSuggestions.select, + entity: entities.select, + expectedErrorMessage: '/values/0/id: must be string', + }, + { + case: 'invalid select values length', + property: properties.select, + rawSuggestion: { + ...validRawSuggestions.select, + values: [ + { id: 'B_id', label: 'B' }, + { id: 'C_id', label: 'C' }, + ], + }, + currentSuggestion: currentSuggestions.select, + entity: entities.select, + expectedErrorMessage: 'Select suggestions must have one or zero values.', + }, + { + case: 'invalid multiselect values type', + property: properties.multiselect, + rawSuggestion: { + ...validRawSuggestions.multiselect, + values: 1, + }, + currentSuggestion: currentSuggestions.multiselect, + entity: entities.multiselect, + expectedErrorMessage: '/values: must be array', + }, + { + case: 'invalid multiselect values subtype', + property: properties.multiselect, + rawSuggestion: { + ...validRawSuggestions.multiselect, + values: [{ id: 1, label: 'value_label' }], + }, + currentSuggestion: currentSuggestions.multiselect, + entity: entities.multiselect, + expectedErrorMessage: '/values/0/id: must be string', + }, + { + case: 'invalid relationship values type', + property: properties.relationship, + rawSuggestion: { + ...validRawSuggestions.relationship, + values: 1, + }, + currentSuggestion: currentSuggestions.relationship, + entity: entities.relationship, + expectedErrorMessage: '/values: must be array', + }, + { + case: 'invalid relationship values subtype', + property: properties.relationship, + rawSuggestion: { + ...validRawSuggestions.relationship, + values: [{ id: 1, label: 'value_label' }], + }, + currentSuggestion: currentSuggestions.relationship, + entity: entities.relationship, + expectedErrorMessage: '/values/0/id: must be string', + }, + ])( + 'should throw error if $case', + async ({ property, rawSuggestion, currentSuggestion, entity, expectedErrorMessage }) => { + const cb = async () => + formatSuggestion( + property, + // @ts-expect-error + rawSuggestion, + currentSuggestion, + entity, + successMessage + ); + await expect(cb).rejects.toThrow(expectedErrorMessage); + } + ); + + it('should allow extra properties', async () => { + const property = properties.text; + const rawSuggestion = { + ...validRawSuggestions.text, + extra: 'extra', + }; + const result = await formatSuggestion( + property, + rawSuggestion, + currentSuggestions.text, + entities.text, + successMessage + ); + expect(result).toEqual({ + ...currentSuggestions.text, + date: expect.any(Number), + suggestedValue: 'recommended_value', + segment: 'new context', + selectionRectangles: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page: '1', + }, + ], + }); + }); + + it.each([ + { + case: 'valid title suggestions', + property: { name: 'title' as 'title', type: 'title' as 'title' }, + rawSuggestion: validRawSuggestions.title, + currentSuggestion: currentSuggestions.title, + entity: entities.title, + expectedResult: { + ...currentSuggestions.title, + date: expect.any(Number), + suggestedValue: 'recommended_value', + segment: 'new context', + selectionRectangles: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page: '1', + }, + ], + }, + }, + { + case: 'valid text suggestions', + property: properties.text, + rawSuggestion: validRawSuggestions.text, + currentSuggestion: currentSuggestions.text, + entity: entities.text, + expectedResult: { + ...currentSuggestions.text, + date: expect.any(Number), + suggestedValue: 'recommended_value', + segment: 'new context', + selectionRectangles: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page: '1', + }, + ], + }, + }, + { + case: 'valid numeric suggestions', + property: properties.numeric, + rawSuggestion: validRawSuggestions.numeric, + currentSuggestion: currentSuggestions.numeric, + entity: entities.numeric, + expectedResult: { + ...currentSuggestions.numeric, + date: expect.any(Number), + suggestedValue: 42, + segment: 'new context', + selectionRectangles: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page: '1', + }, + ], + }, + }, + { + case: 'valid date suggestions', + property: properties.date, + rawSuggestion: validRawSuggestions.date, + currentSuggestion: currentSuggestions.date, + entity: entities.date, + expectedResult: { + ...currentSuggestions.date, + date: expect.any(Number), + suggestedValue: suggestedDateTimeStamp / 1000, + suggestedText: suggestedDateText, + segment: 'new context', + selectionRectangles: [ + { + top: 0, + left: 0, + width: 0, + height: 0, + page: '1', + }, + ], + }, + }, + { + case: 'valid select suggestions', + property: properties.select, + rawSuggestion: validRawSuggestions.select, + currentSuggestion: currentSuggestions.select, + entity: entities.select, + expectedResult: { + ...currentSuggestions.select, + date: expect.any(Number), + suggestedValue: 'B_id', + segment: 'new context', + }, + }, + { + case: 'valid multiselect suggestions', + property: properties.multiselect, + rawSuggestion: validRawSuggestions.multiselect, + currentSuggestion: currentSuggestions.multiselect, + entity: entities.multiselect, + expectedResult: { + ...currentSuggestions.multiselect, + date: expect.any(Number), + suggestedValue: ['C_id', 'D_id'], + segment: 'new context', + }, + }, + { + case: 'valid relationship suggestions', + property: properties.relationship, + rawSuggestion: validRawSuggestions.relationship, + currentSuggestion: currentSuggestions.relationship, + entity: entities.relationship, + expectedResult: { + ...currentSuggestions.relationship, + date: expect.any(Number), + suggestedValue: ['related_1_id', 'related_3_id'], + segment: 'new context', + }, + }, + ])( + 'should return formatted suggestion for $case', + async ({ property, rawSuggestion, currentSuggestion, entity, expectedResult }) => { + const result = await formatSuggestion( + property, + rawSuggestion, + currentSuggestion, + entity, + successMessage + ); + expect(result).toEqual(expectedResult); + } + ); +}); diff --git a/app/api/services/informationextraction/specs/uploads/segmentation/documentK.xml b/app/api/services/informationextraction/specs/uploads/segmentation/documentK.xml new file mode 100644 index 0000000000..ff40832804 --- /dev/null +++ b/app/api/services/informationextraction/specs/uploads/segmentation/documentK.xml @@ -0,0 +1,3 @@ + + K + \ No newline at end of file diff --git a/app/api/services/informationextraction/specs/uploads/segmentation/documentL.xml b/app/api/services/informationextraction/specs/uploads/segmentation/documentL.xml new file mode 100644 index 0000000000..3d8093bb49 --- /dev/null +++ b/app/api/services/informationextraction/specs/uploads/segmentation/documentL.xml @@ -0,0 +1,3 @@ + + L + \ No newline at end of file diff --git a/app/api/services/informationextraction/specs/uploads/segmentation/documentM.xml b/app/api/services/informationextraction/specs/uploads/segmentation/documentM.xml new file mode 100644 index 0000000000..77fe91e68a --- /dev/null +++ b/app/api/services/informationextraction/specs/uploads/segmentation/documentM.xml @@ -0,0 +1,3 @@ + + M + \ No newline at end of file diff --git a/app/api/services/informationextraction/suggestionFormatting.ts b/app/api/services/informationextraction/suggestionFormatting.ts index 919604c768..73839f9f2d 100644 --- a/app/api/services/informationextraction/suggestionFormatting.ts +++ b/app/api/services/informationextraction/suggestionFormatting.ts @@ -1,73 +1,168 @@ -import { stringToTypeOfProperty } from 'shared/stringToTypeOfProperty'; +import Ajv from 'ajv'; + +import date from 'api/utils/date'; import { PropertySchema } from 'shared/types/commonTypes'; import { EntitySchema } from 'shared/types/entityType'; -import { IXSuggestionType } from 'shared/types/suggestionType'; +import { + CommonSuggestion, + IXSuggestionType, + TextSelectionSuggestion, + ValuesSelectionSuggestion, +} from 'shared/types/suggestionType'; +import { + TextSelectionSuggestionSchema, + ValuesSelectionSuggestionSchema, +} from 'shared/types/suggestionSchema'; +import { syncWrapValidator } from 'shared/tsUtils'; import { InternalIXResultsMessage } from './InformationExtraction'; +import { AllowedPropertyTypes, checkTypeIsAllowed } from './ixextractors'; -interface CommonSuggestion { - tenant: string; - id: string; - xml_file_name: string; -} +type RawSuggestion = TextSelectionSuggestion | ValuesSelectionSuggestion; -interface TextSelectionSuggestion extends CommonSuggestion { - text: string; - segment_text: string; - segments_boxes: { - top: number; - left: number; - width: number; - height: number; - page_number: number; - }[]; +class RawSuggestionValidationError extends Error { + constructor(message: string) { + super(message); + this.name = 'RawSuggestionValidationError'; + } } -interface ValuesSelectionSuggestion extends CommonSuggestion { - values: { id: string; label: string }[]; - segment_text: string; -} +type TitleAsProperty = { + name: 'title'; + type: 'title'; +}; -type RawSuggestion = TextSelectionSuggestion | ValuesSelectionSuggestion; +const createAjvValidator = (schema: any) => { + const ajv = new Ajv({ allErrors: true }); + ajv.addVocabulary(['tsType']); + return syncWrapValidator(ajv.compile(schema)); +}; + +const textSelectionAjv = createAjvValidator(TextSelectionSuggestionSchema); +const valuesSelectionAjv = createAjvValidator(ValuesSelectionSuggestionSchema); + +const textSelectionValidator = ( + suggestion: RawSuggestion +): suggestion is TextSelectionSuggestion => { + textSelectionAjv(suggestion); + return true; +}; + +const valuesSelectionValidator = ( + suggestion: RawSuggestion +): suggestion is ValuesSelectionSuggestion => { + valuesSelectionAjv(suggestion); + return true; +}; const VALIDATORS = { - text: (suggestion: RawSuggestion): suggestion is TextSelectionSuggestion => 'text' in suggestion, - select: (suggestion: RawSuggestion): suggestion is ValuesSelectionSuggestion => - 'values' in suggestion && (suggestion.values.length === 1 || suggestion.values.length === 0), - multiselect: (suggestion: RawSuggestion): suggestion is ValuesSelectionSuggestion => - 'values' in suggestion, + title: textSelectionValidator, + text: textSelectionValidator, + numeric: textSelectionValidator, + date: textSelectionValidator, + select: (suggestion: RawSuggestion): suggestion is ValuesSelectionSuggestion => { + if (!valuesSelectionValidator(suggestion)) { + throw new RawSuggestionValidationError('Select suggestion is not valid.'); + } + + if (!('values' in suggestion) || suggestion.values.length > 1) { + throw new RawSuggestionValidationError('Select suggestions must have one or zero values.'); + } + + return true; + }, + multiselect: valuesSelectionValidator, + relationship: valuesSelectionValidator, +}; + +const simpleSuggestion = ( + suggestedValue: string | number | null, + rawSuggestion: TextSelectionSuggestion +) => ({ + suggestedValue, + segment: rawSuggestion.segment_text, + selectionRectangles: rawSuggestion.segments_boxes.map((box: any) => { + const rect = { ...box, page: box.page_number.toString() }; + delete rect.page_number; + return rect; + }), +}); + +function multiValueIdsSuggestion(rawSuggestion: ValuesSelectionSuggestion) { + const suggestedValue = rawSuggestion.values.map(value => value.id); + + const suggestion: Partial = { + suggestedValue, + segment: rawSuggestion.segment_text, + }; + return suggestion; +} + +const textFormatter = ( + rawSuggestion: RawSuggestion, + _currentSuggestion: IXSuggestionType, + _entity: EntitySchema +) => { + if (!VALIDATORS.text(rawSuggestion)) { + throw new Error('Text suggestion is not valid.'); + } + + const rawText = rawSuggestion.text; + const suggestedValue = rawText.trim(); + + const suggestion: Partial = simpleSuggestion(suggestedValue, rawSuggestion); + + return suggestion; }; -const FORMATTERS = { - text: ( +const FORMATTERS: Record< + AllowedPropertyTypes, + ( + rawSuggestion: RawSuggestion, + currentSuggestion: IXSuggestionType, + entity: EntitySchema + ) => Partial +> = { + title: textFormatter, + text: textFormatter, + numeric: ( + rawSuggestion: RawSuggestion, + _currentSuggestion: IXSuggestionType, + _entity: EntitySchema + ) => { + if (!VALIDATORS.numeric(rawSuggestion)) { + throw new Error('Numeric suggestion is not valid.'); + } + + const suggestedValue = parseFloat(rawSuggestion.text.trim()) || null; + const suggestion: Partial = simpleSuggestion(suggestedValue, rawSuggestion); + + return suggestion; + }, + date: ( rawSuggestion: RawSuggestion, - property: PropertySchema | undefined, currentSuggestion: IXSuggestionType, entity: EntitySchema ) => { - if (!VALIDATORS.text(rawSuggestion)) { - throw new Error('Text suggestion is not valid.'); + if (!VALIDATORS.date(rawSuggestion)) { + throw new Error('Date suggestion is not valid.'); } - const suggestedValue = stringToTypeOfProperty( - rawSuggestion.text, - property?.type, + const suggestedValue = date.dateToSeconds( + rawSuggestion.text.trim(), currentSuggestion?.language || entity.language ); - const suggestion: Partial = { - suggestedValue, - ...(property?.type === 'date' ? { suggestedText: rawSuggestion.text } : {}), - segment: rawSuggestion.segment_text, - selectionRectangles: rawSuggestion.segments_boxes.map((box: any) => { - const rect = { ...box, page: box.page_number.toString() }; - delete rect.page_number; - return rect; - }), + ...simpleSuggestion(suggestedValue, rawSuggestion), + suggestedText: rawSuggestion.text, }; return suggestion; }, - select: (rawSuggestion: RawSuggestion) => { + select: ( + rawSuggestion: RawSuggestion, + _currentSuggestion: IXSuggestionType, + _entity: EntitySchema + ) => { if (!VALIDATORS.select(rawSuggestion)) { throw new Error('Select suggestion is not valid.'); } @@ -81,46 +176,57 @@ const FORMATTERS = { return suggestion; }, - multiselect: (rawSuggestion: RawSuggestion) => { + multiselect: ( + rawSuggestion: RawSuggestion, + _currentSuggestion: IXSuggestionType, + _entity: EntitySchema + ) => { if (!VALIDATORS.multiselect(rawSuggestion)) { throw new Error('Multiselect suggestion is not valid.'); } - const suggestedValue = rawSuggestion.values.map(value => value.id); + const suggestion: Partial = multiValueIdsSuggestion(rawSuggestion); - const suggestion: Partial = { - suggestedValue, - segment: rawSuggestion.segment_text, - }; + return suggestion; + }, + relationship: ( + rawSuggestion: RawSuggestion, + _currentSuggestion: IXSuggestionType, + _entity: EntitySchema + ) => { + if (!VALIDATORS.relationship(rawSuggestion)) { + throw new Error('Relationship suggestion is not valid.'); + } + + const suggestion: Partial = multiValueIdsSuggestion(rawSuggestion); return suggestion; }, }; -const DEFAULTFORMATTER = FORMATTERS.text; +type PropertyOrTitle = PropertySchema | TitleAsProperty | undefined; const formatRawSuggestion = ( rawSuggestion: RawSuggestion, - property: PropertySchema | undefined, + property: PropertyOrTitle, currentSuggestion: IXSuggestionType, entity: EntitySchema ) => { - const formatter = - // @ts-ignore - (property?.type || '') in FORMATTERS ? FORMATTERS[property.type] : DEFAULTFORMATTER; - return formatter(rawSuggestion, property, currentSuggestion, entity); + const type = checkTypeIsAllowed(property?.type || ''); + const formatter = FORMATTERS[type]; + return formatter(rawSuggestion, currentSuggestion, entity); }; const readMessageSuccess = (message: InternalIXResultsMessage) => message.success ? {} : { - status: 'failed', + status: 'failed' as 'failed', error: message.error_message ? message.error_message : 'Unknown error', }; const formatSuggestion = async ( - property: PropertySchema | undefined, + property: PropertyOrTitle, rawSuggestion: RawSuggestion, currentSuggestion: IXSuggestionType, entity: EntitySchema, diff --git a/app/api/suggestions/specs/fixtures.ts b/app/api/suggestions/specs/fixtures.ts index 32e81655cd..42b54dfbc2 100644 --- a/app/api/suggestions/specs/fixtures.ts +++ b/app/api/suggestions/specs/fixtures.ts @@ -1407,6 +1407,320 @@ const selectAcceptanceFixtureBase: DBFixture = { ], }; +const relationshipAcceptanceFixtureBase: DBFixture = { + settings: _.cloneDeep(ixSettings), + ixextractors: [ + factory.ixExtractor('relationship_extractor', 'relationship_to_source', ['rel_template']), + factory.ixExtractor( + 'relationship_with_inheritance_extractor', + 'relationship_with_inheritance', + ['rel_template'] + ), + factory.ixExtractor('relationship_to_any_extractor', 'relationship_to_any', ['rel_template']), + ], + ixsuggestions: [], + ixmodels: [ + { + _id: testingDB.id(), + status: 'ready', + creationDate: 1, + extractorId: factory.id('relationship_extractor'), + }, + { + _id: testingDB.id(), + status: 'ready', + creationDate: 1, + extractorId: factory.id('relationship_with_inheritance_extractor'), + }, + { + _id: testingDB.id(), + status: 'ready', + creationDate: 1, + extractorId: factory.id('relationship_to_any_extractor'), + }, + ], + relationtypes: [ + factory.relationType('related'), + factory.relationType('related_with_inheritance'), + factory.relationType('related_to_any'), + ], + templates: [ + factory.template('source_template', [factory.property('text_to_inherit', 'text')]), + factory.template('source_template_2', []), + factory.template('rel_template', [ + factory.property('relationship_to_source', 'relationship', { + content: factory.idString('source_template'), + relationType: factory.idString('related'), + }), + factory.property('relationship_with_inheritance', 'relationship', { + content: factory.idString('source_template'), + relationType: factory.idString('related_with_inheritance'), + inherit: { + property: factory.idString('text_to_inherit'), + type: 'text', + }, + }), + factory.property('relationship_to_any', 'relationship', { + content: '', + relationType: factory.idString('related_to_any'), + }), + ]), + ], + entities: [ + // ---------- sources + { + _id: testingDB.id(), + sharedId: 'S1_sId', + title: 'S1', + language: 'en', + metadata: { text_to_inherit: [{ value: 'inherited text' }] }, + template: factory.id('source_template'), + }, + { + _id: testingDB.id(), + sharedId: 'S1_sId', + title: 'S1_es', + language: 'es', + metadata: { text_to_inherit: [{ value: 'inherited text Spanish' }] }, + template: factory.id('source_template'), + }, + { + _id: testingDB.id(), + sharedId: 'S2_sId', + title: 'S2', + language: 'en', + metadata: { text_to_inherit: [{ value: 'inherited text 2' }] }, + template: factory.id('source_template'), + }, + { + _id: testingDB.id(), + sharedId: 'S2_sId', + title: 'S2_es', + language: 'es', + metadata: { text_to_inherit: [{ value: 'inherited text 2 Spanish' }] }, + template: factory.id('source_template'), + }, + { + _id: testingDB.id(), + sharedId: 'S3_sId', + title: 'S3', + language: 'en', + metadata: { text_to_inherit: [{ value: 'inherited text 3' }] }, + template: factory.id('source_template'), + }, + { + _id: testingDB.id(), + sharedId: 'S3_sId', + title: 'S3_es', + language: 'es', + metadata: { text_to_inherit: [{ value: 'inherited text 3 Spanish' }] }, + template: factory.id('source_template'), + }, + // ---------- other sources + { + _id: testingDB.id(), + sharedId: 'other_source', + title: 'Other Source', + language: 'en', + metadata: {}, + template: factory.id('source_template_2'), + }, + { + _id: testingDB.id(), + sharedId: 'other_source', + title: 'Other Source Spanish', + language: 'es', + metadata: {}, + template: factory.id('source_template_2'), + }, + { + _id: testingDB.id(), + sharedId: 'other_source_2', + title: 'Other Source 2', + language: 'en', + metadata: {}, + template: factory.id('source_template_2'), + }, + { + _id: testingDB.id(), + sharedId: 'other_source_2', + title: 'Other Source 2 Spanish', + language: 'es', + metadata: {}, + template: factory.id('source_template_2'), + }, + // ---------- with relationship + { + _id: testingDB.id(), + sharedId: 'entityWithRelationships_sId', + title: 'entityWithRelationships', + language: 'en', + metadata: { + relationship_to_source: [ + { value: 'S1_sId', label: 'S1' }, + { value: 'S2_sId', label: 'S2' }, + ], + relationship_with_inheritance: [ + { + value: 'S1_sId', + label: 'S1', + inheritedType: 'text', + inheritedValue: [ + { + value: 'inherited text', + }, + ], + }, + { + value: 'S2_sId', + label: 'S2', + inheritedType: 'text', + inheritedValue: [ + { + value: 'inherited text 2', + }, + ], + }, + ], + relationship_to_any: [ + { + value: 'S1_sId', + label: 'S1', + }, + { + value: 'other_source', + label: 'Other Source', + }, + ], + }, + template: factory.id('rel_template'), + }, + { + _id: testingDB.id(), + sharedId: 'entityWithRelationships_sId', + title: 'entityWithRelationshipsEs', + language: 'es', + metadata: { + relationship_to_source: [ + { value: 'S1_sId', label: 'S1_es' }, + { value: 'S2_sId', label: 'S2_es' }, + ], + relationship_with_inheritance: [ + { + value: 'S1_sId', + label: 'S1_es', + inheritedType: 'text', + inheritedValue: [ + { + value: 'inherited text Spanish', + }, + ], + }, + { + value: 'S2_sId', + label: 'S2_es', + inheritedType: 'text', + inheritedValue: [ + { + value: 'inherited text 2 Spanish', + }, + ], + }, + ], + relationship_to_any: [ + { + value: 'S1_sId', + label: 'S1_es', + }, + { + value: 'other_source', + label: 'Other Source Spanish', + }, + ], + }, + template: factory.id('rel_template'), + }, + ], + connections: [ + { + _id: testingDB.id(), + entity: 'entityWithRelationships_sId', + hub: factory.id('hub_S1'), + }, + { + _id: testingDB.id(), + entity: 'S1_sId', + hub: factory.id('hub_S1'), + template: factory.id('related'), + }, + { + _id: testingDB.id(), + entity: 'entityWithRelationships_sId', + hub: factory.id('hub_S2'), + }, + { + _id: testingDB.id(), + entity: 'S2_sId', + hub: factory.id('hub_S2'), + template: factory.id('related'), + }, + { + _id: testingDB.id(), + entity: 'entityWithRelationships_sId', + hub: factory.id('hub_S1_inherited'), + }, + { + _id: testingDB.id(), + entity: 'S1_sId', + hub: factory.id('hub_S1_inherited'), + template: factory.id('related_with_inheritance'), + }, + { + _id: testingDB.id(), + entity: 'entityWithRelationships_sId', + hub: factory.id('hub_S2_inherited'), + }, + { + _id: testingDB.id(), + entity: 'S2_sId', + hub: factory.id('hub_S2_inherited'), + template: factory.id('related_with_inheritance'), + }, + { + _id: testingDB.id(), + entity: 'entityWithRelationships_sId', + hub: factory.id('hub_S1_any'), + }, + { + _id: testingDB.id(), + entity: 'S1_sId', + hub: factory.id('hub_S1_any'), + template: factory.id('related_to_any'), + }, + { + _id: testingDB.id(), + entity: 'entityWithRelationships_sId', + hub: factory.id('hub_other_source_any'), + }, + { + _id: testingDB.id(), + entity: 'other_source', + hub: factory.id('hub_other_source_any'), + template: factory.id('related_to_any'), + }, + ], + files: [ + factory.file( + 'fileForEntityWithRelationships', + 'entityWithRelationships_sId', + 'document', + 'documentWithRelationships.pdf', + 'eng', + 'documentWithRelationships.pdf' + ), + ], +}; + export { factory, file2Id, @@ -1423,4 +1737,5 @@ export { suggestionId, shared2AgeSuggestionId, selectAcceptanceFixtureBase, + relationshipAcceptanceFixtureBase, }; diff --git a/app/api/suggestions/specs/routes.spec.ts b/app/api/suggestions/specs/routes.spec.ts index 40e0e0d18c..10bc43cf2a 100644 --- a/app/api/suggestions/specs/routes.spec.ts +++ b/app/api/suggestions/specs/routes.spec.ts @@ -393,10 +393,7 @@ describe('suggestions routes', () => { title: 'The Penguin', }, ]); - expect(search.indexEntities).toHaveBeenCalledWith( - { _id: { $in: [shared6enId] } }, - '+fullText' - ); + expect(search.indexEntities).toHaveBeenCalledWith({ sharedId: 'shared6' }, '+fullText'); }); it('should reject with unauthorized when user has not admin role', async () => { @@ -434,10 +431,10 @@ describe('suggestions routes', () => { const [entity] = await entities.get({ sharedId: 'entityWithSelects2' }); expect(entity.metadata.property_multiselect).toEqual([ { value: 'A', label: 'A' }, - { value: '1B', label: '1B' }, + { value: '1B', label: '1B', parent: { value: '1', label: '1' } }, ]); expect(search.indexEntities).toHaveBeenCalledWith( - { _id: { $in: [factory.id('entityWithSelects2')] } }, + { sharedId: 'entityWithSelects2' }, '+fullText' ); }); diff --git a/app/api/suggestions/specs/suggestions.spec.ts b/app/api/suggestions/specs/suggestions.spec.ts index 2fa5c756f6..fcf5135583 100644 --- a/app/api/suggestions/specs/suggestions.spec.ts +++ b/app/api/suggestions/specs/suggestions.spec.ts @@ -18,7 +18,9 @@ import { suggestionId, shared2AgeSuggestionId, selectAcceptanceFixtureBase, + relationshipAcceptanceFixtureBase, } from './fixtures'; +import { ObjectId } from 'mongodb'; const getSuggestions = async (filter: IXSuggestionsFilter, size = 5) => Suggestions.get(filter, { page: { size, number: 1 } }); @@ -346,18 +348,20 @@ const matchState = (match: boolean = true): IXSuggestionStateType => ({ error: false, }); -const selectSuggestionBase = (propertyName: string, extractorName: string) => ({ - fileId: factory.id('fileForentityWithSelects'), - entityId: 'entityWithSelects', - entityTemplate: factory.id('templateWithSelects').toString(), - propertyName, - extractorId: factory.id(extractorName), - date: 5, - status: 'ready' as 'ready', - error: '', -}); +type SuggestionBase = Pick< + IXSuggestionType, + | 'fileId' + | 'entityId' + | 'entityTemplate' + | 'propertyName' + | 'extractorId' + | 'date' + | 'status' + | 'error' +>; const prepareAndAcceptSuggestion = async ( + suggestionBase: SuggestionBase, suggestedValue: string | string[], language: string, propertyName: string, @@ -368,7 +372,7 @@ const prepareAndAcceptSuggestion = async ( } = {} ) => { const suggestion = { - ...selectSuggestionBase(propertyName, extractorName), + ...suggestionBase, suggestedValue, language, }; @@ -387,6 +391,69 @@ const prepareAndAcceptSuggestion = async ( return { acceptedSuggestion, metadataValues, allFiles }; }; +const selectSuggestionBase = (propertyName: string, extractorName: string): SuggestionBase => ({ + fileId: factory.id('fileForentityWithSelects'), + entityId: 'entityWithSelects', + entityTemplate: factory.id('templateWithSelects').toString(), + propertyName, + extractorId: factory.id(extractorName), + date: 5, + status: 'ready' as 'ready', + error: '', +}); + +const prepareAndAcceptSelectSuggestion = async ( + suggestedValue: string | string[], + language: string, + propertyName: string, + extractorName: string, + acceptanceParameters: { + addedValues?: string[]; + removedValues?: string[]; + } = {} +) => + prepareAndAcceptSuggestion( + selectSuggestionBase(propertyName, extractorName), + suggestedValue, + language, + propertyName, + extractorName, + acceptanceParameters + ); + +const relationshipSuggestionBase = ( + propertyName: string, + extractorName: string +): SuggestionBase => ({ + fileId: factory.id('fileForEntityWithRelationships'), + entityId: 'entityWithRelationships_sId', + entityTemplate: factory.id('rel_template').toString(), + propertyName, + extractorId: factory.id(extractorName), + date: 5, + status: 'ready' as 'ready', + error: '', +}); + +const prepareAndAcceptRelationshipSuggestion = async ( + suggestedValue: string | string[], + language: string, + propertyName: string, + extractorName: string, + acceptanceParameters: { + addedValues?: string[]; + removedValues?: string[]; + } = {} +) => + prepareAndAcceptSuggestion( + relationshipSuggestionBase(propertyName, extractorName), + suggestedValue, + language, + propertyName, + extractorName, + acceptanceParameters + ); + describe('suggestions', () => { afterAll(async () => { await db.disconnect(); @@ -828,14 +895,14 @@ describe('suggestions', () => { .find({ sharedId: 'shared1' }) .toArray(); const ages1 = entities1?.map(entity => entity.metadata.age[0].value); - expect(ages1).toEqual(['17', '17']); + expect(ages1).toEqual([17, 17]); const entities2 = await db.mongodb ?.collection('entities') .find({ sharedId: 'shared2' }) .toArray(); const ages2 = entities2?.map(entity => entity.metadata.age[0].value); - expect(ages2).toEqual(['20', '20', '20']); + expect(ages2).toEqual([20, 20, 20]); }); }); @@ -846,18 +913,14 @@ describe('suggestions', () => { it('should validate that the id exists in the dictionary', async () => { const action = async () => { - await prepareAndAcceptSuggestion('Z', 'en', 'property_select', 'select_extractor'); + await prepareAndAcceptSelectSuggestion('Z', 'en', 'property_select', 'select_extractor'); }; await expect(action()).rejects.toThrow('Id is invalid: Z (Nested Thesaurus).'); }); it('should update entities of all languages, with the properly translated labels', async () => { - const { acceptedSuggestion, metadataValues, allFiles } = await prepareAndAcceptSuggestion( - 'A', - 'en', - 'property_select', - 'select_extractor' - ); + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptSelectSuggestion('A', 'en', 'property_select', 'select_extractor'); expect(acceptedSuggestion.state).toEqual(matchState()); expect(metadataValues).toMatchObject([ [{ value: 'A', label: 'A' }], @@ -874,7 +937,7 @@ describe('suggestions', () => { it('should validate that the ids exist in the dictionary', async () => { const action = async () => { - await prepareAndAcceptSuggestion( + await prepareAndAcceptSelectSuggestion( ['Z', '1A', 'Y', 'A'], 'en', 'property_multiselect', @@ -884,29 +947,41 @@ describe('suggestions', () => { await expect(action()).rejects.toThrow('Ids are invalid: Z, Y (Nested Thesaurus).'); }); - it('should validate that partial acceptance is allowed only for multiselects', async () => { + it('should validate that partial acceptance is allowed only for multiselects/relationships', async () => { const addAction = async () => { - await prepareAndAcceptSuggestion('1A', 'en', 'property_select', 'select_extractor', { - addedValues: ['1A'], - }); + await prepareAndAcceptSelectSuggestion( + '1A', + 'en', + 'property_select', + 'select_extractor', + { + addedValues: ['1A'], + } + ); }; await expect(addAction()).rejects.toThrow( - 'Partial acceptance is only allowed for multiselects.' + 'Partial acceptance is only allowed for multiselects or relationships.' ); const removeAction = async () => { - await prepareAndAcceptSuggestion('1A', 'en', 'property_select', 'select_extractor', { - removedValues: ['1B'], - }); + await prepareAndAcceptSelectSuggestion( + '1A', + 'en', + 'property_select', + 'select_extractor', + { + removedValues: ['1B'], + } + ); }; await expect(removeAction()).rejects.toThrow( - 'Partial acceptance is only allowed for multiselects.' + 'Partial acceptance is only allowed for multiselects or relationships.' ); }); it("should validate that the accepted id's through partial acceptance do exist on the suggestion", async () => { const action = async () => { - await prepareAndAcceptSuggestion( + await prepareAndAcceptSelectSuggestion( ['1A', '1B'], 'en', 'property_multiselect', @@ -923,7 +998,7 @@ describe('suggestions', () => { it("should validate that the id's to remove through partial acceptance do not exist on the suggestion", async () => { const action = async () => { - await prepareAndAcceptSuggestion( + await prepareAndAcceptSelectSuggestion( ['1A', '1B'], 'en', 'property_multiselect', @@ -939,12 +1014,13 @@ describe('suggestions', () => { }); it('should allow full acceptance, and update entites of all languages, with the properly translated labels', async () => { - const { acceptedSuggestion, metadataValues, allFiles } = await prepareAndAcceptSuggestion( - ['1A', '1B'], - 'en', - 'property_multiselect', - 'multiselect_extractor' - ); + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptSelectSuggestion( + ['1A', '1B'], + 'en', + 'property_multiselect', + 'multiselect_extractor' + ); expect(acceptedSuggestion.state).toEqual(matchState()); expect(metadataValues).toMatchObject([ [ @@ -960,15 +1036,16 @@ describe('suggestions', () => { }); it('should allow partial acceptance, and update entites of all languages, with the properly translated labels', async () => { - const { acceptedSuggestion, metadataValues, allFiles } = await prepareAndAcceptSuggestion( - ['B', '1B'], - 'en', - 'property_multiselect', - 'multiselect_extractor', - { - addedValues: ['B'], - } - ); + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptSelectSuggestion( + ['B', '1B'], + 'en', + 'property_multiselect', + 'multiselect_extractor', + { + addedValues: ['B'], + } + ); expect(acceptedSuggestion.state).toEqual(matchState(false)); expect(metadataValues).toMatchObject([ [ @@ -986,15 +1063,16 @@ describe('suggestions', () => { }); it('should do nothing on partial acceptance if the id is already in the entity metadata', async () => { - const { acceptedSuggestion, metadataValues, allFiles } = await prepareAndAcceptSuggestion( - ['1A', '1B'], - 'en', - 'property_multiselect', - 'multiselect_extractor', - { - addedValues: ['1A'], - } - ); + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptSelectSuggestion( + ['1A', '1B'], + 'en', + 'property_multiselect', + 'multiselect_extractor', + { + addedValues: ['1A'], + } + ); expect(acceptedSuggestion.state).toEqual(matchState(false)); expect(metadataValues).toMatchObject([ [ @@ -1010,15 +1088,16 @@ describe('suggestions', () => { }); it('should allow removal through partial acceptance, and update entities of all languages', async () => { - const { acceptedSuggestion, metadataValues, allFiles } = await prepareAndAcceptSuggestion( - ['1A', '1B'], - 'en', - 'property_multiselect', - 'multiselect_extractor', - { - removedValues: ['A'], - } - ); + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptSelectSuggestion( + ['1A', '1B'], + 'en', + 'property_multiselect', + 'multiselect_extractor', + { + removedValues: ['A'], + } + ); expect(acceptedSuggestion.state).toEqual(matchState(false)); expect(metadataValues).toMatchObject([ [{ value: '1A', label: '1A' }], @@ -1028,15 +1107,16 @@ describe('suggestions', () => { }); it('should do nothing on removal through partial acceptance if the id is not in the entity metadata', async () => { - const { acceptedSuggestion, metadataValues, allFiles } = await prepareAndAcceptSuggestion( - ['1A', 'A'], - 'en', - 'property_multiselect', - 'multiselect_extractor', - { - removedValues: ['B'], - } - ); + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptSelectSuggestion( + ['1A', 'A'], + 'en', + 'property_multiselect', + 'multiselect_extractor', + { + removedValues: ['B'], + } + ); expect(acceptedSuggestion.state).toEqual(matchState()); expect(metadataValues).toMatchObject([ [ @@ -1051,6 +1131,328 @@ describe('suggestions', () => { expect(allFiles).toEqual(selectAcceptanceFixtureBase.files); }); }); + + describe('relationship', () => { + beforeEach(async () => { + await db.setupFixturesAndContext(relationshipAcceptanceFixtureBase); + }); + + it('should validate that the entities in the suggestion exist', async () => { + const action = async () => { + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'X_sId', 'S2_sId', 'Y_sId'], + 'en', + 'relationship_to_source', + 'relationship_extractor' + ); + }; + await expect(action()).rejects.toThrow( + 'The following sharedIds do not exist in the database: X_sId, Y_sId.' + ); + }); + + it("should validate that the accepted id's through partial acceptance do exist on the suggestion", async () => { + const action = async () => { + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'S2_sId'], + 'en', + 'relationship_to_source', + 'relationship_extractor', + { + addedValues: ['S1_sId', 'X_sId', 'Y_sId'], + } + ); + }; + await expect(action()).rejects.toThrow( + 'Some of the accepted values do not exist in the suggestion: X_sId, Y_sId. Cannot accept values that are not suggested.' + ); + }); + + it("should validate that the id's to remove through partial acceptance do not exist on the suggestion", async () => { + const action = async () => { + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'S2_sId'], + 'en', + 'relationship_to_source', + 'relationship_extractor', + { + removedValues: ['S1_sId', 'S0_sId'], + } + ); + }; + await expect(action()).rejects.toThrow( + 'Some of the removed values exist in the suggestion: S1_sId. Cannot remove values that are suggested.' + ); + }); + + it('should allow full acceptance, and update entites of all languages, with the properly translated labels', async () => { + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'S3_sId'], + 'en', + 'relationship_to_source', + 'relationship_extractor' + ); + expect(acceptedSuggestion.state).toEqual(matchState(true)); + expect(metadataValues).toMatchObject([ + [ + { value: 'S1_sId', label: 'S1' }, + { value: 'S3_sId', label: 'S3' }, + ], + [ + { value: 'S1_sId', label: 'S1_es' }, + { value: 'S3_sId', label: 'S3_es' }, + ], + ]); + expect(allFiles).toEqual(relationshipAcceptanceFixtureBase.files); + }); + + it('should allow partial acceptance, and update entites of all languages, with the properly translated labels', async () => { + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'S3_sId'], + 'en', + 'relationship_to_source', + 'relationship_extractor', + { + addedValues: ['S3_sId'], + } + ); + expect(acceptedSuggestion.state).toEqual(matchState(false)); + expect(metadataValues).toMatchObject([ + [ + { value: 'S1_sId', label: 'S1' }, + { value: 'S2_sId', label: 'S2' }, + { value: 'S3_sId', label: 'S3' }, + ], + [ + { value: 'S1_sId', label: 'S1_es' }, + { value: 'S2_sId', label: 'S2_es' }, + { value: 'S3_sId', label: 'S3_es' }, + ], + ]); + expect(allFiles).toEqual(relationshipAcceptanceFixtureBase.files); + }); + + it('should do nothing on partial acceptance if the id is already in the entity metadata', async () => { + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'S3_sId'], + 'en', + 'relationship_to_source', + 'relationship_extractor', + { + addedValues: ['S1_sId'], + } + ); + expect(acceptedSuggestion.state).toEqual(matchState(false)); + expect(metadataValues).toMatchObject([ + [ + { value: 'S1_sId', label: 'S1' }, + { value: 'S2_sId', label: 'S2' }, + ], + [ + { value: 'S1_sId', label: 'S1_es' }, + { value: 'S2_sId', label: 'S2_es' }, + ], + ]); + expect(allFiles).toEqual(relationshipAcceptanceFixtureBase.files); + }); + + it('should allow removal through partial acceptance, and update entities of all languages', async () => { + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'S3_sId'], + 'en', + 'relationship_to_source', + 'relationship_extractor', + { + removedValues: ['S2_sId'], + } + ); + expect(acceptedSuggestion.state).toEqual(matchState(false)); + expect(metadataValues).toMatchObject([ + [{ value: 'S1_sId', label: 'S1' }], + [{ value: 'S1_sId', label: 'S1_es' }], + ]); + expect(allFiles).toEqual(relationshipAcceptanceFixtureBase.files); + }); + + it('should do nothing on removal through partial acceptance if the id is not in the entity metadata', async () => { + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'S2_sId'], + 'en', + 'relationship_to_source', + 'relationship_extractor', + { + removedValues: ['S3_sId'], + } + ); + expect(acceptedSuggestion.state).toEqual(matchState(true)); + expect(metadataValues).toMatchObject([ + [ + { value: 'S1_sId', label: 'S1' }, + { value: 'S2_sId', label: 'S2' }, + ], + [ + { value: 'S1_sId', label: 'S1_es' }, + { value: 'S2_sId', label: 'S2_es' }, + ], + ]); + expect(allFiles).toEqual(relationshipAcceptanceFixtureBase.files); + }); + + it('should update inherited values per language', async () => { + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'S3_sId'], + 'en', + 'relationship_with_inheritance', + 'relationship_with_inheritance_extractor' + ); + expect(acceptedSuggestion.state).toEqual(matchState(true)); + expect(metadataValues).toMatchObject([ + [ + { + value: 'S1_sId', + label: 'S1', + inheritedType: 'text', + inheritedValue: [ + { + value: 'inherited text', + }, + ], + }, + { + value: 'S3_sId', + label: 'S3', + inheritedType: 'text', + inheritedValue: [ + { + value: 'inherited text 3', + }, + ], + }, + ], + [ + { + value: 'S1_sId', + label: 'S1_es', + inheritedType: 'text', + inheritedValue: [ + { + value: 'inherited text Spanish', + }, + ], + }, + { + value: 'S3_sId', + label: 'S3_es', + inheritedType: 'text', + inheritedValue: [ + { + value: 'inherited text 3 Spanish', + }, + ], + }, + ], + ]); + expect(allFiles).toEqual(relationshipAcceptanceFixtureBase.files); + }); + + it('should check if the suggested entities are of the correct template', async () => { + const action = async () => { + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'other_source'], + 'en', + 'relationship_to_source', + 'relationship_extractor' + ); + }; + await expect(action()).rejects.toThrow( + 'The following sharedIds do not match the content template in the relationship property: other_source.' + ); + }); + + it('should handle relationship properties with any template as content', async () => { + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptRelationshipSuggestion( + ['S2_sId', 'other_source_2'], + 'en', + 'relationship_to_any', + 'relationship_to_any_extractor' + ); + expect(acceptedSuggestion.state).toEqual(matchState(true)); + expect(metadataValues).toMatchObject([ + [ + { + value: 'S2_sId', + label: 'S2', + }, + { + value: 'other_source_2', + label: 'Other Source 2', + }, + ], + [ + { + value: 'S2_sId', + label: 'S2_es', + }, + { + value: 'other_source_2', + label: 'Other Source 2 Spanish', + }, + ], + ]); + expect(allFiles).toEqual(relationshipAcceptanceFixtureBase.files); + }); + + it('should remove or create connections as necessary', async () => { + const { acceptedSuggestion, metadataValues, allFiles } = + await prepareAndAcceptRelationshipSuggestion( + ['S1_sId', 'S3_sId'], + 'en', + 'relationship_to_source', + 'relationship_extractor' + ); + expect(acceptedSuggestion.state).toEqual(matchState(true)); + expect(metadataValues).toMatchObject([ + [ + { value: 'S1_sId', label: 'S1' }, + { value: 'S3_sId', label: 'S3' }, + ], + [ + { value: 'S1_sId', label: 'S1_es' }, + { value: 'S3_sId', label: 'S3_es' }, + ], + ]); + expect(allFiles).toEqual(relationshipAcceptanceFixtureBase.files); + + const removedConnection = await db.mongodb + ?.collection('connections') + .findOne({ entity: 'S2_sId', template: factory.id('related') }); + expect(removedConnection).toBeNull(); + + const newConnection = await db.mongodb + ?.collection('connections') + .findOne({ entity: 'S3_sId' }); + expect(newConnection).toMatchObject({ + entity: 'S3_sId', + hub: expect.any(ObjectId), + template: factory.id('related'), + }); + const newHub = newConnection?.hub; + const pairedConnection = await db.mongodb + ?.collection('connections') + .findOne({ entity: 'entityWithRelationships_sId', hub: newHub }); + expect(pairedConnection).toMatchObject({ + entity: 'entityWithRelationships_sId', + hub: newHub, + }); + }); + }); }); describe('save()', () => { diff --git a/app/api/suggestions/suggestions.ts b/app/api/suggestions/suggestions.ts index 225181bcc0..0e5f1ebd21 100644 --- a/app/api/suggestions/suggestions.ts +++ b/app/api/suggestions/suggestions.ts @@ -23,7 +23,7 @@ import { import { objectIndex } from 'shared/data_utils/objectIndex'; import { getSegmentedFilesIds, - propertyTypeIsSelectOrMultiSelect, + propertyTypeIsWithoutExtractedMetadata, } from 'api/services/informationextraction/getFiles'; import { Extractors } from 'api/services/informationextraction/ixextractors'; import { registerEventListeners } from './eventListeners'; @@ -48,7 +48,7 @@ const updateExtractedMetadata = async ( suggestions: IXSuggestionType[], property: PropertySchema ) => { - if (propertyTypeIsSelectOrMultiSelect(property.type)) return; + if (propertyTypeIsWithoutExtractedMetadata(property.type)) return; const fetchedFiles = await files.get({ _id: { $in: suggestions.map(s => s.fileId) } }); const suggestionsByFileId = objectIndex( @@ -205,15 +205,22 @@ const propertyTypesWithAllLanguages = new Set(['numeric', 'date', 'select', 'mul const needsAllLanguages = (propertyType: PropertySchema['type']) => propertyTypesWithAllLanguages.has(propertyType); +const validTypesForPartialAcceptance = new Set(['multiselect', 'relationship']); + +const typeIsValidForPartialAcceptance = (propertyType: string) => + validTypesForPartialAcceptance.has(propertyType); + const validatePartialAcceptanceTypeConstraint = ( acceptedSuggestions: AcceptedSuggestion[], property: PropertySchema ) => { const addedValuesExist = acceptedSuggestions.some(s => s.addedValues); const removedValuesExist = acceptedSuggestions.some(s => s.removedValues); - const multiSelectOnly = addedValuesExist || removedValuesExist; - if (property.type !== 'multiselect' && multiSelectOnly) { - throw new SuggestionAcceptanceError('Partial acceptance is only allowed for multiselects.'); + const partialAcceptanceTriggered = addedValuesExist || removedValuesExist; + if (!typeIsValidForPartialAcceptance(property.type) && partialAcceptanceTriggered) { + throw new SuggestionAcceptanceError( + 'Partial acceptance is only allowed for multiselects or relationships.' + ); } }; diff --git a/app/api/suggestions/updateEntities.ts b/app/api/suggestions/updateEntities.ts index d66e97090c..2639819f95 100644 --- a/app/api/suggestions/updateEntities.ts +++ b/app/api/suggestions/updateEntities.ts @@ -1,9 +1,11 @@ import entities from 'api/entities'; -import translations from 'api/i18n/translations'; +import { checkTypeIsAllowed } from 'api/services/informationextraction/ixextractors'; import thesauri from 'api/thesauri'; import { flatThesaurusValues } from 'api/thesauri/thesauri'; +import { ObjectId } from 'mongodb'; import { arrayBidirectionalDiff } from 'shared/data_utils/arrayBidirectionalDiff'; import { IndexTypes, objectIndex } from 'shared/data_utils/objectIndex'; +import { syncedPromiseLoop } from 'shared/data_utils/promiseUtils'; import { setIntersection } from 'shared/data_utils/setUtils'; import { ObjectIdSchema, PropertySchema } from 'shared/types/commonTypes'; import { EntitySchema } from 'shared/types/entityType'; @@ -19,6 +21,10 @@ interface AcceptedSuggestion { removedValues?: string[]; } +type EntityInfo = Record; + +const fetchNoResources = async () => ({}); + const fetchThesaurus = async (thesaurusId: PropertySchema['content']) => { const dict = await thesauri.getById(thesaurusId); const thesaurusName = dict!.name; @@ -31,39 +37,61 @@ const fetchThesaurus = async (thesaurusId: PropertySchema['content']) => { return { name: thesaurusName, id: thesaurusId, indexedlabels }; }; -const fetchTranslations = async (property: PropertySchema) => { - const trs = await translations.get({ context: property.content }); - const indexed = objectIndex( - trs, - t => t.locale || '', - t => t.contexts?.[0].values +const fetchEntityInfo = async ( + _property: PropertySchema, + acceptedSuggestions: AcceptedSuggestion[], + suggestions: IXSuggestionType[] +): Promise<{ entityInfo: EntityInfo }> => { + const suggestionSharedIds = suggestions.map(s => s.suggestedValue).flat(); + const addedSharedIds = acceptedSuggestions.map(s => s.addedValues || []).flat(); + const expectedSharedIds = Array.from(new Set(suggestionSharedIds.concat(addedSharedIds))); + const entitiesInDb = (await entities.get({ sharedId: { $in: expectedSharedIds } }, [ + 'sharedId', + 'template', + ])) as { sharedId: string; template: ObjectId }[]; + const indexedBySharedId = objectIndex( + entitiesInDb, + e => e.sharedId, + e => e ); - return indexed; + return { entityInfo: indexedBySharedId }; }; const fetchSelectResources = async (property: PropertySchema) => { const thesaurus = await fetchThesaurus(property.content); - const labelTranslations = await fetchTranslations(property); - return { thesaurus, translations: labelTranslations }; + return { thesaurus }; }; const resourceFetchers = { - _default: async () => ({}), + title: fetchNoResources, + text: fetchNoResources, + numeric: fetchNoResources, + date: fetchNoResources, select: fetchSelectResources, multiselect: fetchSelectResources, + relationship: fetchEntityInfo, }; -const fetchResources = async (property: PropertySchema) => { - // @ts-ignore - const fetcher = resourceFetchers[property.type] || resourceFetchers._default; - return fetcher(property); +const fetchResources = async ( + property: PropertySchema, + acceptedSuggestions: AcceptedSuggestion[], + suggestions: IXSuggestionType[] +) => { + const type = checkTypeIsAllowed(property.type); + const fetcher = resourceFetchers[type]; + return fetcher(property, acceptedSuggestions, suggestions); }; +const getAcceptedSuggestion = ( + entity: EntitySchema, + acceptedSuggestionsBySharedId: Record +): AcceptedSuggestion => acceptedSuggestionsBySharedId[entity.sharedId || '']; + const getSuggestion = ( entity: EntitySchema, suggestionsById: Record, acceptedSuggestionsBySharedId: Record -) => suggestionsById[acceptedSuggestionsBySharedId[entity.sharedId || '']._id.toString()]; +) => suggestionsById[getAcceptedSuggestion(entity, acceptedSuggestionsBySharedId)._id.toString()]; const getRawValue = ( entity: EntitySchema, @@ -88,17 +116,6 @@ const checkValuesInThesaurus = ( } }; -const mapLabels = ( - values: string[], - entity: EntitySchema, - thesaurus: { indexedlabels: Record }, - translation: Record> -) => { - const labels = values.map(v => thesaurus.indexedlabels[v]); - const translatedLabels = labels.map(l => translation[entity.language || '']?.[l]); - return values.map((value, index) => ({ value, label: translatedLabels[index] })); -}; - function readAddedValues(acceptedSuggestion: AcceptedSuggestion, suggestionValues: string[]) { const addedValues = acceptedSuggestion.addedValues || []; const addedButNotSuggested = arrayBidirectionalDiff( @@ -146,7 +163,7 @@ function mixFinalValues( return finalValues; } -function arrangeValues( +function arrangeAddedOrRemovedValues( acceptedSuggestion: AcceptedSuggestion, suggestionValues: string[], entity: EntitySchema, @@ -163,36 +180,66 @@ function arrangeValues( return finalValues; } +function checkSharedIds(values: string[], entityInfo: EntityInfo) { + const missingSharedIds = values.filter(v => !(v in entityInfo)); + if (missingSharedIds.length > 0) { + throw new SuggestionAcceptanceError( + `The following sharedIds do not exist in the database: ${missingSharedIds.join(', ')}.` + ); + } +} + +function checkTemplates(property: PropertySchema, values: string[], entityInfo: EntityInfo) { + const { content } = property; + if (!content) return; + const templateId = new ObjectId(content); + const wrongTemplatedSharedIds = values.filter( + v => entityInfo[v].template.toString() !== templateId.toString() + ); + if (wrongTemplatedSharedIds.length > 0) { + throw new SuggestionAcceptanceError( + `The following sharedIds do not match the content template in the relationship property: ${wrongTemplatedSharedIds.join(', ')}.` + ); + } +} + +const getRawValueAsArray = ( + _property: PropertySchema, + entity: EntitySchema, + suggestionsById: Record, + acceptedSuggestionsBySharedId: Record +) => [ + { + value: getRawValue(entity, suggestionsById, acceptedSuggestionsBySharedId), + }, +]; + const valueGetters = { - _default: ( - entity: EntitySchema, - suggestionsById: Record, - acceptedSuggestionsBySharedId: Record - ) => [ - { - value: getRawValue(entity, suggestionsById, acceptedSuggestionsBySharedId), - }, - ], + text: getRawValueAsArray, + date: getRawValueAsArray, + numeric: getRawValueAsArray, select: ( + _property: PropertySchema, entity: EntitySchema, suggestionsById: Record, acceptedSuggestionsBySharedId: Record, resources: any ) => { - const { thesaurus, translations: translation } = resources; + const { thesaurus } = resources; const value = getRawValue(entity, suggestionsById, acceptedSuggestionsBySharedId) as string; checkValuesInThesaurus([value], thesaurus.name, thesaurus.indexedlabels); - return mapLabels([value], entity, thesaurus, translation); + return [{ value }]; }, multiselect: ( + _property: PropertySchema, entity: EntitySchema, suggestionsById: Record, acceptedSuggestionsBySharedId: Record, resources: any ) => { - const { thesaurus, translations: translation } = resources; - const acceptedSuggestion = acceptedSuggestionsBySharedId[entity.sharedId || '']; + const { thesaurus } = resources; + const acceptedSuggestion = getAcceptedSuggestion(entity, acceptedSuggestionsBySharedId); const suggestion = getSuggestion(entity, suggestionsById, acceptedSuggestionsBySharedId); const suggestionValues = getRawValue( entity, @@ -201,14 +248,42 @@ const valueGetters = { ) as string[]; checkValuesInThesaurus(suggestionValues, thesaurus.name, thesaurus.indexedlabels); - const finalValues: string[] = arrangeValues( + const finalValues: string[] = arrangeAddedOrRemovedValues( acceptedSuggestion, suggestionValues, entity, suggestion ); - return mapLabels(finalValues, entity, thesaurus, translation); + return finalValues.map(value => ({ value })); + }, + relationship: ( + property: PropertySchema, + entity: EntitySchema, + suggestionsById: Record, + acceptedSuggestionsBySharedId: Record, + resources: any + ) => { + const { entityInfo } = resources; + + const acceptedSuggestion = getAcceptedSuggestion(entity, acceptedSuggestionsBySharedId); + const suggestion = getSuggestion(entity, suggestionsById, acceptedSuggestionsBySharedId); + const suggestionValues = getRawValue( + entity, + suggestionsById, + acceptedSuggestionsBySharedId + ) as string[]; + checkSharedIds(suggestionValues, entityInfo); + checkTemplates(property, suggestionValues, entityInfo); + + const finalValues: string[] = arrangeAddedOrRemovedValues( + acceptedSuggestion, + suggestionValues, + entity, + suggestion + ); + + return finalValues.map(value => ({ value })); }, }; @@ -219,9 +294,18 @@ const getValue = ( acceptedSuggestionsBySharedId: Record, resources: any ) => { - // @ts-ignore - const getter = valueGetters[property.type] || valueGetters._default; - return getter(entity, suggestionsById, acceptedSuggestionsBySharedId, resources); + const type = checkTypeIsAllowed(property.type); + if (type === 'title') { + throw new SuggestionAcceptanceError('Title should not be handled here.'); + } + const getter = valueGetters[type]; + return getter(property, entity, suggestionsById, acceptedSuggestionsBySharedId, resources); +}; + +const saveEntities = async (entitiesToUpdate: EntitySchema[]) => { + await syncedPromiseLoop(entitiesToUpdate, async (entity: EntitySchema) => { + await entities.save(entity, { user: {}, language: entity.language }); + }); }; const updateEntitiesWithSuggestion = async ( @@ -249,7 +333,7 @@ const updateEntitiesWithSuggestion = async ( s => s ); - const resources = await fetchResources(property); + const resources = await fetchResources(property, acceptedSuggestions, suggestions); const entitiesToUpdate = propertyName !== 'title' @@ -272,7 +356,7 @@ const updateEntitiesWithSuggestion = async ( title: getRawValue(entity, suggestionsById, acceptedSuggestionsBySharedId), })); - await entities.saveMultiple(entitiesToUpdate); + await saveEntities(entitiesToUpdate); }; export { updateEntitiesWithSuggestion, SuggestionAcceptanceError }; diff --git a/app/api/suggestions/updateState.ts b/app/api/suggestions/updateState.ts index 78bb8a1e25..a352033ed0 100644 --- a/app/api/suggestions/updateState.ts +++ b/app/api/suggestions/updateState.ts @@ -2,7 +2,7 @@ import settings from 'api/settings'; import templates from 'api/templates'; import { objectIndex } from 'shared/data_utils/objectIndex'; import { CurrentValue, getSuggestionState, SuggestionValues } from 'shared/getIXSuggestionState'; -import { propertyIsMultiselect } from 'shared/propertyTypes'; +import { propertyIsMultiselect, propertyIsRelationship } from 'shared/propertyTypes'; import { LanguagesListSchema, PropertyTypeSchema } from 'shared/types/commonTypes'; import { IXSuggestionsModel } from './IXSuggestionsModel'; import { @@ -95,7 +95,9 @@ const postProcessCurrentValue = ( suggestion: SuggestionsAggregationResult, propertyType: PropertyTypeSchema ): PostProcessedAggregationResult => { - if (propertyIsMultiselect(propertyType)) return suggestion; + if (propertyIsMultiselect(propertyType) || propertyIsRelationship(propertyType)) { + return suggestion; + } return { ...suggestion, currentValue: suggestion.currentValue.length > 0 ? suggestion.currentValue[0] : '', diff --git a/app/api/users/routes.js b/app/api/users/routes.js index 779df0fd26..a8db8cbfbc 100644 --- a/app/api/users/routes.js +++ b/app/api/users/routes.js @@ -1,6 +1,6 @@ import { parseQuery, validation } from 'api/utils'; import { userSchema } from 'shared/types/userSchema'; -import needsAuthorization from '../auth/authMiddleware'; +import { needsAuthorization, validatePasswordMiddleWare } from '../auth'; import users from './users'; const getDomain = req => `${req.protocol}://${req.get('host')}`; @@ -8,6 +8,7 @@ export default app => { app.post( '/api/users', needsAuthorization(['admin', 'editor', 'collaborator']), + validatePasswordMiddleWare, validation.validateRequest({ type: 'object', properties: { @@ -27,6 +28,7 @@ export default app => { app.post( '/api/users/new', needsAuthorization(), + validatePasswordMiddleWare, validation.validateRequest({ type: 'object', properties: { @@ -45,6 +47,7 @@ export default app => { app.post( '/api/users/unlock', needsAuthorization(), + validatePasswordMiddleWare, validation.validateRequest({ type: 'object', properties: { @@ -149,6 +152,7 @@ export default app => { '/api/users', needsAuthorization(), parseQuery, + validatePasswordMiddleWare, validation.validateRequest({ type: 'object', properties: { diff --git a/app/api/users/specs/routes.spec.ts b/app/api/users/specs/routes.spec.ts index a082be226e..37adaed6dc 100644 --- a/app/api/users/specs/routes.spec.ts +++ b/app/api/users/specs/routes.spec.ts @@ -19,6 +19,16 @@ jest.mock( } ); +jest.mock('../../auth', () => { + const originalModule = jest.requireActual('../../auth'); + return { + ...originalModule, + validatePasswordMiddleWare: jest.fn((_req: Request, _res: Response, next: NextFunction) => { + next(); + }), + }; +}); + const invalidUserProperties = [ { field: 'username', value: undefined, instancePath: '/body', keyword: 'required' }, { field: 'email', value: undefined, instancePath: '/body', keyword: 'required' }, @@ -32,6 +42,7 @@ const invalidUserProperties = [ const adminUser = { _id: 'admin1', username: 'Admin 1', + password: 'admin124', role: UserRole.ADMIN, email: 'admin@test.com', }; diff --git a/app/api/users/specs/users.spec.js b/app/api/users/specs/users.spec.js index 15a8e84ee0..63bb517206 100644 --- a/app/api/users/specs/users.spec.js +++ b/app/api/users/specs/users.spec.js @@ -7,7 +7,7 @@ import mailer from 'api/utils/mailer'; import db from 'api/utils/testing_db'; import * as random from 'shared/uniqueID'; -import encryptPassword, { comparePasswords } from 'api/auth/encryptPassword'; +import { encryptPassword, comparePasswords } from 'api/auth/encryptPassword'; import * as usersUtils from 'api/auth2fa/usersUtils'; import { settingsModel } from 'api/settings/settingsModel'; import userGroups from 'api/usergroups/userGroups'; diff --git a/app/api/users/users.js b/app/api/users/users.js index 2e8dff14a4..1ee897b591 100644 --- a/app/api/users/users.js +++ b/app/api/users/users.js @@ -2,7 +2,7 @@ import SHA256 from 'crypto-js/sha256'; import { createError } from 'api/utils'; import random from 'shared/uniqueID'; -import encryptPassword, { comparePasswords } from 'api/auth/encryptPassword'; +import { encryptPassword, comparePasswords } from 'api/auth/encryptPassword'; import * as usersUtils from 'api/auth2fa/usersUtils'; import { @@ -323,3 +323,5 @@ export default { throw createError('key not found', 403); }, }; + +export { validateUserPassword }; diff --git a/app/react/App/styles/globals.css b/app/react/App/styles/globals.css index 440b13a002..3fad0b9fc4 100644 --- a/app/react/App/styles/globals.css +++ b/app/react/App/styles/globals.css @@ -1833,6 +1833,10 @@ input[type="range"]::-ms-fill-lower { margin-top: 1.5rem; } +.ml-1 { + margin-left: 0.25rem; +} + .block { display: block; } @@ -2924,6 +2928,11 @@ input[type="range"]::-ms-fill-lower { background-color: rgb(67 56 202 / var(--tw-bg-opacity)); } +.bg-primary-900 { + --tw-bg-opacity: 1; + background-color: rgb(49 46 129 / var(--tw-bg-opacity)); +} + .bg-red-100 { --tw-bg-opacity: 1; background-color: rgb(253 232 232 / var(--tw-bg-opacity)); @@ -3125,6 +3134,16 @@ input[type="range"]::-ms-fill-lower { padding-bottom: 3px; } +.py-\[2px\] { + padding-top: 2px; + padding-bottom: 2px; +} + +.py-\[1px\] { + padding-top: 1px; + padding-bottom: 1px; +} + .pb-14 { padding-bottom: 3.5rem; } @@ -3270,6 +3289,14 @@ input[type="range"]::-ms-fill-lower { line-height: 1rem; } +.text-\[8px\] { + font-size: 8px; +} + +.text-\[10px\] { + font-size: 10px; +} + .font-black { font-weight: 900; } @@ -3519,6 +3546,11 @@ input[type="range"]::-ms-fill-lower { color: rgb(114 59 19 / var(--tw-text-opacity)); } +.text-primary-900 { + --tw-text-opacity: 1; + color: rgb(49 46 129 / var(--tw-text-opacity)); +} + .underline { text-decoration-line: underline; } diff --git a/app/react/Markdown/MarkdownViewer.js b/app/react/Markdown/MarkdownViewer.js index 2d459fc3e1..8561a3591e 100644 --- a/app/react/Markdown/MarkdownViewer.js +++ b/app/react/Markdown/MarkdownViewer.js @@ -6,6 +6,7 @@ import { MarkdownLink, SearchBox, MarkdownMedia, ItemList } from './components'; import CustomHookComponents from './CustomHooks'; import markdownToReact from './markdownToReact'; +import { ValidatedElement } from './ValidatedElement'; class MarkdownViewer extends Component { static errorHtml(index, message) { @@ -116,7 +117,11 @@ class MarkdownViewer extends Component { return false; } - return
{ReactFromMarkdown}
; + return ValidatedElement( + 'div', + { className: 'markdown-viewer' }, + ...React.Children.toArray(ReactFromMarkdown) + ); } } diff --git a/app/react/Markdown/ValidatedElement.tsx b/app/react/Markdown/ValidatedElement.tsx new file mode 100644 index 0000000000..3ba6a277d0 --- /dev/null +++ b/app/react/Markdown/ValidatedElement.tsx @@ -0,0 +1,37 @@ +import React from 'react'; +import { validHtmlTags } from './utils'; + +const isValidTagName = (tagName: string): boolean => validHtmlTags.has(tagName); + +const ValidatedElement = ( + type: string | React.JSXElementConstructor, + props: (React.Attributes & { children?: React.ReactNode }) | null, + ...children: React.ReactNode[] +): React.ReactElement | null => { + if (typeof type === 'string' && !isValidTagName(type)) { + return React.createElement('div', { className: 'error' }, `Invalid tag: ${type}`); + } + + const validatedChildren = children.map(child => { + if (Array.isArray(child)) { + return child.map(c => { + const childProps = c.props as React.Attributes & { children?: React.ReactNode }; + return React.isValidElement(c) + ? ValidatedElement(c.type, childProps, ...React.Children.toArray(childProps.children)) + : c; + }); + } + if (React.isValidElement(child)) { + return ValidatedElement( + child.type, + child.props as React.Attributes & { children?: React.ReactNode }, + ...React.Children.toArray(child.props.children) + ); + } + return child; + }); + + return React.createElement(type, props, ...validatedChildren); +}; + +export { ValidatedElement }; diff --git a/app/react/Markdown/specs/MarkdownViewer.spec.js b/app/react/Markdown/specs/MarkdownViewer.spec.js index ab7a4a4f41..7484f7b619 100644 --- a/app/react/Markdown/specs/MarkdownViewer.spec.js +++ b/app/react/Markdown/specs/MarkdownViewer.spec.js @@ -157,13 +157,22 @@ describe('MarkdownViewer', () => { }); describe('when not valid html', () => { - it('should not fail', () => { + it('should not fail on malformed tags', () => { props.markdown = '
'; props.html = true; render(); expect(component).toMatchSnapshot(); }); + + it('should not fail on unsupported tags', () => { + props.html = true; + props.markdown = + "Little red ridding hood\n

I don't know <..long pause..> a minute later <..pause..> a grandma story.\nWhen I heard it

"; + + render(); + expect(component).toMatchSnapshot(); + }); }); it('should render a searchbox', () => { diff --git a/app/react/Markdown/specs/__snapshots__/MarkdownViewer.spec.js.snap b/app/react/Markdown/specs/__snapshots__/MarkdownViewer.spec.js.snap index d6c5660cd8..49a174f09e 100644 --- a/app/react/Markdown/specs/__snapshots__/MarkdownViewer.spec.js.snap +++ b/app/react/Markdown/specs/__snapshots__/MarkdownViewer.spec.js.snap @@ -4,24 +4,18 @@ exports[`MarkdownViewer render should be able to render properly custom componen
-
+
-
+
@@ -33,9 +27,7 @@ exports[`MarkdownViewer render should not render html by default 1`] = `
-

+

<div><h1>should be all a escaped string</h1></div>

@@ -47,9 +39,7 @@ exports[`MarkdownViewer render should remove Dataset and Query tags 1`] = `
-

+

test

@@ -60,16 +50,13 @@ exports[`MarkdownViewer render should remove Dataset and Query tags 1`] = ` -
+
test
@@ -79,75 +66,39 @@ exports[`MarkdownViewer render should remove whitespaces between table tags (to
- - - -
+ + + + - - - - - - + + - - - + -
cadh + cbdp + cidfp + cipst
+
1.1,25.1 - - + + +
+
1.1,21.1,21.2,25,1 - - + + +
@@ -162,18 +113,14 @@ exports[`MarkdownViewer render should render Link 1`] = ` > -

+

label @@ -190,7 +137,6 @@ exports[`MarkdownViewer render should render a searchbox 1`] = ` > @@ -219,27 +165,22 @@ exports[`MarkdownViewer render should render customHook components and show an e ] } component="validcomponent" - key="0" prop="a prop" /> -

+

Should allow markdown between hooks


@@ -270,7 +211,6 @@ exports[`MarkdownViewer render should render list components 1`] = ` "items1", ] } - key="0" link="/library/param1" options={ Object { @@ -286,7 +226,6 @@ exports[`MarkdownViewer render should render list components 1`] = ` "items2", ] } - key="2" link="/library/param2" options={Object {}} /> @@ -298,7 +237,6 @@ exports[`MarkdownViewer render should render list components 1`] = ` "items3", ] } - key="4" link="/library/param3" options={Object {}} /> @@ -311,20 +249,15 @@ exports[`MarkdownViewer render should render markdown 1`] = `

-

+

title

-

+

Some text with a URL @@ -332,19 +265,14 @@ exports[`MarkdownViewer render should render markdown 1`] = `

-

+

Which should be in its own line, separated with TWO line breaks (to create a new <p> Element)

-
+  
     
       Code
 
@@ -359,35 +287,26 @@ exports[`MarkdownViewer render should render media components 1`] = `
 
-
+
-
+
-
+
@@ -399,23 +318,18 @@ exports[`MarkdownViewer render should render properly a selfclosing XML tags 1`]
-

+

test

-
+
test
@@ -425,21 +339,15 @@ exports[`MarkdownViewer render should render single tags as self closing 1`] = `
-

+

test

-
+
-

+

test

@@ -453,11 +361,8 @@ exports[`MarkdownViewer render should support containers with custom classNames >
-

+

text inside a div

@@ -468,13 +373,33 @@ exports[`MarkdownViewer render should support containers with custom classNames exports[`MarkdownViewer render when markdown is invalid should not render anything when its empty 1`] = `""`; -exports[`MarkdownViewer render when not valid html should not fail 1`] = ` +exports[`MarkdownViewer render when not valid html should not fail on malformed tags 1`] = `
-
+
+
+`; + +exports[`MarkdownViewer render when not valid html should not fail on unsupported tags 1`] = ` +
+

+ + Little red ridding hood + +

+ + +

+ I don't know +

+ Invalid tag: ..long +
+

`; @@ -482,16 +407,13 @@ exports[`MarkdownViewer render when passing html true prop should render customC
-
+
-
-
+
+
@@ -520,12 +437,8 @@ exports[`MarkdownViewer render when passing html true prop should render html 1`
-
-

+
+

test

diff --git a/app/react/Markdown/utils.js b/app/react/Markdown/utils.js index ee77685fe9..2c57a9f647 100644 --- a/app/react/Markdown/utils.js +++ b/app/react/Markdown/utils.js @@ -1,4 +1,4 @@ -export const objectPath = (path, object) => +const objectPath = (path, object) => path.split('.').reduce((o, key) => { if (!o || !key) { return o; @@ -6,10 +6,181 @@ export const objectPath = (path, object) => return o.toJS ? o.get(key) : o[key]; }, object); -export const logError = (err, propValueOf, propLabelOf) => { +const logError = (err, propValueOf, propLabelOf) => { /* eslint-disable no-console */ console.error('Error on EntityData: '); console.error('value-of: ', propValueOf, '; label-of: ', propLabelOf); console.error(err); /* eslint-enable no-console */ }; + +const validHtmlTags = new Set([ + 'a', + 'abbr', + 'acronym', + 'address', + 'area', + 'article', + 'aside', + 'audio', + 'b', + 'bdi', + 'bdo', + 'big', + 'blink', + 'blockquote', + 'body', + 'br', + 'button', + 'canvas', + 'caption', + 'center', + 'circle', + 'cite', + 'clipPath', + 'code', + 'col', + 'colgroup', + 'content', + 'data', + 'datalist', + 'dd', + 'decorator', + 'defs', + 'del', + 'desc', + 'details', + 'dfn', + 'dir', + 'div', + 'dl', + 'dt', + 'element', + 'ellipse', + 'em', + 'feBlend', + 'feColorMatrix', + 'feComponentTransfer', + 'feComposite', + 'feConvolveMatrix', + 'feDiffuseLighting', + 'feDisplacementMap', + 'feDistantLight', + 'feFlood', + 'feFuncA', + 'feFuncB', + 'feFuncG', + 'feFuncR', + 'feGaussianBlur', + 'feImage', + 'feMerge', + 'feMergeNode', + 'feMorphology', + 'feOffset', + 'fePointLight', + 'feSpecularLighting', + 'feSpotLight', + 'feTile', + 'feTurbulence', + 'fieldset', + 'figcaption', + 'figure', + 'filter', + 'font', + 'footer', + 'foreignObject', + 'form', + 'g', + 'h1', + 'h2', + 'h3', + 'h4', + 'h5', + 'h6', + 'head', + 'header', + 'hgroup', + 'hr', + 'html', + 'i', + 'iframe', + 'image', + 'img', + 'input', + 'ins', + 'kbd', + 'label', + 'legend', + 'li', + 'line', + 'linearGradient', + 'main', + 'map', + 'mark', + 'marker', + 'marquee', + 'mask', + 'menu', + 'menuitem', + 'metadata', + 'meter', + 'nav', + 'nobr', + 'ol', + 'optgroup', + 'option', + 'output', + 'p', + 'path', + 'pattern', + 'polygon', + 'polyline', + 'pre', + 'progress', + 'q', + 'radialGradient', + 'rect', + 'rp', + 'rt', + 'ruby', + 's', + 'samp', + 'section', + 'select', + 'shadow', + 'small', + 'source', + 'spacer', + 'span', + 'strike', + 'strong', + 'style', + 'sub', + 'summary', + 'sup', + 'svg', + 'switch', + 'symbol', + 'table', + 'tbody', + 'td', + 'template', + 'textarea', + 'tfoot', + 'th', + 'thead', + 'time', + 'tr', + 'track', + 'tspan', + 'tt', + 'u', + 'ul', + 'use', + 'var', + 'video', + 'view', + 'wbr', +]); + +export { objectPath, logError, validHtmlTags }; diff --git a/app/react/Routes.tsx b/app/react/Routes.tsx index 2b175ba633..7a586d3b51 100644 --- a/app/react/Routes.tsx +++ b/app/react/Routes.tsx @@ -113,16 +113,18 @@ const getRoutesLayout = ( )} /> )} /> - )} - loader={IXdashboardLoader(headers)} - /> - )} - /> + + )} + loader={IXdashboardLoader(headers)} + /> + )} + /> + }> {layout} - + {layout} } /> diff --git a/app/react/V2/Components/Forms/MultiselectList.tsx b/app/react/V2/Components/Forms/MultiselectList.tsx index f1d172f781..a981a2aff0 100644 --- a/app/react/V2/Components/Forms/MultiselectList.tsx +++ b/app/react/V2/Components/Forms/MultiselectList.tsx @@ -47,6 +47,7 @@ const MultiselectList = ({ singleSelect = false, allowSelelectAll = false, }: MultiselectListProps) => { + console.log(value); const [selectedItems, setSelectedItems] = useState(value || []); const [showAll, setShowAll] = useState(true); const [searchTerm, setSearchTerm] = useState(''); diff --git a/app/react/V2/Components/Layouts/SettingsContent.tsx b/app/react/V2/Components/Layouts/SettingsContent.tsx index 960ba744f1..8407bc1ff1 100644 --- a/app/react/V2/Components/Layouts/SettingsContent.tsx +++ b/app/react/V2/Components/Layouts/SettingsContent.tsx @@ -1,9 +1,8 @@ /* eslint-disable react/no-multi-comp */ -import { Link } from 'react-router-dom'; import React, { PropsWithChildren } from 'react'; import { Breadcrumb } from 'flowbite-react'; import { ChevronLeftIcon } from '@heroicons/react/20/solid'; -import { Translate } from 'app/I18N'; +import { I18NLink, Translate } from 'app/I18N'; interface SettingsContentProps extends PropsWithChildren { className?: string; @@ -31,12 +30,12 @@ const SettingsContent = ({ children, className }: SettingsContentProps) => ( const SettingsHeader = ({ contextId, title, children, path, className }: SettingsHeaderProps) => (
- + Navigate back - + {Array.from(path?.entries() || []).map(([key, value]) => ( diff --git a/app/react/V2/Components/Layouts/specs/SettingsContent.cy.tsx b/app/react/V2/Components/Layouts/specs/SettingsContent.cy.tsx index e8b5a9c7d2..4bb80c381c 100644 --- a/app/react/V2/Components/Layouts/specs/SettingsContent.cy.tsx +++ b/app/react/V2/Components/Layouts/specs/SettingsContent.cy.tsx @@ -47,7 +47,7 @@ describe('ConfirmationModal', () => { cy.get('[data-testid="settings-content-header"]') .invoke('text') .should('contain', 'Root PathMiddle PathLeafCurrent page'); - cy.get('a[href="/settings"]').should('not.be.visible'); + cy.get('a[href="/en/settings"]').should('not.be.visible'); cy.contains('a', 'Root Path').invoke('attr', 'href').should('include', '#top'); cy.contains('a', 'Middle Path').invoke('attr', 'href').should('include', '#bottom'); cy.contains('a', 'Leaf').invoke('attr', 'href').should('include', '#footer'); @@ -58,6 +58,6 @@ describe('ConfirmationModal', () => { it('should have an arrow to return to settings menu for mobile', () => { cy.viewport(450, 650); render(); - cy.get('a[href="/settings"]').should('be.visible'); + cy.get('a[href="/en/settings"]').should('be.visible'); }); }); diff --git a/app/react/V2/Components/UI/ConfirmationModal.tsx b/app/react/V2/Components/UI/ConfirmationModal.tsx index 2386537bf3..778335f3a3 100644 --- a/app/react/V2/Components/UI/ConfirmationModal.tsx +++ b/app/react/V2/Components/UI/ConfirmationModal.tsx @@ -8,12 +8,13 @@ type confirmationModalType = { size?: modalSizeType; header?: string | React.ReactNode; body?: string | React.ReactNode; - onAcceptClick?: () => void; + onAcceptClick?: (value: string) => void; onCancelClick?: () => void; acceptButton?: string | React.ReactNode; cancelButton?: string | React.ReactNode; warningText?: string | React.ReactNode; confirmWord?: string; + usePassword?: boolean; dangerStyle?: boolean; }; @@ -26,10 +27,12 @@ const ConfirmationModal = ({ cancelButton, warningText, confirmWord, + usePassword, size = 'md', dangerStyle = false, }: confirmationModalType) => { - const [confirmed, setConfirmed] = useState(confirmWord === undefined); + const [inputValue, setInputValue] = useState(''); + const [confirmed, setConfirmed] = useState(!(confirmWord || usePassword)); const renderChild = (child: string | React.ReactNode) => isString(child) ? {child} : child; @@ -71,6 +74,26 @@ const ConfirmationModal = ({ />
)} + + {usePassword && ( +
+ + + + { + setInputValue(e.currentTarget.value); + setConfirmed(e.currentTarget.value.length > 0); + }} + /> +
+ )}

@@ -180,6 +221,24 @@ const Account = () => { {userAccount.using2fa ? null : ( setIsSidepanelOpen(false)} /> )} + + {confirmationModal && ( + setConfirmationModal(false)} + onAcceptClick={value => { + if (formSubmit.current) { + passwordConfirmation.current = value; + formSubmit.current.disabled = false; + formSubmit.current.click(); + formSubmit.current.disabled = true; + setConfirmationModal(false); + } + }} + /> + )}
); }; diff --git a/app/react/V2/Routes/Settings/IX/IXSuggestions.tsx b/app/react/V2/Routes/Settings/IX/IXSuggestions.tsx index 30f7c37abd..1cc8284355 100644 --- a/app/react/V2/Routes/Settings/IX/IXSuggestions.tsx +++ b/app/react/V2/Routes/Settings/IX/IXSuggestions.tsx @@ -60,16 +60,24 @@ const ixmessages = { }; const IXSuggestions = () => { - const { suggestions, extractor, templates, aggregation, currentStatus, totalPages } = - useLoaderData() as { - totalPages: number; - suggestions: TableSuggestion[]; - extractor: IXExtractorInfo; - templates: ClientTemplateSchema[]; - aggregation: any; - currentStatus: ixStatus; - _id: string; - }; + const { + suggestions, + extractor, + templates, + aggregation, + currentStatus, + totalPages, + activeFilters, + } = useLoaderData() as { + totalPages: number; + suggestions: TableSuggestion[]; + extractor: IXExtractorInfo; + templates: ClientTemplateSchema[]; + aggregation: any; + currentStatus: ixStatus; + _id: string; + activeFilters: number; + }; const [currentSuggestions, setCurrentSuggestions] = useState(suggestions); const [property, setProperty] = useState(); @@ -84,7 +92,7 @@ const IXSuggestions = () => { }, [templates, extractor]); useMemo(() => { - if (property?.type === 'multiselect') { + if (property?.type === 'multiselect' || property?.type === 'relationship') { const flatenedSuggestions = suggestions.map(suggestion => generateChildrenRows(suggestion as MultiValueSuggestion) ); @@ -248,6 +256,7 @@ const IXSuggestions = () => { onFiltersButtonClicked={() => { setSidepanel('filters'); }} + activeFilters={activeFilters} /> } enableSelection @@ -259,7 +268,7 @@ const IXSuggestions = () => {
@@ -366,10 +375,14 @@ const IXSuggestionsLoader = if (!extractorId) throw new Error('extractorId is required'); const searchParams = new URLSearchParams(request.url.split('?')[1]); const filter: any = { extractorId }; + let activeFilters = 0; if (searchParams.has('filter')) { filter.customFilter = JSON.parse(searchParams.get('filter')!); + activeFilters = [ + ...Object.values(filter.customFilter.labeled), + ...Object.values(filter.customFilter.nonLabeled), + ].filter(Boolean).length; } - const sortingOption = searchParams.has('sort') ? searchParams.get('sort') : undefined; const suggestionsList: { suggestions: EntitySuggestionType[]; totalPages: number } = @@ -396,6 +409,7 @@ const IXSuggestionsLoader = templates, aggregation, currentStatus: currentStatus.status, + activeFilters, }; }; diff --git a/app/react/V2/Routes/Settings/IX/components/ExtractorModal.tsx b/app/react/V2/Routes/Settings/IX/components/ExtractorModal.tsx index e2c0bb24d9..2549f6cd2f 100644 --- a/app/react/V2/Routes/Settings/IX/components/ExtractorModal.tsx +++ b/app/react/V2/Routes/Settings/IX/components/ExtractorModal.tsx @@ -11,7 +11,10 @@ import { InputField } from 'app/V2/Components/Forms/InputField'; import { RadioSelect } from 'app/V2/Components/Forms'; import { propertyIcons } from './Icons'; -const SUPPORTED_PROPERTIES = ['text', 'numeric', 'date', 'select', 'multiselect']; +const SUPPORTED_PROPERTIES = ['text', 'numeric', 'date', 'select', 'multiselect', 'relationship']; +type SupportedProperty = Omit & { + type: 'text' | 'numeric' | 'date' | 'select' | 'multiselect' | 'relationship'; +}; interface ExtractorModalProps { setShowModal: React.Dispatch>; @@ -21,30 +24,11 @@ interface ExtractorModalProps { extractor?: IXExtractorInfo; } -const getPropertyLabel = (property: ClientPropertySchema, templateId: string) => { - let icon: React.ReactNode; - let propertyTypeTranslationKey = 'property text'; +const getPropertyLabel = (property: SupportedProperty, templateId: string) => { + const { type } = property; - switch (property.type) { - case 'numeric': - icon = propertyIcons.numeric; - propertyTypeTranslationKey = 'property numeric'; - break; - case 'date': - icon = propertyIcons.date; - propertyTypeTranslationKey = 'property date'; - break; - case 'select': - icon = propertyIcons.select; - propertyTypeTranslationKey = 'property select'; - break; - case 'multiselect': - icon = propertyIcons.multiselect; - propertyTypeTranslationKey = 'property multiselect'; - break; - default: - icon = propertyIcons.text; - } + const icon = propertyIcons[type]; + const propertyTypeTranslationKey = `property ${type}`; return (
@@ -77,7 +61,7 @@ const formatOptions = (values: string[], templates: ClientTemplateSchema[]) => { SUPPORTED_PROPERTIES.includes(prop.type) ) .map(prop => ({ - label: getPropertyLabel(prop, template._id), + label: getPropertyLabel(prop as SupportedProperty, template._id), value: `${template._id?.toString()}-${prop.name}`, searchLabel: prop.label, })), @@ -114,7 +98,7 @@ const getPropertyForValue = (value: string, templates: ClientTemplateSchema[]) = const matchedProperty = matchedTemplate?.properties.find( property => property.name === propertyName - ); + ) as SupportedProperty; if (matchedProperty) { return getPropertyLabel(matchedProperty, matchedTemplate!._id.toString()); diff --git a/app/react/V2/Routes/Settings/IX/components/Icons.tsx b/app/react/V2/Routes/Settings/IX/components/Icons.tsx index c02e8dc7d3..daab3aa486 100644 --- a/app/react/V2/Routes/Settings/IX/components/Icons.tsx +++ b/app/react/V2/Routes/Settings/IX/components/Icons.tsx @@ -5,6 +5,7 @@ import { NumericPropertyIcon, SelectPropertyIcon, TextPropertyIcon, + RelationshipPropertyIcon, } from 'V2/Components/CustomIcons'; const propertyIcons = { @@ -14,6 +15,7 @@ const propertyIcons = { markdown: , select: , multiselect: , + relationship: , }; export { propertyIcons }; diff --git a/app/react/V2/Routes/Settings/IX/components/PDFSidepanel.tsx b/app/react/V2/Routes/Settings/IX/components/PDFSidepanel.tsx index cf647bd994..40637ddc42 100644 --- a/app/react/V2/Routes/Settings/IX/components/PDFSidepanel.tsx +++ b/app/react/V2/Routes/Settings/IX/components/PDFSidepanel.tsx @@ -62,7 +62,7 @@ const getFormValue = ( value = dateString; } - if (type === 'select' || type === 'multiselect') { + if (type === 'select' || type === 'multiselect' || type === 'relationship') { value = entityMetadata?.map((metadata: MetadataObjectSchema) => metadata.value); } } @@ -225,10 +225,8 @@ const PDFSidepanel = ({ }, [pdf, setValue, showSidepanel, suggestion]); useEffect(() => { - console.log('pdfContainerRef', pdfContainerRef); if (pdfContainerRef.current) { const { height } = pdfContainerRef.current.getBoundingClientRect(); - console.log('height', height); setPdfContainerHeight(height); } }, [labelInputIsOpen, pdfContainerRef.current]); @@ -313,7 +311,7 @@ const PDFSidepanel = ({ } const inputType = type === 'numeric' ? 'number' : type; return ( -
+
{ @@ -369,7 +367,7 @@ const PDFSidepanel = ({ items?: Option[]; } - const renderSelect = (type: 'select' | 'multiselect') => { + const renderSelect = (type: 'select' | 'multiselect' | 'relationship') => { const options: Option[] = []; thesaurus?.values.forEach((value: any) => { options.push({ @@ -380,7 +378,7 @@ const PDFSidepanel = ({ }); return ( -
+
{ setValue('field', values, { shouldDirty: true }); @@ -402,6 +400,7 @@ const PDFSidepanel = ({ return renderInputText(property?.type); case 'select': case 'multiselect': + case 'relationship': return renderSelect(property?.type); default: return ''; @@ -448,7 +447,7 @@ const PDFSidepanel = ({ {' '}
@@ -462,7 +461,7 @@ const PDFSidepanel = ({ {labelInputIsOpen ? : }
- {labelInputIsOpen && renderLabel()} + {renderLabel()}
diff --git a/app/react/V2/Routes/Settings/IX/components/helpers.ts b/app/react/V2/Routes/Settings/IX/components/helpers.ts index 43bf59487b..7b4e10ad95 100644 --- a/app/react/V2/Routes/Settings/IX/components/helpers.ts +++ b/app/react/V2/Routes/Settings/IX/components/helpers.ts @@ -105,7 +105,7 @@ const updateSuggestionsByEntity = ( if (updatedEntity.metadata[propertyToUpdate]?.length) { const newValue = ( - property?.type === 'multiselect' + property?.type === 'multiselect' || property?.type === 'relationship' ? updatedEntity.metadata[propertyToUpdate]?.map(v => v.value) : updatedEntity.metadata[propertyToUpdate]![0].value ) as SuggestionValue; @@ -119,7 +119,7 @@ const updateSuggestionsByEntity = ( suggestionToUpdate.state.match = suggestionToUpdate.suggestedValue === ''; } - if (property?.type === 'multiselect') { + if (property?.type === 'multiselect' || property?.type === 'relationship') { suggestionToUpdate = generateChildrenRows(suggestionToUpdate as MultiValueSuggestion); } diff --git a/app/react/V2/Routes/Settings/Thesauri/ThesauriList.tsx b/app/react/V2/Routes/Settings/Thesauri/ThesauriList.tsx index 23e7275df6..42fa0492b3 100644 --- a/app/react/V2/Routes/Settings/Thesauri/ThesauriList.tsx +++ b/app/react/V2/Routes/Settings/Thesauri/ThesauriList.tsx @@ -56,14 +56,14 @@ const ThesauriList = () => { }, [thesauri, templates]); const navigateToEditThesaurus = (thesaurus: Row) => { - navigate(`/settings/thesauri/edit/${thesaurus.original._id}`); + navigate(`./edit/${thesaurus.original._id}`); }; const deleteSelectedThesauri = async () => { try { - const requests = selectedThesauri.map(sThesauri => { - return ThesauriAPI.delete({ _id: sThesauri.original._id }); - }); + const requests = selectedThesauri.map(sThesauri => + ThesauriAPI.delete({ _id: sThesauri.original._id }) + ); await Promise.all(requests); setNotifications({ type: 'success', diff --git a/app/react/V2/Routes/Settings/Thesauri/ThesaurusForm.tsx b/app/react/V2/Routes/Settings/Thesauri/ThesaurusForm.tsx index 21dea30079..df3653dd30 100644 --- a/app/react/V2/Routes/Settings/Thesauri/ThesaurusForm.tsx +++ b/app/react/V2/Routes/Settings/Thesauri/ThesaurusForm.tsx @@ -206,7 +206,7 @@ const ThesaurusForm = () => { Thesauri added. ), }); - navigate(`/settings/thesauri/edit/${savedThesaurus._id}`); + navigate(`../edit/${savedThesaurus._id}`); } catch (e) { setNotifications({ type: 'error', @@ -286,7 +286,7 @@ const ThesaurusForm = () => { type: 'success', text: Thesauri updated., }); - navigate(`/settings/thesauri/edit/${savedThesaurus._id}`); + navigate(`../edit/${savedThesaurus._id}`); }} onFailure={() => { setNotifications({ diff --git a/app/react/V2/Routes/Settings/Users/Users.tsx b/app/react/V2/Routes/Settings/Users/Users.tsx index 7a9b7aea30..6362210af9 100644 --- a/app/react/V2/Routes/Settings/Users/Users.tsx +++ b/app/react/V2/Routes/Settings/Users/Users.tsx @@ -1,5 +1,5 @@ /* eslint-disable max-lines */ -import React, { useState } from 'react'; +import React, { useRef, useState } from 'react'; import { IncomingHttpHeaders } from 'http'; import { ActionFunction, LoaderFunction, useFetcher, useLoaderData } from 'react-router-dom'; import { Row } from '@tanstack/react-table'; @@ -37,10 +37,10 @@ const Users = () => { header: 'Delete', body: 'Do you want to delete?', }); - const [bulkActionIntent, setBulkActionIntent] = useState('delete-users'); + const password = useRef(); + const bulkActionIntent = useRef(); const fetcher = useFetcher(); - useHandleNotifications(); const usersTableColumns = getUsersColumns((user: ClientUserSchema) => { @@ -55,7 +55,7 @@ const Users = () => { const handleBulkAction = () => { const formData = new FormData(); - formData.set('intent', bulkActionIntent); + formData.set('intent', bulkActionIntent.current || ''); if (activeTab === 'Users') { formData.set('data', JSON.stringify(selectedUsers.map(user => user.original))); @@ -63,6 +63,8 @@ const Users = () => { formData.set('data', JSON.stringify(selectedGroups.map(group => group.original))); } + formData.set('confirmation', password.current || ''); + fetcher.submit(formData, { method: 'post' }); }; @@ -115,7 +117,7 @@ const Users = () => { header: 'Reset passwords', body: 'Do you want reset the password for the following users?', }); - setBulkActionIntent('bulk-reset-password'); + bulkActionIntent.current = 'bulk-reset-password'; setShowConfirmationModal(true); }} > @@ -129,7 +131,7 @@ const Users = () => { header: 'Reset 2FA', body: 'Do you want disable 2FA for the following users?', }); - setBulkActionIntent('bulk-reset-2fa'); + bulkActionIntent.current = 'bulk-reset-2fa'; setShowConfirmationModal(true); }} > @@ -146,7 +148,8 @@ const Users = () => { header: 'Delete', body: 'Do you want to delete the following items?', }); - setBulkActionIntent(activeTab === 'Users' ? 'delete-users' : 'delete-groups'); + bulkActionIntent.current = + activeTab === 'Users' ? 'delete-users' : 'delete-groups'; setShowConfirmationModal(true); }} > @@ -197,7 +200,13 @@ const Users = () => { header={confirmationModalProps.header} warningText={confirmationModalProps.body} body={} - onAcceptClick={() => { + usePassword={ + (selectedUsers.length > 0 && + ['bulk-reset-2fa', 'delete-users'].includes(bulkActionIntent.current || '')) || + false + } + onAcceptClick={value => { + password.current = value; handleBulkAction(); setShowConfirmationModal(false); setSelectedGroups([]); @@ -226,27 +235,28 @@ const userAction = const formIntent = formData.get('intent') as FormIntent; const formValues = JSON.parse(formData.get('data') as string); + const confirmation = formData.get('confirmation') as string; switch (formIntent) { case 'new-user': - return usersAPI.newUser(formValues); + return usersAPI.newUser(formValues, confirmation); case 'edit-user': - return usersAPI.updateUser(formValues); + return usersAPI.updateUser(formValues, confirmation); case 'delete-users': - return usersAPI.deleteUser(formValues); + return usersAPI.deleteUser(formValues, confirmation); case 'new-group': case 'edit-group': return usersAPI.saveGroup(formValues); case 'delete-groups': return usersAPI.deleteGroup(formValues); case 'unlock-user': - return usersAPI.unlockAccount(formValues); + return usersAPI.unlockAccount(formValues, confirmation); case 'reset-password': case 'bulk-reset-password': return usersAPI.resetPassword(formValues); case 'reset-2fa': case 'bulk-reset-2fa': - return usersAPI.reset2FA(formValues); + return usersAPI.reset2FA(formValues, confirmation); default: return null; } diff --git a/app/react/V2/Routes/Settings/Users/components/GroupFormSidepanel.tsx b/app/react/V2/Routes/Settings/Users/components/GroupFormSidepanel.tsx index 38fe42c757..c3c3ab9ddb 100644 --- a/app/react/V2/Routes/Settings/Users/components/GroupFormSidepanel.tsx +++ b/app/react/V2/Routes/Settings/Users/components/GroupFormSidepanel.tsx @@ -55,21 +55,19 @@ const GroupFormSidepanel = ({ }: GroupFormSidepanelProps) => { const fetcher = useFetcher(); - const defaultValues = { name: '', members: [] } as UserGroupSchema; + const defaultValues = selectedGroup || ({ name: '', members: [] } as UserGroupSchema); const { register, handleSubmit, formState: { errors }, setValue, - reset, } = useForm({ defaultValues, - values: selectedGroup, + values: defaultValues, }); const closeSidepanel = () => { - reset(defaultValues); setSelected(undefined); setShowSidepanel(false); }; @@ -89,8 +87,7 @@ const GroupFormSidepanel = ({ formData.set('data', JSON.stringify(formattedData)); fetcher.submit(formData, { method: 'post' }); - setShowSidepanel(false); - reset(defaultValues); + closeSidepanel(); }; return ( @@ -121,7 +118,7 @@ const GroupFormSidepanel = ({
-
+
diff --git a/app/react/V2/Routes/Settings/Users/components/UserFormSidepanel.tsx b/app/react/V2/Routes/Settings/Users/components/UserFormSidepanel.tsx index d387793312..be5197445d 100644 --- a/app/react/V2/Routes/Settings/Users/components/UserFormSidepanel.tsx +++ b/app/react/V2/Routes/Settings/Users/components/UserFormSidepanel.tsx @@ -1,16 +1,21 @@ +/* eslint-disable max-statements */ /* eslint-disable max-lines */ /* eslint-disable react/jsx-props-no-spreading */ -import React, { useState } from 'react'; +import React, { useEffect, useRef, useState } from 'react'; import { useForm } from 'react-hook-form'; import { useFetcher } from 'react-router-dom'; +import { FetchResponseError } from 'shared/JSONRequest'; import { t, Translate } from 'app/I18N'; import { ClientUserGroupSchema, ClientUserSchema } from 'app/apiResponseTypes'; import { InputField, Select, MultiSelect } from 'V2/Components/Forms'; -import { Button, Card, Sidepanel } from 'V2/Components/UI'; +import { Button, Card, ConfirmationModal, Sidepanel } from 'V2/Components/UI'; +import { validEmailFormat } from 'V2/shared/formatHelpers'; import { UserRole } from 'shared/types/userSchema'; import { QuestionMarkCircleIcon } from '@heroicons/react/20/solid'; import { PermissionsListModal } from './PermissionsListModal'; +type SubmitType = 'formSubmit' | 'reset-2fa' | 'unlock-user' | 'reset-password' | undefined; + interface UserFormSidepanelProps { showSidepanel: boolean; setShowSidepanel: React.Dispatch>; @@ -65,9 +70,10 @@ const getFieldError = (field: 'username' | 'password' | 'email', type?: string) if (field === 'email') { switch (type) { + case 'format': case 'required': - return 'Email is required'; - case 'validate': + return 'A valid email is required'; + case 'isUnique': return 'Duplicated email'; default: break; @@ -91,32 +97,49 @@ const UserFormSidepanel = ({ }: UserFormSidepanelProps) => { const fetcher = useFetcher(); const [showModal, setShowModal] = useState(false); + const [showConfirmationModal, setShowConfirmationModal] = useState(false); + const password = useRef(); + const actionType = useRef(); + const formSubmitRef = useRef(null); - const defaultValues = { - username: '', - email: '', - password: '', - role: 'collaborator', - groups: [], - } as ClientUserSchema; + const defaultValues = + selectedUser || + ({ + username: '', + email: '', + password: '', + role: 'collaborator', + groups: [], + } as ClientUserSchema); const { register, handleSubmit, - reset, + trigger, formState: { errors }, setValue, } = useForm({ defaultValues, - values: selectedUser, + values: defaultValues, }); const closeSidepanel = () => { - reset(defaultValues); setSelected(undefined); setShowSidepanel(false); }; + useEffect(() => { + const { data: response, state } = fetcher; + + if ( + state === 'loading' && + response && + !(response instanceof FetchResponseError || response.status === 403) + ) { + closeSidepanel(); + } + }, [fetcher]); + const formSubmit = async (data: ClientUserSchema) => { const formData = new FormData(); if (data._id) { @@ -126,19 +149,16 @@ const UserFormSidepanel = ({ } formData.set('data', JSON.stringify(data)); + formData.set('confirmation', password.current || ''); fetcher.submit(formData, { method: 'post' }); - setShowSidepanel(false); - reset(defaultValues); }; - const onClickSubmit = (intent: string) => { + const onClickSubmit = () => { const formData = new FormData(); - formData.set('intent', intent); + formData.set('intent', actionType.current || ''); formData.set('data', JSON.stringify(selectedUser)); + formData.set('confirmation', password.current || ''); fetcher.submit(formData, { method: 'post' }); - - setShowSidepanel(false); - reset(defaultValues); }; return ( @@ -198,7 +218,10 @@ const UserFormSidepanel = ({ errorMessage={getFieldError('email', errors.email?.type)} {...register('email', { required: true, - validate: email => isUnique(email, selectedUser, users), + validate: { + isUnique: email => isUnique(email, selectedUser, users), + format: email => validEmailFormat(email), + }, maxLength: 256, })} /> @@ -226,7 +249,10 @@ const UserFormSidepanel = ({ @@ -234,7 +260,10 @@ const UserFormSidepanel = ({ @@ -246,7 +275,10 @@ const UserFormSidepanel = ({ type="button" styling="light" color="error" - onClick={() => onClickSubmit('unlock-user')} + onClick={() => { + actionType.current = 'unlock-user'; + setShowConfirmationModal(true); + }} > Unlock account @@ -254,7 +286,7 @@ const UserFormSidepanel = ({
-
+
@@ -282,14 +314,46 @@ const UserFormSidepanel = ({ > Cancel -
+