Skip to content

Commit

Permalink
v1.172.0
Browse files Browse the repository at this point in the history
  • Loading branch information
varovaro committed Jun 24, 2024
2 parents 40a5cc1 + 9addd68 commit 789953f
Show file tree
Hide file tree
Showing 89 changed files with 3,920 additions and 951 deletions.
8 changes: 8 additions & 0 deletions .github/workflows/ci_check_translations.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,14 @@ jobs:
- name: install dependencies
run: yarn install
- run: yarn blank-state
id: blank-state-attempt1
continue-on-error: true
env:
DBHOST: localhost:27017
ELASTICSEARCH_URL: http://localhost:${{ job.services.elasticsearch.ports[9200] }}
- run: yarn blank-state --force
id: blank-state-attempt2
if: steps.blank-state-attempt1.outcome == 'failure'
env:
DBHOST: localhost:27017
ELASTICSEARCH_URL: http://localhost:${{ job.services.elasticsearch.ports[9200] }}
Expand Down
11 changes: 11 additions & 0 deletions .github/workflows/ci_e2e_cypress.yml
Original file line number Diff line number Diff line change
Expand Up @@ -82,12 +82,23 @@ jobs:
if: ${{ failure() }}
run: cat dummy_extractor_services/logs.log
- run: yarn blank-state
id: blank-state-attempt1
continue-on-error: true
env:
DBHOST: localhost:27017
ELASTICSEARCH_URL: http://localhost:${{ job.services.elasticsearch.ports[9200] }}
DATABASE_NAME: uwazi_e2e
INDEX_NAME: uwazi_e2e
TRANSPILED: true
- run: yarn blank-state --force
id: blank-state-attempt2
if: steps.blank-state-attempt1.outcome == 'failure'
env:
DBHOST: localhost:27017
ELASTICSEARCH_URL: http://localhost:${{ job.services.elasticsearch.ports[9200] }}
DATABASE_NAME: uwazi_e2e
INDEX_NAME: uwazi_e2e
TRANSPILED: true
- run: yarn ix-config
env:
DBHOST: localhost:27017
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ Before anything else you will need to install the application dependencies:
- **ElasticSearch 7.17.6** https://www.elastic.co/downloads/past-releases/elasticsearch-7-17-6 Please note that ElasticSearch requires Java. Follow the instructions to install the package manually, you also probably need to disable ml module in the ElasticSearch config file:
`xpack.ml.enabled: false`
- **ICU Analysis Plugin (recommended)** [installation](https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-icu.html#analysis-icu) Adds support for number sorting in texts and solves other language sorting nuances. This option is activated by setting the env var USE_ELASTIC_ICU=true before running the server (defaults to false/unset).
- **MongoDB 5.0.27** https://docs.mongodb.com/v4.2/installation/ . If you have a previous version installed, please follow the instructions on how to [upgrade here](https://docs.mongodb.com/manual/release-notes/4.2-upgrade-standalone/). The new mongosh dependency needs to be added [installation](https://www.mongodb.com/docs/mongodb-shell/).
- **MongoDB 5.0.27** https://www.mongodb.com/docs/v5.0/installation/ . If you have a previous version installed, please follow the instructions on how to [upgrade here](https://www.mongodb.com/docs/manual/release-notes/5.0-upgrade-standalone/). The new mongosh dependency needs to be added [installation](https://www.mongodb.com/docs/mongodb-shell/).
- **Yarn** https://yarnpkg.com/en/docs/install.
- **pdftotext (Poppler)** tested to work on version 0.86 but it's recommended to use the latest available for your platform https://poppler.freedesktop.org/. Make sure to **install libjpeg-dev** if you build from source.

Expand Down
4 changes: 1 addition & 3 deletions app/api/auth/encryptPassword.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,4 @@ const saltRounds = 10;
const encryptPassword = async plainPassword => bcrypt.hash(plainPassword, saltRounds);
const comparePasswords = async (plain, hashed) => bcrypt.compare(plain, hashed);

export default encryptPassword;

export { comparePasswords };
export { comparePasswords, encryptPassword };
4 changes: 2 additions & 2 deletions app/api/auth/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,5 +3,5 @@ import captchaAuthorization from './captchaMiddleware';
import { CaptchaModel } from './CaptchaModel';

export { needsAuthorization, captchaAuthorization, CaptchaModel };
export { default as encryptPassword } from './encryptPassword';
export { comparePasswords } from './encryptPassword';
export { comparePasswords, encryptPassword } from './encryptPassword';
export { validatePasswordMiddleWare } from './validatePasswordMiddleWare';
122 changes: 122 additions & 0 deletions app/api/auth/specs/validatePasswordMiddleWare.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
import { Request, NextFunction, Response } from 'express';
import { testingEnvironment } from 'api/utils/testingEnvironment';
import { getFixturesFactory } from 'api/utils/fixturesFactory';
import { UserSchema } from 'shared/types/userType';
import { UserRole } from 'shared/types/userSchema';
import { encryptPassword } from '../encryptPassword';
import { validatePasswordMiddleWare } from '../validatePasswordMiddleWare';

const fixturesFactory = getFixturesFactory();

describe('validatePasswordMiddleWare', () => {
const mockResponse = (): Response => {
const res: Partial<Response> = {};
res.status = jest.fn().mockReturnValue(res);
res.json = jest.fn().mockReturnValue(res);
return res as Response;
};

const res = mockResponse();
const users: UserSchema[] = [];
const next: NextFunction = jest.fn();

const createRequest = (request: Partial<Request>): Request =>
<Request>{
...request,
user: request.user,
body: request.body,
headers: request.headers,
};

const gernerateFixtures = async () => {
users.push(
...[
{
...fixturesFactory.user(
'admin',
UserRole.ADMIN,
'[email protected]',
await encryptPassword('admin1234')
),
},
{
...fixturesFactory.user(
'editor',
UserRole.EDITOR,
'[email protected]',
await encryptPassword('editor1234')
),
},
]
);

return {
users,
};
};

beforeAll(async () => {
const fixtures = await gernerateFixtures();
await testingEnvironment.setUp(fixtures);
});

afterEach(() => {
jest.clearAllMocks();
});

afterAll(async () => {
await testingEnvironment.tearDown();
});

it('should reject when the password is incorrect', async () => {
const request = createRequest({
user: { _id: users[0]._id, username: users[0].username, role: users[0].role },
body: { username: 'a_new_user', role: 'collaborator', email: '[email protected]' },
headers: { authorization: 'Basic wrongPass' },
});

await validatePasswordMiddleWare(request, res, next);

expect(next).not.toHaveBeenCalled();
expect(res.status).toHaveBeenCalledWith(403);
expect(res.json).toHaveBeenCalledWith({ message: 'Forbidden', error: 'Password error' });
});

it('should reject when the password is empty', async () => {
const emptyPasswordRequest = createRequest({
user: { _id: users[0]._id, username: users[0].username, role: users[0].role },
body: { username: 'a_new_user', role: 'collaborator', email: '[email protected]' },
headers: { authorization: 'Basic ' },
});

const noAuthHeaderRequest = createRequest({
user: { _id: users[0]._id, username: users[0].username, role: users[0].role },
body: { _id: users[0]._id, username: users[0].username, role: users[0].role },
headers: { cookie: 'some cookie' },
});

await validatePasswordMiddleWare(emptyPasswordRequest, res, next);

expect(next).not.toHaveBeenCalled();
expect(res.status).toHaveBeenCalledWith(403);
expect(res.json).toHaveBeenNthCalledWith(1, { message: 'Forbidden', error: 'Password error' });

await validatePasswordMiddleWare(noAuthHeaderRequest, res, next);

expect(next).not.toHaveBeenCalled();
expect(res.status).toHaveBeenCalledWith(403);
expect(res.json).toHaveBeenNthCalledWith(2, { message: 'Forbidden', error: 'Password error' });
});

it('should succeed when the passwords match', async () => {
const request = createRequest({
user: { _id: users[1]._id, username: users[1].username, role: users[1].role },
body: { _id: users[1]._id, username: users[1].username, role: users[1].role },
headers: { authorization: 'Basic editor1234' },
});

await validatePasswordMiddleWare(request, res, next);

expect(next).toHaveBeenCalled();
});
});
28 changes: 28 additions & 0 deletions app/api/auth/validatePasswordMiddleWare.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
import { Request, Response, NextFunction } from 'express';
import { User } from 'api/users/usersModel';
import usersModel from '../users/users';
import { comparePasswords } from './encryptPassword';

const validatePassword = async (submittedPassword: string, requestUser: User) => {
const user = await usersModel.getById(requestUser._id, '+password');
const currentPassword = user.password;
return comparePasswords(submittedPassword, currentPassword);
};

const validatePasswordMiddleWare = async (req: Request, res: Response, next: NextFunction) => {
const { user, headers } = req;
const submittedPassword = headers?.authorization?.split('Basic ')[1];

if (submittedPassword) {
const validPassword = await validatePassword(submittedPassword, user);

if (validPassword) {
return next();
}
}

res.status(403);
return res.json({ error: 'Password error', message: 'Forbidden' });
};

export { validatePasswordMiddleWare };
2 changes: 2 additions & 0 deletions app/api/auth2fa/routes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import needsAuthorization from 'api/auth/authMiddleware';
import * as usersUtils from 'api/auth2fa/usersUtils';
import { validation } from 'api/utils';
import { ObjectIdAsString } from 'api/utils/ajvSchemas';
import { validatePasswordMiddleWare } from 'api/auth';

export default (app: Application) => {
app.post(
Expand Down Expand Up @@ -50,6 +51,7 @@ export default (app: Application) => {
app.post(
'/api/auth2fa-reset',
needsAuthorization(['admin']),
validatePasswordMiddleWare,
validation.validateRequest({
type: 'object',
properties: {
Expand Down
35 changes: 28 additions & 7 deletions app/api/services/informationextraction/InformationExtraction.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import {
FileWithAggregation,
getFilesForTraining,
getFilesForSuggestions,
propertyTypeIsWithoutExtractedMetadata,
propertyTypeIsSelectOrMultiSelect,
} from 'api/services/informationextraction/getFiles';
import { Suggestions } from 'api/suggestions/suggestions';
Expand Down Expand Up @@ -95,6 +96,16 @@ type MaterialsData =
| TextSelectionMaterialsData
| ValuesSelectionMaterialsData;

async function fetchCandidates(property: PropertySchema) {
const defaultLanguageKey = (await settings.getDefaultLanguage()).key;
const query: { template?: ObjectId; language: string } = {
language: defaultLanguageKey,
};
if (property.content !== '') query.template = new ObjectId(property.content);
const candidates = await entities.getUnrestricted(query, ['title', 'sharedId']);
return candidates;
}

class InformationExtraction {
static SERVICE_NAME = 'information_extraction';

Expand Down Expand Up @@ -142,9 +153,9 @@ class InformationExtraction {

let data: MaterialsData = { ..._data, language_iso };

const isSelect = propertyTypeIsSelectOrMultiSelect(propertyType);
const noExtractedData = propertyTypeIsWithoutExtractedMetadata(propertyType);

if (!isSelect && propertyLabeledData) {
if (!noExtractedData && propertyLabeledData) {
data = {
...data,
label_text: propertyValue || propertyLabeledData?.selection?.text,
Expand All @@ -155,7 +166,7 @@ class InformationExtraction {
};
}

if (isSelect) {
if (noExtractedData) {
if (!Array.isArray(propertyValue)) {
throw new Error('Property value should be an array');
}
Expand Down Expand Up @@ -184,7 +195,7 @@ class InformationExtraction {
);
const { propertyValue, propertyType } = file;

const missingData = propertyTypeIsSelectOrMultiSelect(propertyType)
const missingData = propertyTypeIsWithoutExtractedMetadata(propertyType)
? !propertyValue
: type === 'labeled_data' && !propertyLabeledData;

Expand Down Expand Up @@ -281,7 +292,10 @@ class InformationExtraction {
templates,
template => template.properties || []
);
const property = allProps.find(p => p.name === extractor.property);
const property =
extractor.property === 'title'
? { name: 'title' as 'title', type: 'title' as 'title' }
: allProps.find(p => p.name === extractor.property);

const suggestion = await formatSuggestion(
property,
Expand Down Expand Up @@ -380,15 +394,22 @@ class InformationExtraction {

const params: TaskParameters = {
id: extractorId.toString(),
multi_value: property.type === 'multiselect',
multi_value: property.type === 'multiselect' || property.type === 'relationship',
};

if (property.type === 'select' || property.type === 'multiselect') {
if (propertyTypeIsSelectOrMultiSelect(property.type)) {
const thesauri = await dictionatiesModel.getById(property.content);

params.options =
thesauri?.values?.map(value => ({ label: value.label, id: value.id as string })) || [];
}
if (property.type === 'relationship') {
const candidates = await fetchCandidates(property);
params.options = candidates.map(candidate => ({
label: candidate.title || '',
id: candidate.sharedId || '',
}));
}

await this.taskManager.startTask({
task: 'create_model',
Expand Down
18 changes: 14 additions & 4 deletions app/api/services/informationextraction/getFiles.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,16 @@ type FileEnforcedNotUndefined = {
};

const selectProperties: Set<string> = new Set([propertyTypes.select, propertyTypes.multiselect]);
const propertiesWithoutExtractedMetadata: Set<string> = new Set([
...Array.from(selectProperties),
propertyTypes.relationship,
]);

const propertyTypeIsSelectOrMultiSelect = (type: string) => selectProperties.has(type);

const propertyTypeIsWithoutExtractedMetadata = (type: string) =>
propertiesWithoutExtractedMetadata.has(type);

async function getFilesWithAggregations(files: (FileType & FileEnforcedNotUndefined)[]) {
const filesNames = files.filter(x => x.filename).map(x => x.filename);

Expand Down Expand Up @@ -98,7 +105,7 @@ async function fileQuery(
propertyType: string,
entitiesFromTrainingTemplatesIds: string[]
) {
const needsExtractedMetadata = !propertyTypeIsSelectOrMultiSelect(propertyType);
const needsExtractedMetadata = !propertyTypeIsWithoutExtractedMetadata(propertyType);
const query: {
type: string;
filename: { $exists: Boolean };
Expand All @@ -125,7 +132,7 @@ function entityForTrainingQuery(
const query: {
[key: string]: { $in?: ObjectIdSchema[]; $exists?: Boolean; $ne?: any[] };
} = { template: { $in: templates } };
if (propertyTypeIsSelectOrMultiSelect(propertyType)) {
if (propertyTypeIsWithoutExtractedMetadata(propertyType)) {
query[`metadata.${property}`] = { $exists: true, $ne: [] };
}
return query;
Expand Down Expand Up @@ -162,8 +169,8 @@ async function getFilesForTraining(templates: ObjectIdSchema[], property: string
return { ...file, propertyType };
}

if (propertyTypeIsSelectOrMultiSelect(propertyType)) {
const propertyValue = (entity.metadata[property] || []).map(({ value, label }) => ({
if (propertyTypeIsWithoutExtractedMetadata(propertyType)) {
const propertyValue = (entity.metadata?.[property] || []).map(({ value, label }) => ({
value: ensure<string>(value),
label: ensure<string>(label),
}));
Expand All @@ -174,6 +181,8 @@ async function getFilesForTraining(templates: ObjectIdSchema[], property: string
let stringValue: string;
if (propertyType === propertyTypes.date) {
stringValue = moment(<number>value * 1000).format('YYYY-MM-DD');
} else if (propertyType === propertyTypes.numeric) {
stringValue = value?.toString() || '';
} else {
stringValue = <string>value;
}
Expand Down Expand Up @@ -221,5 +230,6 @@ export {
getFilesForSuggestions,
getSegmentedFilesIds,
propertyTypeIsSelectOrMultiSelect,
propertyTypeIsWithoutExtractedMetadata,
};
export type { FileWithAggregation };
Loading

0 comments on commit 789953f

Please sign in to comment.