v1.174.0

app/api/api.js

@@ -42,4 +42,5 @@ export default (app, server) => {
   require('./preserve/routes').PreserveRoutes(app);
   require('./relationships.v2/routes/routes').default(app);
   require('./stats/routes').default(app);
+  require('./testing_errors/routes').default(app);
 };

app/api/config.ts

@@ -4,6 +4,7 @@ import { version } from '../../package.json';
 
 const {
   ROOT_PATH,
+  JSON_LOGS,
   UPLOADS_FOLDER,
   CUSTOM_UPLOADS_FOLDER,
   ACTIVITY_LOGS_FOLDER,
@@ -30,6 +31,8 @@ const onlyDBHOST = () => (DBHOST ? `mongodb://${DBHOST}/` : 'mongodb://127.0.0.1
 export const config = {
   VERSION: ENVIRONMENT ? version : `development-${version}`,
 
+  JSON_LOGS: JSON_LOGS || false,
+
   ENVIRONMENT: ENVIRONMENT || 'development',
 
   PORT: process.env.PORT || 3000,

app/api/entities/entities.js

@@ -11,6 +11,7 @@ import templates from 'api/templates/templates';
 import { generateNames } from 'api/templates/utils';
 import date from 'api/utils/date';
 import { unique } from 'api/utils/filters';
+import { objectIndex } from 'shared/data_utils/objectIndex';
 import { AccessLevels } from 'shared/types/permissionSchema';
 import { propertyTypes } from 'shared/propertyTypes';
 import ID from 'shared/uniqueID';
@@ -534,8 +535,17 @@ export default {
   },
 
   async saveMultiple(docs) {
-    await docs.reduce(async (prev, doc) => {
+    const templateIds = Array.from(new Set(docs.map(d => d.template)));
+    const indexedTemplates = objectIndex(
+      await templates.get({ _id: { $in: templateIds } }),
+      t => t._id.toString(),
+      t => t
+    );
+
+    await docs.reduce(async (prev, _doc) => {
       await prev;
+      const template = indexedTemplates[_doc.template];
+      const doc = this.sanitize(_doc, template);
       await validateEntity(doc);
     }, Promise.resolve());
 

app/api/entities/specs/entities.spec.js

@@ -563,6 +563,25 @@ describe('entities', () => {
       });
     });
 
+    it('should sanitize the entities', async () => {
+      const sanitizationSpy = jest.spyOn(entities, 'sanitize');
+      await entities.updateMetdataFromRelationships(['shared'], 'en');
+
+      expect(sanitizationSpy.mock.calls).toMatchObject([
+        [
+          {
+            sharedId: 'shared',
+            language: 'en',
+            title: 'Batman finishes',
+          },
+          {
+            name: 'template_test',
+          },
+        ],
+      ]);
+      sanitizationSpy.mockRestore();
+    });
+
     describe('unrestricted for collaborator', () => {
       it('should save the entity with unrestricted access', async () => {
         userFactory.mock({
@@ -1137,6 +1156,70 @@ describe('entities', () => {
         })
         .catch(done.fail);
     });
+
+    it('should sanitize the entities', async () => {
+      const sanitizationSpy = jest.spyOn(entities, 'sanitize');
+      const docsToSave = [
+        {
+          title: 'Batman begins',
+          template: templateId,
+          language: 'es',
+          metadata: {
+            multiselect: [{ value: 'country_one' }, { value: 'country_two' }],
+            friends: [{ value: 'id1' }, { value: 'id2' }],
+          },
+        },
+        {
+          title: 'Batman begins',
+          template: templateId,
+          language: 'en',
+          metadata: {
+            multiselect: [{ value: 'country_one' }, { value: 'country_two' }],
+            friends: [{ value: 'id1' }, { value: 'id2' }],
+          },
+        },
+        {
+          title: 'Batman Goes On',
+          template: entityGetTestTemplateId,
+          language: 'en',
+          metadata: {
+            some_property: [{ value: 'some value' }],
+          },
+        },
+      ];
+      await entities.saveMultiple(docsToSave);
+      expect(sanitizationSpy.mock.calls).toMatchObject([
+        [
+          {
+            title: 'Batman begins',
+            language: 'es',
+          },
+          {
+            name: 'template_test',
+          },
+        ],
+        [
+          {
+            title: 'Batman begins',
+            language: 'en',
+          },
+          {
+            name: 'template_test',
+          },
+        ],
+        [
+          {
+            title: 'Batman Goes On',
+            language: 'en',
+          },
+          {
+            name: 'entityGetTestTemplate',
+          },
+        ],
+      ]);
+
+      sanitizationSpy.mockRestore();
+    });
   });
 
   describe('updateMetadataProperties', () => {

app/api/log/errorLog.ts

@@ -1,6 +1,7 @@
 import winston from 'winston';
 import GrayLogTransport from './GrayLogTransport';
-import { formatter } from './infoFormat';
+import { formatter, jsonFormatter } from './infoFormat';
+import { config } from '../config';
 
 let DATABASE_NAME = 'localhost';
 let LOGS_DIR = './log';
@@ -24,19 +25,31 @@ const createConsoleTransport = () =>
     format: formatter(DATABASE_NAME),
   });
 
+const createJSONConsoleTransport = () =>
+  new winston.transports.Console({
+    handleExceptions: true,
+    level: 'error',
+    format: jsonFormatter(DATABASE_NAME),
+  });
+
 const createErrorLog = () => {
   DATABASE_NAME = process.env.DATABASE_NAME ? process.env.DATABASE_NAME : 'localhost';
   LOGS_DIR = process.env.LOGS_DIR ? process.env.LOGS_DIR : './log';
 
+  let transports = [createFileTransport(), createConsoleTransport()];
+  if (config.JSON_LOGS) {
+    transports = [createJSONConsoleTransport()];
+  }
+
   const logger: ExtendedLogger = winston.createLogger({
-    transports: [createFileTransport(), createConsoleTransport()],
+    transports,
   });
 
   logger.closeGraylog = (cb = () => {}) => {
     cb();
   };
 
-  if (process.env.USE_GRAYLOG) {
+  if (process.env.USE_GRAYLOG && !config.JSON_LOGS) {
     const graylogTransport = new GrayLogTransport({
       format: formatter(DATABASE_NAME),
       instance_name: DATABASE_NAME,

app/api/log/infoFormat.ts

@@ -24,11 +24,27 @@ const formatInfo = (info: any) => {
   }`;
 };
 
+const jsonFormatter = (DATABASE_NAME: String) =>
+  winston.format.combine(
+    winston.format.timestamp(),
+    winston.format(addTenant)({ instanceName: DATABASE_NAME }),
+    winston.format.printf(info =>
+      JSON.stringify({
+        application_name: 'Uwazi',
+        timestamp: info.timestamp,
+        environment: config.ENVIRONMENT,
+        tenant: info.tenant,
+        message: formatInfo(info),
+        tenantError: info.tenantError,
+      })
+    )
+  );
+
 const formatter = (DATABASE_NAME: String) =>
   winston.format.combine(
     winston.format.timestamp(),
     winston.format(addTenant)({ instanceName: DATABASE_NAME }),
     winston.format.printf(info => formatInfo(info))
   );
 
-export { formatter, addTenant, formatInfo };
+export { jsonFormatter, formatter, addTenant, formatInfo };

app/api/migrations/migrations/167-default_empty_metadata/index.ts

@@ -0,0 +1,115 @@
+/* eslint-disable no-await-in-loop */
+import { Db } from 'mongodb';
+import { Entity, Template } from './types';
+
+let entitiesToUpdate: Entity[] = [];
+
+const propDoesNotExist = (obj: Record<string, unknown>, prop: string) =>
+  !obj.hasOwnProperty(prop) || obj[prop] === null || obj[prop] === undefined;
+
+export default {
+  delta: 167,
+
+  name: 'default_empty_metadata',
+
+  description: 'Adds empty array as default metadata for all entities.',
+
+  batchSize: 1000,
+
+  reindex: false,
+
+  propertiesByTemplate: {} as Record<string, string[]>,
+
+  fullEmptyMetadataByTemplate: {} as Record<string, Record<string, []>>,
+
+  async readProperties(db: Db) {
+    const templates = await db.collection<Template>('templates').find().toArray();
+    this.propertiesByTemplate = {};
+    this.fullEmptyMetadataByTemplate = {};
+    templates.forEach(template => {
+      const properties = template.properties?.map(property => property.name) || [];
+      const idString = template._id?.toString() || '';
+      this.propertiesByTemplate[idString] = properties;
+      this.fullEmptyMetadataByTemplate[idString] = {};
+      properties.forEach(property => {
+        this.fullEmptyMetadataByTemplate[idString][property] = [];
+      });
+    });
+  },
+
+  async flushUpdates(db: Db) {
+    if (!entitiesToUpdate.length) return;
+    const entitiesCollection = db.collection<Entity>('entities');
+    const operations = entitiesToUpdate.map(entity => ({
+      updateOne: {
+        filter: { _id: entity._id },
+        update: { $set: entity },
+      },
+    }));
+    await entitiesCollection.bulkWrite(operations);
+    this.reindex = true;
+    entitiesToUpdate = [];
+  },
+
+  async performUpdates(db: Db) {
+    if (entitiesToUpdate.length >= this.batchSize) {
+      await this.flushUpdates(db);
+    }
+  },
+
+  repairMetadata(templateId: string) {
+    return { newMetadata: this.fullEmptyMetadataByTemplate[templateId], repaired: true };
+  },
+
+  repairProperties(templateId: string, entity: Entity) {
+    const properties = this.propertiesByTemplate[templateId];
+    const newMetadata = { ...(entity.metadata || {}) };
+    const missingProperties = properties.filter(prop => propDoesNotExist(newMetadata, prop));
+    if (missingProperties.length) {
+      missingProperties.forEach(prop => {
+        newMetadata[prop] = [];
+      });
+    }
+    return { newMetadata, repaired: missingProperties.length > 0 };
+  },
+
+  repairEntity(entity: Entity) {
+    const templateId = entity.template?.toString() || '';
+    let repaired = false;
+    let newMetadata: NonNullable<Entity['metadata']> = {};
+    if (propDoesNotExist(entity, 'metadata')) {
+      ({ newMetadata, repaired } = this.repairMetadata(templateId));
+    } else {
+      ({ newMetadata, repaired } = this.repairProperties(templateId, entity));
+    }
+    return { newEntity: { ...entity, metadata: newMetadata }, repaired };
+  },
+
+  async handleEntity(db: Db, entity: Entity | null) {
+    if (!entity) return;
+    const { newEntity, repaired } = this.repairEntity(entity);
+    if (repaired) {
+      entitiesToUpdate.push(newEntity);
+      await this.performUpdates(db);
+    }
+  },
+
+  async handleEntities(db: Db) {
+    const entitiesCollection = db.collection<Entity>('entities');
+    const entityCursor = entitiesCollection.find({});
+
+    while (await entityCursor.hasNext()) {
+      const entity = await entityCursor.next();
+      await this.handleEntity(db, entity);
+    }
+    if (entitiesToUpdate.length) {
+      await this.flushUpdates(db);
+    }
+  },
+
+  async up(db: Db) {
+    process.stdout.write(`${this.name}...\r\n`);
+    await this.readProperties(db);
+    await this.handleEntities(db);
+  },
+};