From 8c7a7c5c3713aab9d518ee031043e9d24aedd5ce Mon Sep 17 00:00:00 2001 From: Michael Garcia Date: Thu, 19 Sep 2024 16:37:13 +0200 Subject: [PATCH 01/14] feat(api): anonymise newfields not specified by default --- .../anonymization/PlanDeTransport/ligneBus.js | 6 +- .../PlanDeTransport/modificationBus.js | 6 +- api/src/anonymization/application.js | 17 +- api/src/anonymization/cohesionCenter.js | 7 +- api/src/anonymization/contract.js | 44 +- api/src/anonymization/departmentService.js | 5 +- api/src/anonymization/meetingPoint.js | 6 +- api/src/anonymization/mission.js | 6 +- api/src/anonymization/missionEquivalence.js | 6 +- api/src/anonymization/referent.js | 5 +- api/src/anonymization/sessionPhase1.js | 6 +- api/src/anonymization/structure.js | 19 +- api/src/anonymization/waitingList.js | 6 +- api/src/anonymization/young.js | 55 +- api/src/utils/anonymise-model-fields.js | 806 ++++++++++++++++++ 15 files changed, 986 insertions(+), 14 deletions(-) create mode 100644 api/src/utils/anonymise-model-fields.js diff --git a/api/src/anonymization/PlanDeTransport/ligneBus.js b/api/src/anonymization/PlanDeTransport/ligneBus.js index 512c15cbc4..1abf52ea43 100644 --- a/api/src/anonymization/PlanDeTransport/ligneBus.js +++ b/api/src/anonymization/PlanDeTransport/ligneBus.js @@ -1,4 +1,5 @@ const { generateRandomName, generateBirthdate, generateRandomEmail, generateNewPhoneNumber } = require("../../utils/anonymise"); +const { ligneBusSchemaFields, anonymizeNewFields } = require("../../utils/anonymise-model-fields"); function anonymize(item) { item.team && @@ -10,7 +11,10 @@ function anonymize(item) { t.phone && (t.phone = generateNewPhoneNumber()); return t; })); - return item; + + const knownFields = ["team.lastName", "team.firstName", "team.birthdate", "team.mail", "team.phone"]; + + return anonymizeNewFields(item, knownFields, ligneBusSchemaFields); } module.exports = anonymize; diff --git a/api/src/anonymization/PlanDeTransport/modificationBus.js b/api/src/anonymization/PlanDeTransport/modificationBus.js index 05f05a459a..3ea0a620df 100644 --- a/api/src/anonymization/PlanDeTransport/modificationBus.js +++ b/api/src/anonymization/PlanDeTransport/modificationBus.js @@ -1,4 +1,5 @@ const { starify } = require("../../utils/anonymise"); +const { modificationBusSchemaFields, anonymizeNewFields } = require("../../utils/anonymise-model-fields"); function anonymize(item) { item.requestMessage && (item.requestMessage = starify(item.requestMessage)); @@ -11,7 +12,10 @@ function anonymize(item) { message.userName = starify(message.userName); return message; })); - return item; + + const knownFields = ["requestMessage", "requestUserName", "statusUserName", "opinionUserName", "messages.message", "messages.userName"]; + + return anonymizeNewFields(item, knownFields, modificationBusSchemaFields); } module.exports = anonymize; diff --git a/api/src/anonymization/application.js b/api/src/anonymization/application.js index 529d55abb6..a89d6aac4e 100644 --- a/api/src/anonymization/application.js +++ b/api/src/anonymization/application.js @@ -1,4 +1,5 @@ const { starify } = require("../utils/anonymise"); +const { applicationSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); function anonymize(item) { item.youngEmail && (item.youngEmail = "*****@*******.***"); @@ -14,7 +15,21 @@ function anonymize(item) { item.feedBackExperienceFiles && (item.feedBackExperienceFiles = []); item.othersFiles && (item.othersFiles = []); - return item; + const knownFields = [ + "youngEmail", + "youngFirstName", + "youngLastName", + "youngBirthdateAt", + "tutorName", + "missionName", + "contractStatus", + "contractAvenantFiles", + "justificatifsFiles", + "feedBackExperienceFiles", + "othersFiles", + ]; + + return anonymizeNewFields(item, knownFields, applicationSchemaFields); } module.exports = anonymize; diff --git a/api/src/anonymization/cohesionCenter.js b/api/src/anonymization/cohesionCenter.js index c291baa5dd..21f2fc9bcf 100644 --- a/api/src/anonymization/cohesionCenter.js +++ b/api/src/anonymization/cohesionCenter.js @@ -1,7 +1,12 @@ +const { cohesionCenterSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); + function anonymize(item) { item.code2022 && (item.code2022 = "02022"); item.code && (item.code = "00000"); - return item; + + const knownFields = ["code2022", "code"]; + + return anonymizeNewFields(item, knownFields, cohesionCenterSchemaFields); } module.exports = anonymize; diff --git a/api/src/anonymization/contract.js b/api/src/anonymization/contract.js index 79851e2930..00815e9fa9 100644 --- a/api/src/anonymization/contract.js +++ b/api/src/anonymization/contract.js @@ -1,4 +1,5 @@ const { generateBirthdate, generateNewPhoneNumber, starify, STAR_EMAIL } = require("../utils/anonymise"); +const { contractSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); function anonymize(item) { item.tutorFirstName && (item.tutorFirstName = starify(item.tutorFirstName)); @@ -41,7 +42,48 @@ function anonymize(item) { item.parent2Token = ""; item.youngContractToken = ""; - return item; + const knownFields = [ + "tutorFirstName", + "tutorLastName", + "tutorEmail", + "youngFirstName", + "youngLastName", + "youngBirthdate", + "youngEmail", + "youngPhone", + "youngAddress", + "parent1FirstName", + "parent1LastName", + "parent1Email", + "parent1Address", + "parent1Phone", + "parent2FirstName", + "parent2LastName", + "parent2Email", + "parent2Address", + "parent2Phone", + "missionName", + "missionAddress", + "missionZip", + "missionObjective", + "missionAction", + "missionFrequence", + "missionDuration", + "projectManagerFirstName", + "projectManagerLastName", + "projectManagerEmail", + "structureName", + "structureManagerEmail", + "structureManagerFirstName", + "structureManagerLastName", + "parent1Token", + "projectManagerToken", + "structureManagerToken", + "parent2Token", + "youngContractToken", + ]; + + return anonymizeNewFields(item, knownFields, contractSchemaFields); } module.exports = anonymize; diff --git a/api/src/anonymization/departmentService.js b/api/src/anonymization/departmentService.js index bfce8b8cf5..7b3d5fa6ca 100644 --- a/api/src/anonymization/departmentService.js +++ b/api/src/anonymization/departmentService.js @@ -1,4 +1,5 @@ const { generateAddress, generateRandomName, generateRandomEmail, generateNewPhoneNumber } = require("../utils/anonymise"); +const { departmentServiceSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); function anonymize(item) { item.email && (item.email = generateRandomEmail()); @@ -24,7 +25,9 @@ function anonymize(item) { email: generateRandomEmail(), }); - return item; + const knownFields = ["email", "contactPhone", "address", "directionName", "contactName", "contactMail", "contacts", "representantEtat"]; + + return anonymizeNewFields(item, knownFields, departmentServiceSchemaFields); } module.exports = anonymize; diff --git a/api/src/anonymization/meetingPoint.js b/api/src/anonymization/meetingPoint.js index 1bf98e53e5..68e6607021 100644 --- a/api/src/anonymization/meetingPoint.js +++ b/api/src/anonymization/meetingPoint.js @@ -1,9 +1,13 @@ const { generateAddress, starify } = require("../utils/anonymise"); +const { meetingPointSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); function anonymize(item) { item.departureAddress && (item.departureAddress = generateAddress()); item.centerCode && (item.centerCode = starify(item.centerCode)); - return item; + + const knownFields = ["departureAddress", "centerCode"]; + + return anonymizeNewFields(item, knownFields, meetingPointSchemaFields); } module.exports = anonymize; diff --git a/api/src/anonymization/mission.js b/api/src/anonymization/mission.js index 6dc5805abc..c66691d51e 100644 --- a/api/src/anonymization/mission.js +++ b/api/src/anonymization/mission.js @@ -1,4 +1,5 @@ const { generateAddress, generateRandomName, starify } = require("../utils/anonymise"); +const { missionSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); function anonymize(item) { item.name && (item.name = `Mission ${generateRandomName()}`); @@ -12,7 +13,10 @@ function anonymize(item) { item.contraintes && (item.contraintes = starify(item.contraintes)); item.frequence && (item.frequence = starify(item.frequence)); item.jvaRawData && (item.jvaRawData = undefined); - return item; + + const knownFields = ["name", "description", "address", "actions", "structureName", "tutorName", "justifications", "contraintes", "frequence", "jvaRawData"]; + + return anonymizeNewFields(item, knownFields, missionSchemaFields); } module.exports = anonymize; diff --git a/api/src/anonymization/missionEquivalence.js b/api/src/anonymization/missionEquivalence.js index 74e53194d8..627f5ae636 100644 --- a/api/src/anonymization/missionEquivalence.js +++ b/api/src/anonymization/missionEquivalence.js @@ -1,4 +1,5 @@ const { generateRandomName, generateRandomEmail, generateAddress, starify } = require("../utils/anonymise"); +const { missionEquivalenceSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); function anonymize(item) { item.message && (item.message = starify(item.message)); @@ -7,7 +8,10 @@ function anonymize(item) { item.contactFullName && (item.contactFullName = generateRandomName() + generateRandomName()); item.structureName && (item.structureName = generateRandomName()); item.files && (item.files = []); - return item; + + const knownFields = ["message", "address", "contactEmail", "contactFullName", "structureName", "files"]; + + return anonymizeNewFields(item, knownFields, missionEquivalenceSchemaFields); } module.exports = anonymize; diff --git a/api/src/anonymization/referent.js b/api/src/anonymization/referent.js index fd104a2286..a7585d4ca3 100644 --- a/api/src/anonymization/referent.js +++ b/api/src/anonymization/referent.js @@ -1,4 +1,5 @@ const { generateRandomName, generateRandomEmail, generateNewPhoneNumber } = require("../utils/anonymise"); +const { referenteSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); function anonymize(item) { item.phone && (item.phone = generateNewPhoneNumber()); @@ -11,7 +12,9 @@ function anonymize(item) { item.forgotPasswordResetToken = ""; item.token2FA = ""; - return item; + const knownFields = ["phone", "mobile", "email", "firstName", "lastName", "invitationToken", "forgotPasswordResetToken", "token2FA"]; + + return anonymizeNewFields(item, knownFields, referenteSchemaFields); } module.exports = anonymize; diff --git a/api/src/anonymization/sessionPhase1.js b/api/src/anonymization/sessionPhase1.js index 1469580125..64dd47ad2b 100644 --- a/api/src/anonymization/sessionPhase1.js +++ b/api/src/anonymization/sessionPhase1.js @@ -1,4 +1,5 @@ const { starify } = require("../utils/anonymise"); +const { sessionPhase1SchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); function anonymize(item) { item.zipCentre && (item.zipCentre = starify(item.zipCentre)); @@ -14,7 +15,10 @@ function anonymize(item) { member.phone && (member.phone = starify(member.phone)); return member; })); - return item; + + const knownFields = ["zipCentre", "codeCentre", "nameCentre", "cityCentre", "status", "team.firstName", "team.lastName", "team.email", "team.phone"]; + + return anonymizeNewFields(item, knownFields, sessionPhase1SchemaFields); } module.exports = anonymize; diff --git a/api/src/anonymization/structure.js b/api/src/anonymization/structure.js index f41e374724..eb0d3006e4 100644 --- a/api/src/anonymization/structure.js +++ b/api/src/anonymization/structure.js @@ -1,4 +1,5 @@ const { generateAddress, generateRandomName, starify, generateRandomEmail, generateNewPhoneNumber } = require("../utils/anonymise"); +const { structureSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); function anonymize(item) { item.name && (item.name = generateRandomName().toUpperCase()); @@ -14,7 +15,23 @@ function anonymize(item) { item.structureManager?.lastName && (item.structureManager.lastName = generateRandomName()); item.structureManager?.mobile && (item.structureManager.mobile = generateNewPhoneNumber()); item.structureManager?.email && (item.structureManager.email = generateRandomEmail()); - return item; + + const knownFields = [ + "name", + "siret", + "address", + "website", + "description", + "twitter", + "facebook", + "instagram", + "structureManager.firstName", + "structureManager.lastName", + "structureManager.mobile", + "structureManager.email", + ]; + + return anonymizeNewFields(item, knownFields, structureSchemaFields); } module.exports = anonymize; diff --git a/api/src/anonymization/waitingList.js b/api/src/anonymization/waitingList.js index ada204d62c..e8d09b05b2 100644 --- a/api/src/anonymization/waitingList.js +++ b/api/src/anonymization/waitingList.js @@ -1,9 +1,13 @@ const { generateRandomEmail, generateBirthdate } = require("../utils/anonymise"); +const { waitingListSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); function anonymize(item) { item.mail && (item.mail = generateRandomEmail()); item.birthdateAt && (item.birthdateAt = generateBirthdate()); - return item; + + const knownFields = ["mail", "birthdateAt"]; + + return anonymizeNewFields(item, knownFields, waitingListSchemaFields); } module.exports = anonymize; diff --git a/api/src/anonymization/young.js b/api/src/anonymization/young.js index 2b9682b165..68c57bf32d 100644 --- a/api/src/anonymization/young.js +++ b/api/src/anonymization/young.js @@ -1,4 +1,5 @@ const { generateAddress, generateRandomName, generateRandomEmail, generateBirthdate, getYoungLocation, generateNewPhoneNumber, starify } = require("../utils/anonymise"); +const { youngSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); function anonymize(item) { item.email && (item.email = generateRandomEmail()); @@ -70,7 +71,59 @@ function anonymize(item) { item.parent1Inscription2023Token = ""; item.parent2Inscription2023Token = ""; - return item; + const knownFields = [ + "email", + "newEmail", + "parent1Email", + "parent2Email", + "firstName", + "lastName", + "parent1FirstName", + "parent1LastName", + "parent2FirstName", + "parent2LastName", + "historic", + "phone", + "parent1Phone", + "parent2Phone", + "address", + "parent1Address", + "parent2Address", + "birthdateAt", + "engagedDescription", + "motivations", + "parentConsentmentFilesCompliantInfo", + "withdrawnReason", + "withdrawnMessage", + "correctionRequests.message", + "correctionRequests.reason", + "notes.note", + "notes.referent.firstName", + "notes.referent.lastName", + "location", + "cniFiles", + "highSkilledActivityProofFiles", + "dataProcessingConsentmentFiles", + "parentConsentmentFiles", + "imageRightFiles", + "autoTestPCRFiles", + "rulesFiles", + "militaryPreparationFilesIdentity", + "militaryPreparationFilesCensus", + "militaryPreparationFilesAuthorization", + "militaryPreparationFilesCertificate", + "militaryPreparationCorrectionMessage", + "files", + "token2FA", + "tokenEmailValidation", + "forgotPasswordResetToken", + "invitationToken", + "phase3Token", + "parent1Inscription2023Token", + "parent2Inscription2023Token", + ]; + + return anonymizeNewFields(item, knownFields, youngSchemaFields); } module.exports = anonymize; diff --git a/api/src/utils/anonymise-model-fields.js b/api/src/utils/anonymise-model-fields.js new file mode 100644 index 0000000000..f3a5b8e4e6 --- /dev/null +++ b/api/src/utils/anonymise-model-fields.js @@ -0,0 +1,806 @@ +const applicationSchemaFields = [ + "apiEngagementId", + "youngId", + "youngFirstName", + "youngLastName", + "youngEmail", + "youngBirthdateAt", + "youngCity", + "youngDepartment", + "youngCohort", + "cohortId", + "missionId", + "isJvaMission", + "missionName", + "missionDepartment", + "missionRegion", + "missionDuration", + "structureId", + "tutorId", + "contractId", + "contractStatus", + "tutorName", + "priority", + "hidden", + "status", + "statusComment", + "contractAvenantFiles", + "justificatifsFiles", + "feedBackExperienceFiles", + "othersFiles", + "filesType", + "createdAt", + "updatedAt", +]; + +const cohesionCenterSchemaFields = [ + "name", + "code2022", + "address", + "city", + "zip", + "department", + "region", + "addressVerified", + "placesTotal", + "pmr", + "cohorts", + "cohortIds", + "academy", + "typology", + "domain", + "complement", + "centerDesignation", + "placesLeft", + "outfitDelivered", + "observations", + "waitingList", + "COR", + "code", + "country", + "departmentCode", + "sessionStatus", + "createdAt", + "updatedAt", +]; + +const contractSchemaFields = [ + "youngId", + "structureId", + "applicationId", + "missionId", + "tutorId", + "isYoungAdult", + "parent1Token", + "projectManagerToken", + "structureManagerToken", + "parent2Token", + "youngContractToken", + "parent1Status", + "projectManagerStatus", + "structureManagerStatus", + "parent2Status", + "youngContractStatus", + "parent1ValidationDate", + "projectManagerValidationDate", + "structureManagerValidationDate", + "parent2ValidationDate", + "youngContractValidationDate", + "invitationSent", + "youngFirstName", + "youngLastName", + "youngBirthdate", + "youngAddress", + "youngCity", + "youngDepartment", + "youngEmail", + "youngPhone", + "parent1FirstName", + "parent1LastName", + "parent1Address", + "parent1City", + "parent1Department", + "parent1Phone", + "parent1Email", + "parent2FirstName", + "parent2LastName", + "parent2Address", + "parent2City", + "parent2Department", + "parent2Phone", + "parent2Email", + "missionName", + "missionObjective", + "missionAction", + "missionStartAt", + "missionEndAt", + "missionAddress", + "missionCity", + "missionZip", + "missionDuration", + "missionFrequence", + "date", + "projectManagerFirstName", + "projectManagerLastName", + "projectManagerRole", + "projectManagerEmail", + "structureManagerFirstName", + "structureManagerLastName", + "structureManagerRole", + "structureManagerEmail", + "tutorFirstName", + "tutorLastName", + "tutorRole", + "tutorEmail", + "structureSiret", + "structureName", + "createdAt", + "updatedAt", +]; + +const departmentServiceFields = [ + "contacts.cohort", + "contacts.cohortId", + "contacts.contactName", + "contacts.contactPhone", + "contacts.contactMail", + "department", + "region", + "directionName", + "serviceName", + "serviceNumber", + "address", + "complementAddress", + "zip", + "city", + "description", + "contactName", + "contactPhone", + "contactMail", + "representantEtat.firstName", + "representantEtat.lastName", + "representantEtat.mobile", + "representantEtat.email", + "representantEtat.role", + "createdAt", + "updatedAt", +]; + +const meetingPointSchemaFields = [ + "isValid", + "cohort", + "cohortId", + "busId", + "busExcelId", + "centerId", + "centerCode", + "departureAddress", + "departureZip", + "departureCity", + "departureDepartment", + "departureRegion", + "hideDepartmentInConvocation", + "departureAt", + "departureAtString", + "realDepartureAtString", + "returnAt", + "returnAtString", + "realReturnAtString", + "createdAt", + "updatedAt", + "deletedAt", +]; + +const missionSchemaFields = [ + "name", + "domains", + "mainDomain", + "startAt", + "endAt", + "duration", + "format", + "frequence", + "period", + "subPeriod", + "placesTotal", + "placesLeft", + "pendingApplications", + "actions", + "description", + "justifications", + "contraintes", + "structureId", + "structureName", + "status", + "visibility", + "statusComment", + "hebergement", + "hebergementPayant", + "tutorId", + "tutorName", + "address", + "zip", + "city", + "department", + "region", + "country", + "location.lat", + "location.lon", + "addressVerified", + "remote", + "isMilitaryPreparation", + "createdAt", + "updatedAt", + "lastSyncAt", + "isJvaMission", + "jvaMissionId", + "apiEngagementId", + "jvaRawData", + "applicationStatus", + "placesStatus", +]; + +const missionEquivalenceFields = [ + "youngId", + "status", + "type", + "desc", + "sousType", + "structureName", + "address", + "zip", + "city", + "startDate", + "endDate", + "frequency.nombre", + "frequency.duree", + "frequency.frequence", + "missionDuration", + "contactFullName", + "contactEmail", + "files", + "message", + "createdAt", + "updatedAt", +]; + +const referentSchemaFields = [ + "firstName", + "lastName", + "email", + "emailValidatedAt", + "emailWaitingValidation", + "password", + "loginAttempts", + "token2FA", + "token2FAExpires", + "attempts2FA", + "acceptCGU", + "lastLoginAt", + "lastActivityAt", + "lastLogoutAt", + "passwordChangedAt", + "registredAt", + "nextLoginAttemptIn", + "forgotPasswordResetToken", + "forgotPasswordResetExpires", + "invitationToken", + "invitationExpires", + "role", + "subRole", + "region", + "department", + "structureId", + "sessionPhase1Id", + "cohorts", + "cohortIds", + "cohesionCenterId", + "cohesionCenterName", + "phone", + "mobile", + "metadata", + "deletedAt", + "createdAt", + "updatedAt", +]; + +const sessionPhase1SchemaFields = [ + "cohesionCenterId", + "cohort", + "cohortId", + "department", + "region", + "codeCentre", + "nameCentre", + "zipCentre", + "cityCentre", + "headCenterId", + "team.firstName", + "team.lastName", + "team.role", + "team.email", + "team.phone", + "waitingList", + "placesTotal", + "placesLeft", + "timeScheduleFiles._id", + "timeScheduleFiles.name", + "timeScheduleFiles.uploadedAt", + "timeScheduleFiles.size", + "timeScheduleFiles.mimetype", + "hasTimeSchedule", + "pedagoProjectFiles._id", + "pedagoProjectFiles.name", + "pedagoProjectFiles.uploadedAt", + "pedagoProjectFiles.size", + "pedagoProjectFiles.mimetype", + "hasPedagoProject", + "dateStart", + "dateEnd", + "status", + "sanitaryContactEmail", + "createdAt", + "updatedAt", +]; + +const structureSchemaFields = [ + "name", + "siret", + "description", + "website", + "facebook", + "twitter", + "instagram", + "status", + "isNetwork", + "networkId", + "networkName", + "legalStatus", + "types", + "sousType", + "associationTypes", + "structurePubliqueType", + "structurePubliqueEtatType", + "structurePriveeType", + "address", + "zip", + "city", + "department", + "region", + "country", + "location.lon", + "location.lat", + "addressVerified", + "state", + "isMilitaryPreparation", + "isJvaStructure", + "jvaStructureId", + "jvaRawData", + "structureManager.firstName", + "structureManager.lastName", + "structureManager.mobile", + "structureManager.email", + "structureManager.role", + "createdAt", + "updatedAt", +]; + +const waitingListSchemaFields = ["zip", "mail", "birthdateAt", "createdAt", "updatedAt"]; + +const youngSchemaFields = [ + "firstName", + "lastName", + "frenchNationality", + "birthCountry", + "birthCity", + "birthCityZip", + "email", + "emailVerified", + "newEmail", + "phone", + "phoneZone", + "gender", + "birthdateAt", + "cohort", + "cohortId", + "originalCohort", + "originalCohortId", + "cohortChangeReason", + "cohortDetailedChangeReason", + "phase", + "status", + "statusPhase1", + "statusPhase1Tmp", + "statusPhase1Motif", + "statusPhase1MotifDetail", + "statusPhase2", + "statusPhase2UpdatedAt", + "statusPhase2OpenedAt", + "statusPhase2ValidatedAt", + "statusPhase2Contract", + "statusPhase3", + "statusPhase3UpdatedAt", + "statusPhase3ValidatedAt", + "lastStatusAt", + "withdrawnReason", + "withdrawnMessage", + "hasStartedReinscription", + "reinscriptionStep2023", + "inscriptionStep2023", + "inscriptionStep", + "inscriptionDoneDate", + "cohesion2020Step", + "inscriptionCorrectionMessage", + "inscriptionRefusedMessage", + "historic", + "password", + "token2FA", + "token2FAExpires", + "attempts2FA", + "tokenEmailValidation", + "tokenEmailValidationExpires", + "attemptsEmailValidation", + "loginAttempts", + "lastLoginAt", + "lastActivityAt", + "lastLogoutAt", + "passwordChangedAt", + "nextLoginAttemptIn", + "forgotPasswordResetToken", + "forgotPasswordResetExpires", + "invitationToken", + "invitationExpires", + "acceptCGU", + "acceptRI", + "cniFiles", + "cohesionStayPresence", + "presenceJDM", + "cohesionStayMedicalFileReceived", + "departInform", + "departSejourAt", + "departSejourMotif", + "departSejourMotifComment", + "cohesionStayMedicalFileDownload", + "convocationFileDownload", + "classeId", + "etablissementId", + "source", + "sessionPhase1Id", + "sessionPhase1IdTmp", + "cohesionCenterId", + "ligneId", + "meetingPointId", + "deplacementPhase1Autonomous", + "transportInfoGivenByLocal", + "hasMeetingInformation", + "isTravelingByPlane", + "codeCenterTmp", + "busTmp", + "cohesionCenterName", + "cohesionCenterZip", + "cohesionCenterCity", + "autoAffectationPhase1ExpiresAt", + "phase2ApplicationStatus", + "phase2ApplicationFilesType", + "phase2NumberHoursDone", + "phase2NumberHoursEstimated", + "phase3StructureName", + "phase3MissionDomain", + "phase3MissionDescription", + "phase3MissionStartAt", + "phase3MissionEndAt", + "phase3TutorFirstName", + "phase3TutorLastName", + "phase3TutorEmail", + "phase3TutorPhone", + "phase3TutorNote", + "phase3Token", + "address", + "coordinatesAccuracyLevel", + "complementAddress", + "zip", + "city", + "addressVerified", + "cityCode", + "populationDensity", + "isRegionRural", + "department", + "region", + "country", + "location.lat", + "location.lon", + "qpv", + "foreignAddress", + "foreignCity", + "foreignZip", + "foreignCountry", + "situation", + "grade", + "schoolCertification", + "schooled", + "schoolName", + "schoolNameOld", + "schoolType", + "schoolAddress", + "schoolComplementAdresse", + "schoolZip", + "schoolCity", + "schoolDepartment", + "schoolRegion", + "schoolCountry", + "schoolLocation.lat", + "schoolLocation.lon", + "schoolId", + "academy", + "employed", + "parent1Status", + "parent1FirstName", + "parent1LastName", + "parent1Email", + "parent1Phone", + "parent1PhoneZone", + "parent1OwnAddress", + "parent1Address", + "parent1coordinatesAccuracyLevel", + "parent1ComplementAddress", + "parent1Zip", + "parent1City", + "parent1CityCode", + "parent1Department", + "parent1Region", + "parent1Country", + "parent1Location.lat", + "parent1Location.lon", + "parent1FromFranceConnect", + "parent1Inscription2023Token", + "parent1DataVerified", + "parent1AddressVerified", + "parent1AllowCovidAutotest", + "parent1AllowImageRights", + "parent1ContactPreference", + "parent2Status", + "parent2FirstName", + "parent2LastName", + "parent2Email", + "parent2Phone", + "parent2PhoneZone", + "parent2OwnAddress", + "parent2Address", + "parent2coordinatesAccuracyLevel", + "parent2ComplementAddress", + "parent2Zip", + "parent2City", + "parent2CityCode", + "parent2Department", + "parent2Region", + "parent2Country", + "parent2Location.lat", + "parent2Location.lon", + "parent2FromFranceConnect", + "parent2Inscription2023Token", + "parent2AllowImageRights", + "parent2AllowImageRightsReset", + "parent2ContactPreference", + "hostLastName", + "hostFirstName", + "hostRelationship", + "hostCity", + "hostZip", + "hostAddress", + "hostDepartment", + "hostRegion", + "handicap", + "allergies", + "handicapInSameDepartment", + "reducedMobilityAccess", + "ppsBeneficiary", + "paiBeneficiary", + "medicosocialStructure", + "medicosocialStructureName", + "medicosocialStructureAddress", + "medicosocialStructureComplementAddress", + "medicosocialStructureZip", + "medicosocialStructureCity", + "medicosocialStructureDepartment", + "medicosocialStructureRegion", + "medicosocialStructureLocation.lat", + "medicosocialStructureLocation.lon", + "engagedStructure", + "sameSchoolCLE", + "specificAmenagment", + "specificAmenagmentType", + "highSkilledActivity", + "highSkilledActivityInSameDepartment", + "highSkilledActivityType", + "highSkilledActivityProofFiles", + "parentAllowSNU", + "parent1AllowSNU", + "parent2AllowSNU", + "parent1ValidationDate", + "parent2ValidationDate", + "parent2RejectSNUComment", + "dataProcessingConsentmentFiles", + "parentConsentment", + "parentConsentmentFiles", + "parentConsentmentFilesCompliant", + "parentConsentmentFilesCompliantInfo", + "consentment", + "imageRight", + "imageRightFiles", + "imageRightFilesStatus", + "psc1Info", + "imageRightFilesComment", + "autoTestPCR", + "autoTestPCRFiles", + "autoTestPCRFilesStatus", + "autoTestPCRFilesComment", + "rulesYoung", + "rulesParent1", + "rulesParent2", + "rulesFiles", + "informationAccuracy", + "aknowledgmentTerminaleSessionAvailability", + "parentStatementOfHonorInvalidId", + "jdc", + "motivations", + "domains", + "professionnalProject", + "professionnalProjectPrecision", + "period", + "periodRanking", + "mobilityNearSchool", + "mobilityNearHome", + "mobilityNearRelative", + "mobilityNearRelativeName", + "mobilityNearRelativeAddress", + "mobilityNearRelativeZip", + "mobilityNearRelativeCity", + "mobilityTransport", + "mobilityTransportOther", + "missionFormat", + "engaged", + "engagedDescription", + "desiredLocation", + "militaryPreparationFilesIdentity", + "militaryPreparationFilesCensus", + "militaryPreparationFilesAuthorization", + "militaryPreparationFilesCertificate", + "statusMilitaryPreparationFiles", + "militaryPreparationCorrectionMessage", + "files.cniFiles", + "files.highSkilledActivityProofFiles", + "files.dataProcessingConsentmentFiles", + "files.parentConsentmentFiles", + "files.imageRightFiles", + "files.autoTestPCRFiles", + "files.rulesFiles", + "files.militaryPreparationFilesIdentity", + "files.militaryPreparationFilesCensus", + "files.militaryPreparationFilesAuthorization", + "files.militaryPreparationFilesCertificate", + "latestCNIFileExpirationDate", + "CNIFileNotValidOnStart", + "latestCNIFileCategory", + "missionsInMail.missionId", + "missionsInMail.date", + "youngPhase1Agreement", + "status_equivalence", + "correctionRequests", + "notes", + "hasNotes", + "defenseInterest", + "defenseTypeInterest", + "defenseDomainInterest", + "defenseMotivationInterest", + "securityInterest", + "securityDomainInterest", + "solidarityInterest", + "healthInterest", + "educationInterest", + "cultureInterest", + "sportInterest", + "environmentInterest", + "citizenshipInterest", + "deletedAt", + "createdAt", + "updatedAt", +]; + +const ligneBusSchemaFields = [ + "cohort", + "cohortId", + "busId", + "departuredDate", + "returnDate", + "youngCapacity", + "totalCapacity", + "followerCapacity", + "youngSeatsTaken", + "travelTime", + "lunchBreak", + "lunchBreakReturn", + "sessionId", + "centerId", + "centerArrivalTime", + "centerDepartureTime", + "classeId", + "meetingPointsIds", + "team", + "delayedForth", + "delayedBack", + "mergedBusIds", + "createdAt", + "updatedAt", + "deletedAt", +]; + +const modificationBusSchemaFields = [ + "cohort", + "cohortId", + "lineId", + "lineName", + "requestMessage", + "requestUserId", + "requestUserName", + "requestUserRole", + "tagIds", + "status", + "statusUserId", + "statusUserName", + "statusDate", + "opinion", + "opinionUserId", + "opinionUserName", + "opinionDate", + "messages", + "createdAt", + "updatedAt", + "deletedAt", +]; + +function getNestedValue(obj, path) { + return path.split(".").reduce((acc, part) => acc && acc[part], obj); +} + +function setNestedValue(obj, path, value) { + const keys = path.split("."); + const lastKey = keys.pop(); + const deepObj = keys.reduce((acc, key) => (acc[key] = acc[key] || {}), obj); + deepObj[lastKey] = value; +} + +function anonymizeNewFields(item, knownFields, schemaFields) { + for (const path of schemaFields) { + if (!knownFields.includes(path)) { + const value = getNestedValue(item, path); + if (value !== undefined) { + if (Array.isArray(value)) { + setNestedValue(item, path, []); + } else if (typeof value === "string") { + setNestedValue(item, path, ""); + } else { + setNestedValue(item, path, null); + } + } + } + } + return item; +} + +module.exports = { + applicationSchemaFields, + cohesionCenterSchemaFields, + contractSchemaFields, + departmentServiceFields, + meetingPointSchemaFields, + missionSchemaFields, + missionEquivalenceFields, + referentSchemaFields, + sessionPhase1SchemaFields, + structureSchemaFields, + waitingListSchemaFields, + youngSchemaFields, + ligneBusSchemaFields, + modificationBusSchemaFields, + anonymizeNewFields, +}; From 6235726b45ffdb3975696dde68066ac9305a883d Mon Sep 17 00:00:00 2001 From: Michael Garcia Date: Thu, 19 Sep 2024 18:22:56 +0200 Subject: [PATCH 02/14] [WIP] --- api/src/utils/anonymise-model-fields.js | 107 +++++++++++++++--------- 1 file changed, 69 insertions(+), 38 deletions(-) diff --git a/api/src/utils/anonymise-model-fields.js b/api/src/utils/anonymise-model-fields.js index f3a5b8e4e6..25917bee82 100644 --- a/api/src/utils/anonymise-model-fields.js +++ b/api/src/utils/anonymise-model-fields.js @@ -1,4 +1,5 @@ const applicationSchemaFields = [ + "_id.$oid", "apiEngagementId", "youngId", "youngFirstName", @@ -29,11 +30,12 @@ const applicationSchemaFields = [ "feedBackExperienceFiles", "othersFiles", "filesType", - "createdAt", - "updatedAt", + "createdAt.$date", + "updatedAt.$date", ]; const cohesionCenterSchemaFields = [ + "_id.$oid", "name", "code2022", "address", @@ -60,11 +62,12 @@ const cohesionCenterSchemaFields = [ "country", "departmentCode", "sessionStatus", - "createdAt", - "updatedAt", + "createdAt.$date", + "updatedAt.$date", ]; const contractSchemaFields = [ + "_id.$oid", "youngId", "structureId", "applicationId", @@ -134,11 +137,12 @@ const contractSchemaFields = [ "tutorEmail", "structureSiret", "structureName", - "createdAt", - "updatedAt", + "createdAt.$date", + "updatedAt.$date", ]; const departmentServiceFields = [ + "_id.$oid", "contacts.cohort", "contacts.cohortId", "contacts.contactName", @@ -162,11 +166,12 @@ const departmentServiceFields = [ "representantEtat.mobile", "representantEtat.email", "representantEtat.role", - "createdAt", - "updatedAt", + "createdAt.$date", + "updatedAt.$date", ]; const meetingPointSchemaFields = [ + "_id.$oid", "isValid", "cohort", "cohortId", @@ -186,12 +191,13 @@ const meetingPointSchemaFields = [ "returnAt", "returnAtString", "realReturnAtString", - "createdAt", - "updatedAt", - "deletedAt", + "createdAt.$date", + "updatedAt.$date", + "deletedAt.$date", ]; const missionSchemaFields = [ + "_id.$oid", "name", "domains", "mainDomain", @@ -229,8 +235,8 @@ const missionSchemaFields = [ "addressVerified", "remote", "isMilitaryPreparation", - "createdAt", - "updatedAt", + "createdAt.$date", + "updatedAt.$date", "lastSyncAt", "isJvaMission", "jvaMissionId", @@ -241,6 +247,7 @@ const missionSchemaFields = [ ]; const missionEquivalenceFields = [ + "_id.$oid", "youngId", "status", "type", @@ -260,11 +267,12 @@ const missionEquivalenceFields = [ "contactEmail", "files", "message", - "createdAt", - "updatedAt", + "createdAt.$date", + "updatedAt.$date", ]; const referentSchemaFields = [ + "_id.$oid", "firstName", "lastName", "email", @@ -299,12 +307,13 @@ const referentSchemaFields = [ "phone", "mobile", "metadata", - "deletedAt", - "createdAt", - "updatedAt", + "deletedAt.$date", + "createdAt.$date", + "updatedAt.$date", ]; const sessionPhase1SchemaFields = [ + "_id.$oid", "cohesionCenterId", "cohort", "cohortId", @@ -339,11 +348,12 @@ const sessionPhase1SchemaFields = [ "dateEnd", "status", "sanitaryContactEmail", - "createdAt", - "updatedAt", + "createdAt.$date", + "updatedAt.$date", ]; const structureSchemaFields = [ + "_id.$oid", "name", "siret", "description", @@ -381,13 +391,14 @@ const structureSchemaFields = [ "structureManager.mobile", "structureManager.email", "structureManager.role", - "createdAt", - "updatedAt", + "createdAt.$date", + "updatedAt.$date", ]; -const waitingListSchemaFields = ["zip", "mail", "birthdateAt", "createdAt", "updatedAt"]; +const waitingListSchemaFields = ["_id.$oid", "zip", "mail", "birthdateAt", "createdAt.$date", "updatedAt.$date"]; const youngSchemaFields = [ + "_id.$oid", "firstName", "lastName", "frenchNationality", @@ -414,12 +425,12 @@ const youngSchemaFields = [ "statusPhase1Motif", "statusPhase1MotifDetail", "statusPhase2", - "statusPhase2UpdatedAt", + "statusPhase2updatedAt.$date", "statusPhase2OpenedAt", "statusPhase2ValidatedAt", "statusPhase2Contract", "statusPhase3", - "statusPhase3UpdatedAt", + "statusPhase3updatedAt.$date", "statusPhase3ValidatedAt", "lastStatusAt", "withdrawnReason", @@ -701,17 +712,18 @@ const youngSchemaFields = [ "sportInterest", "environmentInterest", "citizenshipInterest", - "deletedAt", - "createdAt", - "updatedAt", + "deletedAt.$date", + "createdAt.$date", + "updatedAt.$date", ]; const ligneBusSchemaFields = [ + "_id.$oid", "cohort", "cohortId", "busId", - "departuredDate", - "returnDate", + "departuredDate.$date", + "returnDate.$date", "youngCapacity", "totalCapacity", "followerCapacity", @@ -729,12 +741,14 @@ const ligneBusSchemaFields = [ "delayedForth", "delayedBack", "mergedBusIds", - "createdAt", - "updatedAt", - "deletedAt", + "createdAt.$date.$date", + "updatedAt.$date.$date", + "deletedAt.$date.$date", + "__v", ]; const modificationBusSchemaFields = [ + "_id.$oid", "cohort", "cohortId", "lineId", @@ -753,9 +767,9 @@ const modificationBusSchemaFields = [ "opinionUserName", "opinionDate", "messages", - "createdAt", - "updatedAt", - "deletedAt", + "createdAt.$date", + "updatedAt.$date", + "deletedAt.$date", ]; function getNestedValue(obj, path) { @@ -769,9 +783,24 @@ function setNestedValue(obj, path, value) { deepObj[lastKey] = value; } +function getAllPaths(obj, parentPath = "") { + let paths = []; + for (let key in obj) { + const currentPath = parentPath ? `${parentPath}.${key}` : key; + if (typeof obj[key] === "object" && obj[key] !== null && !Array.isArray(obj[key])) { + paths = paths.concat(getAllPaths(obj[key], currentPath)); + } else { + paths.push(currentPath); + } + } + return paths; +} + function anonymizeNewFields(item, knownFields, schemaFields) { - for (const path of schemaFields) { - if (!knownFields.includes(path)) { + const allPaths = getAllPaths(item); + + for (const path of allPaths) { + if (!knownFields.includes(path) && !schemaFields.includes(path)) { const value = getNestedValue(item, path); if (value !== undefined) { if (Array.isArray(value)) { @@ -784,6 +813,8 @@ function anonymizeNewFields(item, knownFields, schemaFields) { } } } + + console.log("Anonymized Item:", item); return item; } From 0458ed82edb6c6ab7cd2fb756ce634292a97534f Mon Sep 17 00:00:00 2001 From: Michael Garcia Date: Tue, 24 Sep 2024 12:29:06 +0200 Subject: [PATCH 03/14] uppppp --- api/src/anonymization/referent.js | 4 +- api/src/utils/anonymise-model-fields.js | 134 ++++++++++++++---------- 2 files changed, 79 insertions(+), 59 deletions(-) diff --git a/api/src/anonymization/referent.js b/api/src/anonymization/referent.js index a7585d4ca3..4aa1a69461 100644 --- a/api/src/anonymization/referent.js +++ b/api/src/anonymization/referent.js @@ -1,5 +1,5 @@ const { generateRandomName, generateRandomEmail, generateNewPhoneNumber } = require("../utils/anonymise"); -const { referenteSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); +const { referentSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); function anonymize(item) { item.phone && (item.phone = generateNewPhoneNumber()); @@ -14,7 +14,7 @@ function anonymize(item) { const knownFields = ["phone", "mobile", "email", "firstName", "lastName", "invitationToken", "forgotPasswordResetToken", "token2FA"]; - return anonymizeNewFields(item, knownFields, referenteSchemaFields); + return anonymizeNewFields(item, knownFields, referentSchemaFields); } module.exports = anonymize; diff --git a/api/src/utils/anonymise-model-fields.js b/api/src/utils/anonymise-model-fields.js index 25917bee82..ff840ebff7 100644 --- a/api/src/utils/anonymise-model-fields.js +++ b/api/src/utils/anonymise-model-fields.js @@ -64,6 +64,7 @@ const cohesionCenterSchemaFields = [ "sessionStatus", "createdAt.$date", "updatedAt.$date", + "__v", ]; const contractSchemaFields = [ @@ -84,10 +85,10 @@ const contractSchemaFields = [ "structureManagerStatus", "parent2Status", "youngContractStatus", - "parent1ValidationDate", - "projectManagerValidationDate", - "structureManagerValidationDate", - "parent2ValidationDate", + "parent1ValidationDate.$date", + "projectManagerValidationDate.$date", + "structureManagerValidationDate.$date", + "parent2ValidationDate.$date", "youngContractValidationDate", "invitationSent", "youngFirstName", @@ -139,9 +140,10 @@ const contractSchemaFields = [ "structureName", "createdAt.$date", "updatedAt.$date", + "__v", ]; -const departmentServiceFields = [ +const departmentServiceSchemaFields = [ "_id.$oid", "contacts.cohort", "contacts.cohortId", @@ -168,6 +170,7 @@ const departmentServiceFields = [ "representantEtat.role", "createdAt.$date", "updatedAt.$date", + "__v", ]; const meetingPointSchemaFields = [ @@ -185,15 +188,16 @@ const meetingPointSchemaFields = [ "departureDepartment", "departureRegion", "hideDepartmentInConvocation", - "departureAt", + "departureAt.$date", "departureAtString", "realDepartureAtString", - "returnAt", + "returnAt.$date", "returnAtString", "realReturnAtString", "createdAt.$date", "updatedAt.$date", "deletedAt.$date", + "__v", ]; const missionSchemaFields = [ @@ -201,8 +205,8 @@ const missionSchemaFields = [ "name", "domains", "mainDomain", - "startAt", - "endAt", + "startAt.$date", + "endAt.$date", "duration", "format", "frequence", @@ -237,16 +241,17 @@ const missionSchemaFields = [ "isMilitaryPreparation", "createdAt.$date", "updatedAt.$date", - "lastSyncAt", + "lastSyncAt.$date", "isJvaMission", "jvaMissionId", "apiEngagementId", "jvaRawData", "applicationStatus", "placesStatus", + "__v", ]; -const missionEquivalenceFields = [ +const missionEquivalenceSchemaFields = [ "_id.$oid", "youngId", "status", @@ -257,8 +262,8 @@ const missionEquivalenceFields = [ "address", "zip", "city", - "startDate", - "endDate", + "startDate.$date", + "endDate.$date", "frequency.nombre", "frequency.duree", "frequency.frequence", @@ -269,6 +274,7 @@ const missionEquivalenceFields = [ "message", "createdAt.$date", "updatedAt.$date", + "__v", ]; const referentSchemaFields = [ @@ -276,24 +282,24 @@ const referentSchemaFields = [ "firstName", "lastName", "email", - "emailValidatedAt", + "emailValidatedAt.$date", "emailWaitingValidation", "password", "loginAttempts", "token2FA", - "token2FAExpires", + "token2FAExpires.$date", "attempts2FA", "acceptCGU", - "lastLoginAt", - "lastActivityAt", - "lastLogoutAt", - "passwordChangedAt", - "registredAt", - "nextLoginAttemptIn", + "lastLoginAt.$date", + "lastActivityAt.$date", + "lastLogoutAt.$date", + "passwordChangedAt.$date", + "registredAt.$date", + "nextLoginAttemptIn.$date", "forgotPasswordResetToken", - "forgotPasswordResetExpires", + "forgotPasswordResetExpires.$date", "invitationToken", - "invitationExpires", + "invitationExpires.$date", "role", "subRole", "region", @@ -306,10 +312,11 @@ const referentSchemaFields = [ "cohesionCenterName", "phone", "mobile", - "metadata", + "metadata._id.$oid", "deletedAt.$date", "createdAt.$date", "updatedAt.$date", + "__v", ]; const sessionPhase1SchemaFields = [ @@ -344,12 +351,13 @@ const sessionPhase1SchemaFields = [ "pedagoProjectFiles.size", "pedagoProjectFiles.mimetype", "hasPedagoProject", - "dateStart", - "dateEnd", + "dateStart.$date", + "dateEnd.$date", "status", "sanitaryContactEmail", "createdAt.$date", "updatedAt.$date", + "__v", ]; const structureSchemaFields = [ @@ -393,6 +401,7 @@ const structureSchemaFields = [ "structureManager.role", "createdAt.$date", "updatedAt.$date", + "__v", ]; const waitingListSchemaFields = ["_id.$oid", "zip", "mail", "birthdateAt", "createdAt.$date", "updatedAt.$date"]; @@ -411,7 +420,7 @@ const youngSchemaFields = [ "phone", "phoneZone", "gender", - "birthdateAt", + "birthdateAt.$date", "cohort", "cohortId", "originalCohort", @@ -426,41 +435,41 @@ const youngSchemaFields = [ "statusPhase1MotifDetail", "statusPhase2", "statusPhase2updatedAt.$date", - "statusPhase2OpenedAt", - "statusPhase2ValidatedAt", + "statusPhase2OpenedAt.$date", + "statusPhase2ValidatedAt.$date", "statusPhase2Contract", "statusPhase3", "statusPhase3updatedAt.$date", - "statusPhase3ValidatedAt", - "lastStatusAt", + "statusPhase3ValidatedAt.$date", + "lastStatusAt.$date", "withdrawnReason", "withdrawnMessage", "hasStartedReinscription", "reinscriptionStep2023", "inscriptionStep2023", "inscriptionStep", - "inscriptionDoneDate", + "inscriptionDoneDate.$date", "cohesion2020Step", "inscriptionCorrectionMessage", "inscriptionRefusedMessage", "historic", "password", "token2FA", - "token2FAExpires", + "token2FAExpires.$date", "attempts2FA", "tokenEmailValidation", - "tokenEmailValidationExpires", + "tokenEmailValidationExpires.$date", "attemptsEmailValidation", "loginAttempts", - "lastLoginAt", - "lastActivityAt", - "lastLogoutAt", - "passwordChangedAt", - "nextLoginAttemptIn", + "lastLoginAt.$date", + "lastActivityAt.$date", + "lastLogoutAt.$date", + "passwordChangedAt.$date", + "nextLoginAttemptIn.$date", "forgotPasswordResetToken", - "forgotPasswordResetExpires", + "forgotPasswordResetExpires.$date", "invitationToken", - "invitationExpires", + "invitationExpires.$date", "acceptCGU", "acceptRI", "cniFiles", @@ -468,7 +477,7 @@ const youngSchemaFields = [ "presenceJDM", "cohesionStayMedicalFileReceived", "departInform", - "departSejourAt", + "departSejourAt.$date", "departSejourMotif", "departSejourMotifComment", "cohesionStayMedicalFileDownload", @@ -490,7 +499,7 @@ const youngSchemaFields = [ "cohesionCenterName", "cohesionCenterZip", "cohesionCenterCity", - "autoAffectationPhase1ExpiresAt", + "autoAffectationPhase1ExpiresAt.$date", "phase2ApplicationStatus", "phase2ApplicationFilesType", "phase2NumberHoursDone", @@ -498,8 +507,8 @@ const youngSchemaFields = [ "phase3StructureName", "phase3MissionDomain", "phase3MissionDescription", - "phase3MissionStartAt", - "phase3MissionEndAt", + "phase3MissionStartAt.$date", + "phase3MissionEndAt.$date", "phase3TutorFirstName", "phase3TutorLastName", "phase3TutorEmail", @@ -627,8 +636,8 @@ const youngSchemaFields = [ "parentAllowSNU", "parent1AllowSNU", "parent2AllowSNU", - "parent1ValidationDate", - "parent2ValidationDate", + "parent1ValidationDate.$date", + "parent2ValidationDate.$date", "parent2RejectSNUComment", "dataProcessingConsentmentFiles", "parentConsentment", @@ -639,6 +648,7 @@ const youngSchemaFields = [ "imageRight", "imageRightFiles", "imageRightFilesStatus", + "rulesFilesStatus", "psc1Info", "imageRightFilesComment", "autoTestPCR", @@ -689,11 +699,11 @@ const youngSchemaFields = [ "files.militaryPreparationFilesCensus", "files.militaryPreparationFilesAuthorization", "files.militaryPreparationFilesCertificate", - "latestCNIFileExpirationDate", + "latestCNIFileExpirationDate.$date", "CNIFileNotValidOnStart", "latestCNIFileCategory", "missionsInMail.missionId", - "missionsInMail.date", + "missionsInMail.date.$date", "youngPhase1Agreement", "status_equivalence", "correctionRequests", @@ -715,6 +725,7 @@ const youngSchemaFields = [ "deletedAt.$date", "createdAt.$date", "updatedAt.$date", + "__v", ]; const ligneBusSchemaFields = [ @@ -741,9 +752,9 @@ const ligneBusSchemaFields = [ "delayedForth", "delayedBack", "mergedBusIds", - "createdAt.$date.$date", - "updatedAt.$date.$date", - "deletedAt.$date.$date", + "createdAt.$date", + "updatedAt.$date", + "deletedAt.$date", "__v", ]; @@ -761,15 +772,19 @@ const modificationBusSchemaFields = [ "status", "statusUserId", "statusUserName", - "statusDate", + "statusDate.$date", "opinion", "opinionUserId", "opinionUserName", - "opinionDate", - "messages", + "opinionDate.$date", + "messages.message", + "messages.userId", + "messages.userName", + "messages.date.$date", "createdAt.$date", "updatedAt.$date", "deletedAt.$date", + "__v", ]; function getNestedValue(obj, path) { @@ -802,6 +817,11 @@ function anonymizeNewFields(item, knownFields, schemaFields) { for (const path of allPaths) { if (!knownFields.includes(path) && !schemaFields.includes(path)) { const value = getNestedValue(item, path); + + if (Array.isArray(value) && value.length > 0 && typeof value[0] === "object") { + continue; + } + if (value !== undefined) { if (Array.isArray(value)) { setNestedValue(item, path, []); @@ -822,10 +842,10 @@ module.exports = { applicationSchemaFields, cohesionCenterSchemaFields, contractSchemaFields, - departmentServiceFields, + departmentServiceSchemaFields, meetingPointSchemaFields, missionSchemaFields, - missionEquivalenceFields, + missionEquivalenceSchemaFields, referentSchemaFields, sessionPhase1SchemaFields, structureSchemaFields, From 18d6f2591b390db922c68f195e48a180ddab4583 Mon Sep 17 00:00:00 2001 From: Michael Garcia Date: Fri, 27 Sep 2024 15:36:40 +0200 Subject: [PATCH 04/14] up --- .../anonymization/PlanDeTransport/ligneBus.js | 43 +- .../PlanDeTransport/modificationBus.js | 38 +- api/src/anonymization/application.js | 58 +- api/src/anonymization/cohesionCenter.js | 43 +- api/src/anonymization/contract.js | 124 ++- api/src/anonymization/departmentService.js | 39 +- api/src/anonymization/meetingPoint.js | 36 +- api/src/anonymization/mission.js | 60 +- api/src/anonymization/missionEquivalence.js | 35 +- api/src/anonymization/referent.js | 51 +- api/src/anonymization/sessionPhase1.js | 50 +- api/src/anonymization/structure.js | 66 +- api/src/anonymization/waitingList.js | 12 +- api/src/anonymization/young.js | 381 ++++++-- api/src/utils/anonymise-model-fields.js | 811 +----------------- 15 files changed, 857 insertions(+), 990 deletions(-) diff --git a/api/src/anonymization/PlanDeTransport/ligneBus.js b/api/src/anonymization/PlanDeTransport/ligneBus.js index 1abf52ea43..8b006b6f20 100644 --- a/api/src/anonymization/PlanDeTransport/ligneBus.js +++ b/api/src/anonymization/PlanDeTransport/ligneBus.js @@ -1,7 +1,42 @@ const { generateRandomName, generateBirthdate, generateRandomEmail, generateNewPhoneNumber } = require("../../utils/anonymise"); -const { ligneBusSchemaFields, anonymizeNewFields } = require("../../utils/anonymise-model-fields"); +const { anonymizeNonDeclaredFields } = require("../../utils/anonymise-model-fields"); + +function anonymize(itemToAnonymize) { + const whitelist = [ + "_id.$oid", + "cohort", + "cohortId", + "busId", + "departuredDate.$date", + "returnDate.$date", + "youngCapacity", + "totalCapacity", + "followerCapacity", + "youngSeatsTaken", + "travelTime", + "lunchBreak", + "lunchBreakReturn", + "sessionId", + "centerId", + "centerArrivalTime", + "centerDepartureTime", + "classeId", + "meetingPointsIds", + "team.lastName", + "team.firstName", + "team.birthdate", + "team.mail", + "team.phone", + "delayedForth", + "delayedBack", + "mergedBusIds", + "createdAt.$date", + "updatedAt.$date", + "deletedAt.$date", + "__v", + ]; + const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); -function anonymize(item) { item.team && (item.team = item.team.map((t) => { t.lastName && (t.lastName = generateRandomName()); @@ -12,9 +47,7 @@ function anonymize(item) { return t; })); - const knownFields = ["team.lastName", "team.firstName", "team.birthdate", "team.mail", "team.phone"]; - - return anonymizeNewFields(item, knownFields, ligneBusSchemaFields); + return item; } module.exports = anonymize; diff --git a/api/src/anonymization/PlanDeTransport/modificationBus.js b/api/src/anonymization/PlanDeTransport/modificationBus.js index 3ea0a620df..4ee8be6912 100644 --- a/api/src/anonymization/PlanDeTransport/modificationBus.js +++ b/api/src/anonymization/PlanDeTransport/modificationBus.js @@ -1,7 +1,37 @@ const { starify } = require("../../utils/anonymise"); -const { modificationBusSchemaFields, anonymizeNewFields } = require("../../utils/anonymise-model-fields"); +const { anonymizeNonDeclaredFields } = require("../../utils/anonymise-model-fields"); + +function anonymize(itemToAnonymize) { + const whitelist = [ + "_id.$oid", + "cohort", + "cohortId", + "lineId", + "lineName", + "requestMessage", + "requestUserId", + "requestUserName", + "requestUserRole", + "tagIds", + "status", + "statusUserId", + "statusUserName", + "statusDate.$date", + "opinion", + "opinionUserId", + "opinionUserName", + "opinionDate.$date", + "messages.message", + "messages.userId", + "messages.userName", + "messages.date.$date", + "createdAt.$date", + "updatedAt.$date", + "deletedAt.$date", + "__v", + ]; + const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); -function anonymize(item) { item.requestMessage && (item.requestMessage = starify(item.requestMessage)); item.requestUserName && (item.requestUserName = starify(item.requestUserName)); item.statusUserName && (item.statusUserName = starify(item.statusUserName)); @@ -13,9 +43,7 @@ function anonymize(item) { return message; })); - const knownFields = ["requestMessage", "requestUserName", "statusUserName", "opinionUserName", "messages.message", "messages.userName"]; - - return anonymizeNewFields(item, knownFields, modificationBusSchemaFields); + return item; } module.exports = anonymize; diff --git a/api/src/anonymization/application.js b/api/src/anonymization/application.js index a89d6aac4e..fd5cefb82e 100644 --- a/api/src/anonymization/application.js +++ b/api/src/anonymization/application.js @@ -1,7 +1,45 @@ const { starify } = require("../utils/anonymise"); -const { applicationSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); +const { anonymizeNonDeclaredFields } = require("../utils/anonymise-model-fields"); + +function anonymize(itemToAnonymize) { + const whitelist = [ + "_id.$oid", + "apiEngagementId", + "youngId", + "youngFirstName", + "youngLastName", + "youngEmail", + "youngBirthdateAt", + "youngCity", + "youngDepartment", + "youngCohort", + "cohortId", + "missionId", + "isJvaMission", + "missionName", + "missionDepartment", + "missionRegion", + "missionDuration", + "structureId", + "tutorId", + "contractId", + "contractStatus", + "tutorName", + "priority", + "hidden", + "status", + "statusComment", + "contractAvenantFiles", + "justificatifsFiles", + "feedBackExperienceFiles", + "othersFiles", + "filesType", + "createdAt.$date", + "updatedAt.$date", + ]; + + const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); -function anonymize(item) { item.youngEmail && (item.youngEmail = "*****@*******.***"); item.youngFirstName && (item.youngFirstName = starify(item.youngFirstName)); item.youngLastName && (item.youngLastName = starify(item.youngLastName)); @@ -15,21 +53,7 @@ function anonymize(item) { item.feedBackExperienceFiles && (item.feedBackExperienceFiles = []); item.othersFiles && (item.othersFiles = []); - const knownFields = [ - "youngEmail", - "youngFirstName", - "youngLastName", - "youngBirthdateAt", - "tutorName", - "missionName", - "contractStatus", - "contractAvenantFiles", - "justificatifsFiles", - "feedBackExperienceFiles", - "othersFiles", - ]; - - return anonymizeNewFields(item, knownFields, applicationSchemaFields); + return item; } module.exports = anonymize; diff --git a/api/src/anonymization/cohesionCenter.js b/api/src/anonymization/cohesionCenter.js index 21f2fc9bcf..4d75c09105 100644 --- a/api/src/anonymization/cohesionCenter.js +++ b/api/src/anonymization/cohesionCenter.js @@ -1,12 +1,45 @@ -const { cohesionCenterSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); +const { anonymizeNonDeclaredFields } = require("../utils/anonymise-model-fields"); + +function anonymize(itemToAnonymize) { + const whitelist = [ + "_id.$oid", + "name", + "code2022", + "address", + "city", + "zip", + "department", + "region", + "addressVerified", + "placesTotal", + "pmr", + "cohorts", + "cohortIds", + "academy", + "typology", + "domain", + "complement", + "centerDesignation", + "placesLeft", + "outfitDelivered", + "observations", + "waitingList", + "COR", + "code", + "country", + "departmentCode", + "sessionStatus", + "createdAt.$date", + "updatedAt.$date", + "__v", + ]; + + const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); -function anonymize(item) { item.code2022 && (item.code2022 = "02022"); item.code && (item.code = "00000"); - const knownFields = ["code2022", "code"]; - - return anonymizeNewFields(item, knownFields, cohesionCenterSchemaFields); + return item; } module.exports = anonymize; diff --git a/api/src/anonymization/contract.js b/api/src/anonymization/contract.js index 00815e9fa9..402ef17e72 100644 --- a/api/src/anonymization/contract.js +++ b/api/src/anonymization/contract.js @@ -1,7 +1,84 @@ const { generateBirthdate, generateNewPhoneNumber, starify, STAR_EMAIL } = require("../utils/anonymise"); -const { contractSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); +const { anonymizeNonDeclaredFields } = require("../utils/anonymise-model-fields"); + +function anonymize(itemToAnonymize) { + const whitelist = [ + "_id.$oid", + "youngId", + "structureId", + "applicationId", + "missionId", + "tutorId", + "isYoungAdult", + "parent1Token", + "projectManagerToken", + "structureManagerToken", + "parent2Token", + "youngContractToken", + "parent1Status", + "projectManagerStatus", + "structureManagerStatus", + "parent2Status", + "youngContractStatus", + "parent1ValidationDate.$date", + "projectManagerValidationDate.$date", + "structureManagerValidationDate.$date", + "parent2ValidationDate.$date", + "youngContractValidationDate", + "invitationSent", + "youngFirstName", + "youngLastName", + "youngBirthdate", + "youngAddress", + "youngCity", + "youngDepartment", + "youngEmail", + "youngPhone", + "parent1FirstName", + "parent1LastName", + "parent1Address", + "parent1City", + "parent1Department", + "parent1Phone", + "parent1Email", + "parent2FirstName", + "parent2LastName", + "parent2Address", + "parent2City", + "parent2Department", + "parent2Phone", + "parent2Email", + "missionName", + "missionObjective", + "missionAction", + "missionStartAt", + "missionEndAt", + "missionAddress", + "missionCity", + "missionZip", + "missionDuration", + "missionFrequence", + "date", + "projectManagerFirstName", + "projectManagerLastName", + "projectManagerRole", + "projectManagerEmail", + "structureManagerFirstName", + "structureManagerLastName", + "structureManagerRole", + "structureManagerEmail", + "tutorFirstName", + "tutorLastName", + "tutorRole", + "tutorEmail", + "structureSiret", + "structureName", + "createdAt.$date", + "updatedAt.$date", + "__v", + ]; + const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); -function anonymize(item) { item.tutorFirstName && (item.tutorFirstName = starify(item.tutorFirstName)); item.tutorLastName && (item.tutorLastName = starify(item.tutorLastName)); item.tutorEmail && (item.tutorEmail = STAR_EMAIL); @@ -42,48 +119,7 @@ function anonymize(item) { item.parent2Token = ""; item.youngContractToken = ""; - const knownFields = [ - "tutorFirstName", - "tutorLastName", - "tutorEmail", - "youngFirstName", - "youngLastName", - "youngBirthdate", - "youngEmail", - "youngPhone", - "youngAddress", - "parent1FirstName", - "parent1LastName", - "parent1Email", - "parent1Address", - "parent1Phone", - "parent2FirstName", - "parent2LastName", - "parent2Email", - "parent2Address", - "parent2Phone", - "missionName", - "missionAddress", - "missionZip", - "missionObjective", - "missionAction", - "missionFrequence", - "missionDuration", - "projectManagerFirstName", - "projectManagerLastName", - "projectManagerEmail", - "structureName", - "structureManagerEmail", - "structureManagerFirstName", - "structureManagerLastName", - "parent1Token", - "projectManagerToken", - "structureManagerToken", - "parent2Token", - "youngContractToken", - ]; - - return anonymizeNewFields(item, knownFields, contractSchemaFields); + return item; } module.exports = anonymize; diff --git a/api/src/anonymization/departmentService.js b/api/src/anonymization/departmentService.js index 7b3d5fa6ca..ecb7b04f6c 100644 --- a/api/src/anonymization/departmentService.js +++ b/api/src/anonymization/departmentService.js @@ -1,7 +1,38 @@ const { generateAddress, generateRandomName, generateRandomEmail, generateNewPhoneNumber } = require("../utils/anonymise"); -const { departmentServiceSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); +const { anonymizeNonDeclaredFields } = require("../utils/anonymise-model-fields"); + +function anonymize(itemToAnonymize) { + const whitelist = [ + "_id.$oid", + "contacts.cohort", + "contacts.cohortId", + "contacts.contactName", + "contacts.contactPhone", + "contacts.contactMail", + "department", + "region", + "directionName", + "serviceName", + "serviceNumber", + "address", + "complementAddress", + "zip", + "city", + "description", + "contactName", + "contactPhone", + "contactMail", + "representantEtat.firstName", + "representantEtat.lastName", + "representantEtat.mobile", + "representantEtat.email", + "representantEtat.role", + "createdAt.$date", + "updatedAt.$date", + "__v", + ]; + const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); -function anonymize(item) { item.email && (item.email = generateRandomEmail()); item.contactPhone && (item.contactPhone = generateNewPhoneNumber()); item.address && (item.address = generateAddress()); @@ -25,9 +56,7 @@ function anonymize(item) { email: generateRandomEmail(), }); - const knownFields = ["email", "contactPhone", "address", "directionName", "contactName", "contactMail", "contacts", "representantEtat"]; - - return anonymizeNewFields(item, knownFields, departmentServiceSchemaFields); + return item; } module.exports = anonymize; diff --git a/api/src/anonymization/meetingPoint.js b/api/src/anonymization/meetingPoint.js index 68e6607021..cce21349c2 100644 --- a/api/src/anonymization/meetingPoint.js +++ b/api/src/anonymization/meetingPoint.js @@ -1,13 +1,39 @@ const { generateAddress, starify } = require("../utils/anonymise"); -const { meetingPointSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); +const { anonymizeNonDeclaredFields } = require("../utils/anonymise-model-fields"); + +function anonymize(itemToAnonymize) { + const whitelist = [ + "_id.$oid", + "isValid", + "cohort", + "cohortId", + "busId", + "busExcelId", + "centerId", + "centerCode", + "departureAddress", + "departureZip", + "departureCity", + "departureDepartment", + "departureRegion", + "hideDepartmentInConvocation", + "departureAt.$date", + "departureAtString", + "realDepartureAtString", + "returnAt.$date", + "returnAtString", + "realReturnAtString", + "createdAt.$date", + "updatedAt.$date", + "deletedAt.$date", + "__v", + ]; + const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); -function anonymize(item) { item.departureAddress && (item.departureAddress = generateAddress()); item.centerCode && (item.centerCode = starify(item.centerCode)); - const knownFields = ["departureAddress", "centerCode"]; - - return anonymizeNewFields(item, knownFields, meetingPointSchemaFields); + return item; } module.exports = anonymize; diff --git a/api/src/anonymization/mission.js b/api/src/anonymization/mission.js index c66691d51e..f595bfe9f7 100644 --- a/api/src/anonymization/mission.js +++ b/api/src/anonymization/mission.js @@ -1,7 +1,59 @@ const { generateAddress, generateRandomName, starify } = require("../utils/anonymise"); -const { missionSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); +const { anonymizeNonDeclaredFields } = require("../utils/anonymise-model-fields"); + +function anonymize(itemToAnonymize) { + const whitelist = [ + "_id.$oid", + "name", + "domains", + "mainDomain", + "startAt.$date", + "endAt.$date", + "duration", + "format", + "frequence", + "period", + "subPeriod", + "placesTotal", + "placesLeft", + "pendingApplications", + "actions", + "description", + "justifications", + "contraintes", + "structureId", + "structureName", + "status", + "visibility", + "statusComment", + "hebergement", + "hebergementPayant", + "tutorId", + "tutorName", + "address", + "zip", + "city", + "department", + "region", + "country", + "location.lat", + "location.lon", + "addressVerified", + "remote", + "isMilitaryPreparation", + "createdAt.$date", + "updatedAt.$date", + "lastSyncAt.$date", + "isJvaMission", + "jvaMissionId", + "apiEngagementId", + "jvaRawData", + "applicationStatus", + "placesStatus", + "__v", + ]; + const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); -function anonymize(item) { item.name && (item.name = `Mission ${generateRandomName()}`); item.description && (item.description = starify(item.description)); item.address && (item.address = generateAddress()); @@ -14,9 +66,7 @@ function anonymize(item) { item.frequence && (item.frequence = starify(item.frequence)); item.jvaRawData && (item.jvaRawData = undefined); - const knownFields = ["name", "description", "address", "actions", "structureName", "tutorName", "justifications", "contraintes", "frequence", "jvaRawData"]; - - return anonymizeNewFields(item, knownFields, missionSchemaFields); + return item; } module.exports = anonymize; diff --git a/api/src/anonymization/missionEquivalence.js b/api/src/anonymization/missionEquivalence.js index 627f5ae636..0846291a28 100644 --- a/api/src/anonymization/missionEquivalence.js +++ b/api/src/anonymization/missionEquivalence.js @@ -1,7 +1,34 @@ const { generateRandomName, generateRandomEmail, generateAddress, starify } = require("../utils/anonymise"); -const { missionEquivalenceSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); +const { anonymizeNonDeclaredFields } = require("../utils/anonymise-model-fields"); + +function anonymize(itemToAnonymize) { + const whitelist = [ + "_id.$oid", + "youngId", + "status", + "type", + "desc", + "sousType", + "structureName", + "address", + "zip", + "city", + "startDate.$date", + "endDate.$date", + "frequency.nombre", + "frequency.duree", + "frequency.frequence", + "missionDuration", + "contactFullName", + "contactEmail", + "files", + "message", + "createdAt.$date", + "updatedAt.$date", + "__v", + ]; + const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); -function anonymize(item) { item.message && (item.message = starify(item.message)); item.address && (item.address = generateAddress()); item.contactEmail && (item.contactEmail = generateRandomEmail()); @@ -9,9 +36,7 @@ function anonymize(item) { item.structureName && (item.structureName = generateRandomName()); item.files && (item.files = []); - const knownFields = ["message", "address", "contactEmail", "contactFullName", "structureName", "files"]; - - return anonymizeNewFields(item, knownFields, missionEquivalenceSchemaFields); + return item; } module.exports = anonymize; diff --git a/api/src/anonymization/referent.js b/api/src/anonymization/referent.js index 4aa1a69461..f167f25a59 100644 --- a/api/src/anonymization/referent.js +++ b/api/src/anonymization/referent.js @@ -1,7 +1,50 @@ const { generateRandomName, generateRandomEmail, generateNewPhoneNumber } = require("../utils/anonymise"); -const { referentSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); +const { anonymizeNonDeclaredFields } = require("../utils/anonymise-model-fields"); + +function anonymize(itemToAnonymize) { + const whitelist = [ + "_id.$oid", + "firstName", + "lastName", + "email", + "emailValidatedAt.$date", + "emailWaitingValidation", + "password", + "loginAttempts", + "token2FA", + "token2FAExpires.$date", + "attempts2FA", + "acceptCGU", + "lastLoginAt.$date", + "lastActivityAt.$date", + "lastLogoutAt.$date", + "passwordChangedAt.$date", + "registredAt.$date", + "nextLoginAttemptIn.$date", + "forgotPasswordResetToken", + "forgotPasswordResetExpires.$date", + "invitationToken", + "invitationExpires.$date", + "role", + "subRole", + "region", + "department", + "structureId", + "sessionPhase1Id", + "cohorts", + "cohortIds", + "cohesionCenterId", + "cohesionCenterName", + "phone", + "mobile", + "metadata._id.$oid", + "deletedAt.$date", + "createdAt.$date", + "updatedAt.$date", + "__v", + ]; + const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); -function anonymize(item) { item.phone && (item.phone = generateNewPhoneNumber()); item.mobile && (item.mobile = generateNewPhoneNumber()); item.email && (item.email = generateRandomEmail()); @@ -12,9 +55,7 @@ function anonymize(item) { item.forgotPasswordResetToken = ""; item.token2FA = ""; - const knownFields = ["phone", "mobile", "email", "firstName", "lastName", "invitationToken", "forgotPasswordResetToken", "token2FA"]; - - return anonymizeNewFields(item, knownFields, referentSchemaFields); + return item; } module.exports = anonymize; diff --git a/api/src/anonymization/sessionPhase1.js b/api/src/anonymization/sessionPhase1.js index 64dd47ad2b..9ad549ecb6 100644 --- a/api/src/anonymization/sessionPhase1.js +++ b/api/src/anonymization/sessionPhase1.js @@ -1,7 +1,49 @@ const { starify } = require("../utils/anonymise"); -const { sessionPhase1SchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); +const { anonymizeNonDeclaredFields } = require("../utils/anonymise-model-fields"); + +function anonymize(itemToAnonymize) { + const whitelist = [ + "_id.$oid", + "cohesionCenterId", + "cohort", + "cohortId", + "department", + "region", + "codeCentre", + "nameCentre", + "zipCentre", + "cityCentre", + "headCenterId", + "team.firstName", + "team.lastName", + "team.role", + "team.email", + "team.phone", + "waitingList", + "placesTotal", + "placesLeft", + "timeScheduleFiles._id", + "timeScheduleFiles.name", + "timeScheduleFiles.uploadedAt", + "timeScheduleFiles.size", + "timeScheduleFiles.mimetype", + "hasTimeSchedule", + "pedagoProjectFiles._id", + "pedagoProjectFiles.name", + "pedagoProjectFiles.uploadedAt", + "pedagoProjectFiles.size", + "pedagoProjectFiles.mimetype", + "hasPedagoProject", + "dateStart.$date", + "dateEnd.$date", + "status", + "sanitaryContactEmail", + "createdAt.$date", + "updatedAt.$date", + "__v", + ]; + const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); -function anonymize(item) { item.zipCentre && (item.zipCentre = starify(item.zipCentre)); item.codeCentre && (item.codeCentre = starify(item.codeCentre)); item.nameCentre && (item.nameCentre = starify(item.nameCentre)); @@ -16,9 +58,7 @@ function anonymize(item) { return member; })); - const knownFields = ["zipCentre", "codeCentre", "nameCentre", "cityCentre", "status", "team.firstName", "team.lastName", "team.email", "team.phone"]; - - return anonymizeNewFields(item, knownFields, sessionPhase1SchemaFields); + return item; } module.exports = anonymize; diff --git a/api/src/anonymization/structure.js b/api/src/anonymization/structure.js index eb0d3006e4..f10bc08801 100644 --- a/api/src/anonymization/structure.js +++ b/api/src/anonymization/structure.js @@ -1,7 +1,52 @@ const { generateAddress, generateRandomName, starify, generateRandomEmail, generateNewPhoneNumber } = require("../utils/anonymise"); -const { structureSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); +const { anonymizeNonDeclaredFields } = require("../utils/anonymise-model-fields"); + +function anonymize(itemToAnonymize) { + const whitelist = [ + "_id.$oid", + "name", + "siret", + "description", + "website", + "facebook", + "twitter", + "instagram", + "status", + "isNetwork", + "networkId", + "networkName", + "legalStatus", + "types", + "sousType", + "associationTypes", + "structurePubliqueType", + "structurePubliqueEtatType", + "structurePriveeType", + "address", + "zip", + "city", + "department", + "region", + "country", + "location.lon", + "location.lat", + "addressVerified", + "state", + "isMilitaryPreparation", + "isJvaStructure", + "jvaStructureId", + "jvaRawData", + "structureManager.firstName", + "structureManager.lastName", + "structureManager.mobile", + "structureManager.email", + "structureManager.role", + "createdAt.$date", + "updatedAt.$date", + "__v", + ]; + const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); -function anonymize(item) { item.name && (item.name = generateRandomName().toUpperCase()); item.siret && (item.siret = starify(item.siret)); item.address && (item.address = generateAddress()); @@ -16,22 +61,7 @@ function anonymize(item) { item.structureManager?.mobile && (item.structureManager.mobile = generateNewPhoneNumber()); item.structureManager?.email && (item.structureManager.email = generateRandomEmail()); - const knownFields = [ - "name", - "siret", - "address", - "website", - "description", - "twitter", - "facebook", - "instagram", - "structureManager.firstName", - "structureManager.lastName", - "structureManager.mobile", - "structureManager.email", - ]; - - return anonymizeNewFields(item, knownFields, structureSchemaFields); + return item; } module.exports = anonymize; diff --git a/api/src/anonymization/waitingList.js b/api/src/anonymization/waitingList.js index e8d09b05b2..fdd1712554 100644 --- a/api/src/anonymization/waitingList.js +++ b/api/src/anonymization/waitingList.js @@ -1,13 +1,15 @@ const { generateRandomEmail, generateBirthdate } = require("../utils/anonymise"); -const { waitingListSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); +const { anonymizeNonDeclaredFields } = require("../utils/anonymise-model-fields"); + +function anonymize(itemToAnonymize) { + const whitelist = ["_id.$oid", "zip", "mail", "birthdateAt", "createdAt.$date", "updatedAt.$date"]; + + const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); -function anonymize(item) { item.mail && (item.mail = generateRandomEmail()); item.birthdateAt && (item.birthdateAt = generateBirthdate()); - const knownFields = ["mail", "birthdateAt"]; - - return anonymizeNewFields(item, knownFields, waitingListSchemaFields); + return item; } module.exports = anonymize; diff --git a/api/src/anonymization/young.js b/api/src/anonymization/young.js index 68c57bf32d..372fdfa168 100644 --- a/api/src/anonymization/young.js +++ b/api/src/anonymization/young.js @@ -1,7 +1,330 @@ const { generateAddress, generateRandomName, generateRandomEmail, generateBirthdate, getYoungLocation, generateNewPhoneNumber, starify } = require("../utils/anonymise"); -const { youngSchemaFields, anonymizeNewFields } = require("../utils/anonymise-model-fields"); +const { anonymizeNonDeclaredFields } = require("../utils/anonymise-model-fields"); + +function anonymize(itemToAnonymize) { + const whitelist = [ + "_id.$oid", + "firstName", + "lastName", + "frenchNationality", + "birthCountry", + "birthCity", + "birthCityZip", + "email", + "emailVerified", + "newEmail", + "phone", + "phoneZone", + "gender", + "birthdateAt.$date", + "cohort", + "cohortId", + "originalCohort", + "originalCohortId", + "cohortChangeReason", + "cohortDetailedChangeReason", + "phase", + "status", + "statusPhase1", + "statusPhase1Tmp", + "statusPhase1Motif", + "statusPhase1MotifDetail", + "statusPhase2", + "statusPhase2updatedAt.$date", + "statusPhase2OpenedAt.$date", + "statusPhase2ValidatedAt.$date", + "statusPhase2Contract", + "statusPhase3", + "statusPhase3updatedAt.$date", + "statusPhase3ValidatedAt.$date", + "lastStatusAt.$date", + "withdrawnReason", + "withdrawnMessage", + "hasStartedReinscription", + "reinscriptionStep2023", + "inscriptionStep2023", + "inscriptionStep", + "inscriptionDoneDate.$date", + "cohesion2020Step", + "inscriptionCorrectionMessage", + "inscriptionRefusedMessage", + "historic", + "password", + "token2FA", + "token2FAExpires.$date", + "attempts2FA", + "tokenEmailValidation", + "tokenEmailValidationExpires.$date", + "attemptsEmailValidation", + "loginAttempts", + "lastLoginAt.$date", + "lastActivityAt.$date", + "lastLogoutAt.$date", + "passwordChangedAt.$date", + "nextLoginAttemptIn.$date", + "forgotPasswordResetToken", + "forgotPasswordResetExpires.$date", + "invitationToken", + "invitationExpires.$date", + "acceptCGU", + "acceptRI", + "cniFiles", + "cohesionStayPresence", + "presenceJDM", + "cohesionStayMedicalFileReceived", + "departInform", + "departSejourAt.$date", + "departSejourMotif", + "departSejourMotifComment", + "cohesionStayMedicalFileDownload", + "convocationFileDownload", + "classeId", + "etablissementId", + "source", + "sessionPhase1Id", + "sessionPhase1IdTmp", + "cohesionCenterId", + "ligneId", + "meetingPointId", + "deplacementPhase1Autonomous", + "transportInfoGivenByLocal", + "hasMeetingInformation", + "isTravelingByPlane", + "codeCenterTmp", + "busTmp", + "cohesionCenterName", + "cohesionCenterZip", + "cohesionCenterCity", + "autoAffectationPhase1ExpiresAt.$date", + "phase2ApplicationStatus", + "phase2ApplicationFilesType", + "phase2NumberHoursDone", + "phase2NumberHoursEstimated", + "phase3StructureName", + "phase3MissionDomain", + "phase3MissionDescription", + "phase3MissionStartAt.$date", + "phase3MissionEndAt.$date", + "phase3TutorFirstName", + "phase3TutorLastName", + "phase3TutorEmail", + "phase3TutorPhone", + "phase3TutorNote", + "phase3Token", + "address", + "coordinatesAccuracyLevel", + "complementAddress", + "zip", + "city", + "addressVerified", + "cityCode", + "populationDensity", + "isRegionRural", + "department", + "region", + "country", + "location.lat", + "location.lon", + "qpv", + "foreignAddress", + "foreignCity", + "foreignZip", + "foreignCountry", + "situation", + "grade", + "schoolCertification", + "schooled", + "schoolName", + "schoolNameOld", + "schoolType", + "schoolAddress", + "schoolComplementAdresse", + "schoolZip", + "schoolCity", + "schoolDepartment", + "schoolRegion", + "schoolCountry", + "schoolLocation.lat", + "schoolLocation.lon", + "schoolId", + "academy", + "employed", + "parent1Status", + "parent1FirstName", + "parent1LastName", + "parent1Email", + "parent1Phone", + "parent1PhoneZone", + "parent1OwnAddress", + "parent1Address", + "parent1coordinatesAccuracyLevel", + "parent1ComplementAddress", + "parent1Zip", + "parent1City", + "parent1CityCode", + "parent1Department", + "parent1Region", + "parent1Country", + "parent1Location.lat", + "parent1Location.lon", + "parent1FromFranceConnect", + "parent1Inscription2023Token", + "parent1DataVerified", + "parent1AddressVerified", + "parent1AllowCovidAutotest", + "parent1AllowImageRights", + "parent1ContactPreference", + "parent2Status", + "parent2FirstName", + "parent2LastName", + "parent2Email", + "parent2Phone", + "parent2PhoneZone", + "parent2OwnAddress", + "parent2Address", + "parent2coordinatesAccuracyLevel", + "parent2ComplementAddress", + "parent2Zip", + "parent2City", + "parent2CityCode", + "parent2Department", + "parent2Region", + "parent2Country", + "parent2Location.lat", + "parent2Location.lon", + "parent2FromFranceConnect", + "parent2Inscription2023Token", + "parent2AllowImageRights", + "parent2AllowImageRightsReset", + "parent2ContactPreference", + "hostLastName", + "hostFirstName", + "hostRelationship", + "hostCity", + "hostZip", + "hostAddress", + "hostDepartment", + "hostRegion", + "handicap", + "allergies", + "handicapInSameDepartment", + "reducedMobilityAccess", + "ppsBeneficiary", + "paiBeneficiary", + "medicosocialStructure", + "medicosocialStructureName", + "medicosocialStructureAddress", + "medicosocialStructureComplementAddress", + "medicosocialStructureZip", + "medicosocialStructureCity", + "medicosocialStructureDepartment", + "medicosocialStructureRegion", + "medicosocialStructureLocation.lat", + "medicosocialStructureLocation.lon", + "engagedStructure", + "sameSchoolCLE", + "specificAmenagment", + "specificAmenagmentType", + "highSkilledActivity", + "highSkilledActivityInSameDepartment", + "highSkilledActivityType", + "highSkilledActivityProofFiles", + "parentAllowSNU", + "parent1AllowSNU", + "parent2AllowSNU", + "parent1ValidationDate.$date", + "parent2ValidationDate.$date", + "parent2RejectSNUComment", + "dataProcessingConsentmentFiles", + "parentConsentment", + "parentConsentmentFiles", + "parentConsentmentFilesCompliant", + "parentConsentmentFilesCompliantInfo", + "consentment", + "imageRight", + "imageRightFiles", + "imageRightFilesStatus", + "rulesFilesStatus", + "psc1Info", + "imageRightFilesComment", + "autoTestPCR", + "autoTestPCRFiles", + "autoTestPCRFilesStatus", + "autoTestPCRFilesComment", + "rulesYoung", + "rulesParent1", + "rulesParent2", + "rulesFiles", + "informationAccuracy", + "aknowledgmentTerminaleSessionAvailability", + "parentStatementOfHonorInvalidId", + "jdc", + "motivations", + "domains", + "professionnalProject", + "professionnalProjectPrecision", + "period", + "periodRanking", + "mobilityNearSchool", + "mobilityNearHome", + "mobilityNearRelative", + "mobilityNearRelativeName", + "mobilityNearRelativeAddress", + "mobilityNearRelativeZip", + "mobilityNearRelativeCity", + "mobilityTransport", + "mobilityTransportOther", + "missionFormat", + "engaged", + "engagedDescription", + "desiredLocation", + "militaryPreparationFilesIdentity", + "militaryPreparationFilesCensus", + "militaryPreparationFilesAuthorization", + "militaryPreparationFilesCertificate", + "statusMilitaryPreparationFiles", + "militaryPreparationCorrectionMessage", + "files.cniFiles", + "files.highSkilledActivityProofFiles", + "files.dataProcessingConsentmentFiles", + "files.parentConsentmentFiles", + "files.imageRightFiles", + "files.autoTestPCRFiles", + "files.rulesFiles", + "files.militaryPreparationFilesIdentity", + "files.militaryPreparationFilesCensus", + "files.militaryPreparationFilesAuthorization", + "files.militaryPreparationFilesCertificate", + "latestCNIFileExpirationDate.$date", + "CNIFileNotValidOnStart", + "latestCNIFileCategory", + "missionsInMail.missionId", + "missionsInMail.date.$date", + "youngPhase1Agreement", + "status_equivalence", + "correctionRequests", + "notes", + "hasNotes", + "defenseInterest", + "defenseTypeInterest", + "defenseDomainInterest", + "defenseMotivationInterest", + "securityInterest", + "securityDomainInterest", + "solidarityInterest", + "healthInterest", + "educationInterest", + "cultureInterest", + "sportInterest", + "environmentInterest", + "citizenshipInterest", + "deletedAt.$date", + "createdAt.$date", + "updatedAt.$date", + "__v", + ]; + const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); -function anonymize(item) { item.email && (item.email = generateRandomEmail()); item.newEmail && (item.newEmail = generateRandomEmail()); item.parent1Email && (item.parent1Email = generateRandomEmail()); @@ -71,59 +394,7 @@ function anonymize(item) { item.parent1Inscription2023Token = ""; item.parent2Inscription2023Token = ""; - const knownFields = [ - "email", - "newEmail", - "parent1Email", - "parent2Email", - "firstName", - "lastName", - "parent1FirstName", - "parent1LastName", - "parent2FirstName", - "parent2LastName", - "historic", - "phone", - "parent1Phone", - "parent2Phone", - "address", - "parent1Address", - "parent2Address", - "birthdateAt", - "engagedDescription", - "motivations", - "parentConsentmentFilesCompliantInfo", - "withdrawnReason", - "withdrawnMessage", - "correctionRequests.message", - "correctionRequests.reason", - "notes.note", - "notes.referent.firstName", - "notes.referent.lastName", - "location", - "cniFiles", - "highSkilledActivityProofFiles", - "dataProcessingConsentmentFiles", - "parentConsentmentFiles", - "imageRightFiles", - "autoTestPCRFiles", - "rulesFiles", - "militaryPreparationFilesIdentity", - "militaryPreparationFilesCensus", - "militaryPreparationFilesAuthorization", - "militaryPreparationFilesCertificate", - "militaryPreparationCorrectionMessage", - "files", - "token2FA", - "tokenEmailValidation", - "forgotPasswordResetToken", - "invitationToken", - "phase3Token", - "parent1Inscription2023Token", - "parent2Inscription2023Token", - ]; - - return anonymizeNewFields(item, knownFields, youngSchemaFields); + return item; } module.exports = anonymize; diff --git a/api/src/utils/anonymise-model-fields.js b/api/src/utils/anonymise-model-fields.js index ff840ebff7..9f83364ded 100644 --- a/api/src/utils/anonymise-model-fields.js +++ b/api/src/utils/anonymise-model-fields.js @@ -1,792 +1,3 @@ -const applicationSchemaFields = [ - "_id.$oid", - "apiEngagementId", - "youngId", - "youngFirstName", - "youngLastName", - "youngEmail", - "youngBirthdateAt", - "youngCity", - "youngDepartment", - "youngCohort", - "cohortId", - "missionId", - "isJvaMission", - "missionName", - "missionDepartment", - "missionRegion", - "missionDuration", - "structureId", - "tutorId", - "contractId", - "contractStatus", - "tutorName", - "priority", - "hidden", - "status", - "statusComment", - "contractAvenantFiles", - "justificatifsFiles", - "feedBackExperienceFiles", - "othersFiles", - "filesType", - "createdAt.$date", - "updatedAt.$date", -]; - -const cohesionCenterSchemaFields = [ - "_id.$oid", - "name", - "code2022", - "address", - "city", - "zip", - "department", - "region", - "addressVerified", - "placesTotal", - "pmr", - "cohorts", - "cohortIds", - "academy", - "typology", - "domain", - "complement", - "centerDesignation", - "placesLeft", - "outfitDelivered", - "observations", - "waitingList", - "COR", - "code", - "country", - "departmentCode", - "sessionStatus", - "createdAt.$date", - "updatedAt.$date", - "__v", -]; - -const contractSchemaFields = [ - "_id.$oid", - "youngId", - "structureId", - "applicationId", - "missionId", - "tutorId", - "isYoungAdult", - "parent1Token", - "projectManagerToken", - "structureManagerToken", - "parent2Token", - "youngContractToken", - "parent1Status", - "projectManagerStatus", - "structureManagerStatus", - "parent2Status", - "youngContractStatus", - "parent1ValidationDate.$date", - "projectManagerValidationDate.$date", - "structureManagerValidationDate.$date", - "parent2ValidationDate.$date", - "youngContractValidationDate", - "invitationSent", - "youngFirstName", - "youngLastName", - "youngBirthdate", - "youngAddress", - "youngCity", - "youngDepartment", - "youngEmail", - "youngPhone", - "parent1FirstName", - "parent1LastName", - "parent1Address", - "parent1City", - "parent1Department", - "parent1Phone", - "parent1Email", - "parent2FirstName", - "parent2LastName", - "parent2Address", - "parent2City", - "parent2Department", - "parent2Phone", - "parent2Email", - "missionName", - "missionObjective", - "missionAction", - "missionStartAt", - "missionEndAt", - "missionAddress", - "missionCity", - "missionZip", - "missionDuration", - "missionFrequence", - "date", - "projectManagerFirstName", - "projectManagerLastName", - "projectManagerRole", - "projectManagerEmail", - "structureManagerFirstName", - "structureManagerLastName", - "structureManagerRole", - "structureManagerEmail", - "tutorFirstName", - "tutorLastName", - "tutorRole", - "tutorEmail", - "structureSiret", - "structureName", - "createdAt.$date", - "updatedAt.$date", - "__v", -]; - -const departmentServiceSchemaFields = [ - "_id.$oid", - "contacts.cohort", - "contacts.cohortId", - "contacts.contactName", - "contacts.contactPhone", - "contacts.contactMail", - "department", - "region", - "directionName", - "serviceName", - "serviceNumber", - "address", - "complementAddress", - "zip", - "city", - "description", - "contactName", - "contactPhone", - "contactMail", - "representantEtat.firstName", - "representantEtat.lastName", - "representantEtat.mobile", - "representantEtat.email", - "representantEtat.role", - "createdAt.$date", - "updatedAt.$date", - "__v", -]; - -const meetingPointSchemaFields = [ - "_id.$oid", - "isValid", - "cohort", - "cohortId", - "busId", - "busExcelId", - "centerId", - "centerCode", - "departureAddress", - "departureZip", - "departureCity", - "departureDepartment", - "departureRegion", - "hideDepartmentInConvocation", - "departureAt.$date", - "departureAtString", - "realDepartureAtString", - "returnAt.$date", - "returnAtString", - "realReturnAtString", - "createdAt.$date", - "updatedAt.$date", - "deletedAt.$date", - "__v", -]; - -const missionSchemaFields = [ - "_id.$oid", - "name", - "domains", - "mainDomain", - "startAt.$date", - "endAt.$date", - "duration", - "format", - "frequence", - "period", - "subPeriod", - "placesTotal", - "placesLeft", - "pendingApplications", - "actions", - "description", - "justifications", - "contraintes", - "structureId", - "structureName", - "status", - "visibility", - "statusComment", - "hebergement", - "hebergementPayant", - "tutorId", - "tutorName", - "address", - "zip", - "city", - "department", - "region", - "country", - "location.lat", - "location.lon", - "addressVerified", - "remote", - "isMilitaryPreparation", - "createdAt.$date", - "updatedAt.$date", - "lastSyncAt.$date", - "isJvaMission", - "jvaMissionId", - "apiEngagementId", - "jvaRawData", - "applicationStatus", - "placesStatus", - "__v", -]; - -const missionEquivalenceSchemaFields = [ - "_id.$oid", - "youngId", - "status", - "type", - "desc", - "sousType", - "structureName", - "address", - "zip", - "city", - "startDate.$date", - "endDate.$date", - "frequency.nombre", - "frequency.duree", - "frequency.frequence", - "missionDuration", - "contactFullName", - "contactEmail", - "files", - "message", - "createdAt.$date", - "updatedAt.$date", - "__v", -]; - -const referentSchemaFields = [ - "_id.$oid", - "firstName", - "lastName", - "email", - "emailValidatedAt.$date", - "emailWaitingValidation", - "password", - "loginAttempts", - "token2FA", - "token2FAExpires.$date", - "attempts2FA", - "acceptCGU", - "lastLoginAt.$date", - "lastActivityAt.$date", - "lastLogoutAt.$date", - "passwordChangedAt.$date", - "registredAt.$date", - "nextLoginAttemptIn.$date", - "forgotPasswordResetToken", - "forgotPasswordResetExpires.$date", - "invitationToken", - "invitationExpires.$date", - "role", - "subRole", - "region", - "department", - "structureId", - "sessionPhase1Id", - "cohorts", - "cohortIds", - "cohesionCenterId", - "cohesionCenterName", - "phone", - "mobile", - "metadata._id.$oid", - "deletedAt.$date", - "createdAt.$date", - "updatedAt.$date", - "__v", -]; - -const sessionPhase1SchemaFields = [ - "_id.$oid", - "cohesionCenterId", - "cohort", - "cohortId", - "department", - "region", - "codeCentre", - "nameCentre", - "zipCentre", - "cityCentre", - "headCenterId", - "team.firstName", - "team.lastName", - "team.role", - "team.email", - "team.phone", - "waitingList", - "placesTotal", - "placesLeft", - "timeScheduleFiles._id", - "timeScheduleFiles.name", - "timeScheduleFiles.uploadedAt", - "timeScheduleFiles.size", - "timeScheduleFiles.mimetype", - "hasTimeSchedule", - "pedagoProjectFiles._id", - "pedagoProjectFiles.name", - "pedagoProjectFiles.uploadedAt", - "pedagoProjectFiles.size", - "pedagoProjectFiles.mimetype", - "hasPedagoProject", - "dateStart.$date", - "dateEnd.$date", - "status", - "sanitaryContactEmail", - "createdAt.$date", - "updatedAt.$date", - "__v", -]; - -const structureSchemaFields = [ - "_id.$oid", - "name", - "siret", - "description", - "website", - "facebook", - "twitter", - "instagram", - "status", - "isNetwork", - "networkId", - "networkName", - "legalStatus", - "types", - "sousType", - "associationTypes", - "structurePubliqueType", - "structurePubliqueEtatType", - "structurePriveeType", - "address", - "zip", - "city", - "department", - "region", - "country", - "location.lon", - "location.lat", - "addressVerified", - "state", - "isMilitaryPreparation", - "isJvaStructure", - "jvaStructureId", - "jvaRawData", - "structureManager.firstName", - "structureManager.lastName", - "structureManager.mobile", - "structureManager.email", - "structureManager.role", - "createdAt.$date", - "updatedAt.$date", - "__v", -]; - -const waitingListSchemaFields = ["_id.$oid", "zip", "mail", "birthdateAt", "createdAt.$date", "updatedAt.$date"]; - -const youngSchemaFields = [ - "_id.$oid", - "firstName", - "lastName", - "frenchNationality", - "birthCountry", - "birthCity", - "birthCityZip", - "email", - "emailVerified", - "newEmail", - "phone", - "phoneZone", - "gender", - "birthdateAt.$date", - "cohort", - "cohortId", - "originalCohort", - "originalCohortId", - "cohortChangeReason", - "cohortDetailedChangeReason", - "phase", - "status", - "statusPhase1", - "statusPhase1Tmp", - "statusPhase1Motif", - "statusPhase1MotifDetail", - "statusPhase2", - "statusPhase2updatedAt.$date", - "statusPhase2OpenedAt.$date", - "statusPhase2ValidatedAt.$date", - "statusPhase2Contract", - "statusPhase3", - "statusPhase3updatedAt.$date", - "statusPhase3ValidatedAt.$date", - "lastStatusAt.$date", - "withdrawnReason", - "withdrawnMessage", - "hasStartedReinscription", - "reinscriptionStep2023", - "inscriptionStep2023", - "inscriptionStep", - "inscriptionDoneDate.$date", - "cohesion2020Step", - "inscriptionCorrectionMessage", - "inscriptionRefusedMessage", - "historic", - "password", - "token2FA", - "token2FAExpires.$date", - "attempts2FA", - "tokenEmailValidation", - "tokenEmailValidationExpires.$date", - "attemptsEmailValidation", - "loginAttempts", - "lastLoginAt.$date", - "lastActivityAt.$date", - "lastLogoutAt.$date", - "passwordChangedAt.$date", - "nextLoginAttemptIn.$date", - "forgotPasswordResetToken", - "forgotPasswordResetExpires.$date", - "invitationToken", - "invitationExpires.$date", - "acceptCGU", - "acceptRI", - "cniFiles", - "cohesionStayPresence", - "presenceJDM", - "cohesionStayMedicalFileReceived", - "departInform", - "departSejourAt.$date", - "departSejourMotif", - "departSejourMotifComment", - "cohesionStayMedicalFileDownload", - "convocationFileDownload", - "classeId", - "etablissementId", - "source", - "sessionPhase1Id", - "sessionPhase1IdTmp", - "cohesionCenterId", - "ligneId", - "meetingPointId", - "deplacementPhase1Autonomous", - "transportInfoGivenByLocal", - "hasMeetingInformation", - "isTravelingByPlane", - "codeCenterTmp", - "busTmp", - "cohesionCenterName", - "cohesionCenterZip", - "cohesionCenterCity", - "autoAffectationPhase1ExpiresAt.$date", - "phase2ApplicationStatus", - "phase2ApplicationFilesType", - "phase2NumberHoursDone", - "phase2NumberHoursEstimated", - "phase3StructureName", - "phase3MissionDomain", - "phase3MissionDescription", - "phase3MissionStartAt.$date", - "phase3MissionEndAt.$date", - "phase3TutorFirstName", - "phase3TutorLastName", - "phase3TutorEmail", - "phase3TutorPhone", - "phase3TutorNote", - "phase3Token", - "address", - "coordinatesAccuracyLevel", - "complementAddress", - "zip", - "city", - "addressVerified", - "cityCode", - "populationDensity", - "isRegionRural", - "department", - "region", - "country", - "location.lat", - "location.lon", - "qpv", - "foreignAddress", - "foreignCity", - "foreignZip", - "foreignCountry", - "situation", - "grade", - "schoolCertification", - "schooled", - "schoolName", - "schoolNameOld", - "schoolType", - "schoolAddress", - "schoolComplementAdresse", - "schoolZip", - "schoolCity", - "schoolDepartment", - "schoolRegion", - "schoolCountry", - "schoolLocation.lat", - "schoolLocation.lon", - "schoolId", - "academy", - "employed", - "parent1Status", - "parent1FirstName", - "parent1LastName", - "parent1Email", - "parent1Phone", - "parent1PhoneZone", - "parent1OwnAddress", - "parent1Address", - "parent1coordinatesAccuracyLevel", - "parent1ComplementAddress", - "parent1Zip", - "parent1City", - "parent1CityCode", - "parent1Department", - "parent1Region", - "parent1Country", - "parent1Location.lat", - "parent1Location.lon", - "parent1FromFranceConnect", - "parent1Inscription2023Token", - "parent1DataVerified", - "parent1AddressVerified", - "parent1AllowCovidAutotest", - "parent1AllowImageRights", - "parent1ContactPreference", - "parent2Status", - "parent2FirstName", - "parent2LastName", - "parent2Email", - "parent2Phone", - "parent2PhoneZone", - "parent2OwnAddress", - "parent2Address", - "parent2coordinatesAccuracyLevel", - "parent2ComplementAddress", - "parent2Zip", - "parent2City", - "parent2CityCode", - "parent2Department", - "parent2Region", - "parent2Country", - "parent2Location.lat", - "parent2Location.lon", - "parent2FromFranceConnect", - "parent2Inscription2023Token", - "parent2AllowImageRights", - "parent2AllowImageRightsReset", - "parent2ContactPreference", - "hostLastName", - "hostFirstName", - "hostRelationship", - "hostCity", - "hostZip", - "hostAddress", - "hostDepartment", - "hostRegion", - "handicap", - "allergies", - "handicapInSameDepartment", - "reducedMobilityAccess", - "ppsBeneficiary", - "paiBeneficiary", - "medicosocialStructure", - "medicosocialStructureName", - "medicosocialStructureAddress", - "medicosocialStructureComplementAddress", - "medicosocialStructureZip", - "medicosocialStructureCity", - "medicosocialStructureDepartment", - "medicosocialStructureRegion", - "medicosocialStructureLocation.lat", - "medicosocialStructureLocation.lon", - "engagedStructure", - "sameSchoolCLE", - "specificAmenagment", - "specificAmenagmentType", - "highSkilledActivity", - "highSkilledActivityInSameDepartment", - "highSkilledActivityType", - "highSkilledActivityProofFiles", - "parentAllowSNU", - "parent1AllowSNU", - "parent2AllowSNU", - "parent1ValidationDate.$date", - "parent2ValidationDate.$date", - "parent2RejectSNUComment", - "dataProcessingConsentmentFiles", - "parentConsentment", - "parentConsentmentFiles", - "parentConsentmentFilesCompliant", - "parentConsentmentFilesCompliantInfo", - "consentment", - "imageRight", - "imageRightFiles", - "imageRightFilesStatus", - "rulesFilesStatus", - "psc1Info", - "imageRightFilesComment", - "autoTestPCR", - "autoTestPCRFiles", - "autoTestPCRFilesStatus", - "autoTestPCRFilesComment", - "rulesYoung", - "rulesParent1", - "rulesParent2", - "rulesFiles", - "informationAccuracy", - "aknowledgmentTerminaleSessionAvailability", - "parentStatementOfHonorInvalidId", - "jdc", - "motivations", - "domains", - "professionnalProject", - "professionnalProjectPrecision", - "period", - "periodRanking", - "mobilityNearSchool", - "mobilityNearHome", - "mobilityNearRelative", - "mobilityNearRelativeName", - "mobilityNearRelativeAddress", - "mobilityNearRelativeZip", - "mobilityNearRelativeCity", - "mobilityTransport", - "mobilityTransportOther", - "missionFormat", - "engaged", - "engagedDescription", - "desiredLocation", - "militaryPreparationFilesIdentity", - "militaryPreparationFilesCensus", - "militaryPreparationFilesAuthorization", - "militaryPreparationFilesCertificate", - "statusMilitaryPreparationFiles", - "militaryPreparationCorrectionMessage", - "files.cniFiles", - "files.highSkilledActivityProofFiles", - "files.dataProcessingConsentmentFiles", - "files.parentConsentmentFiles", - "files.imageRightFiles", - "files.autoTestPCRFiles", - "files.rulesFiles", - "files.militaryPreparationFilesIdentity", - "files.militaryPreparationFilesCensus", - "files.militaryPreparationFilesAuthorization", - "files.militaryPreparationFilesCertificate", - "latestCNIFileExpirationDate.$date", - "CNIFileNotValidOnStart", - "latestCNIFileCategory", - "missionsInMail.missionId", - "missionsInMail.date.$date", - "youngPhase1Agreement", - "status_equivalence", - "correctionRequests", - "notes", - "hasNotes", - "defenseInterest", - "defenseTypeInterest", - "defenseDomainInterest", - "defenseMotivationInterest", - "securityInterest", - "securityDomainInterest", - "solidarityInterest", - "healthInterest", - "educationInterest", - "cultureInterest", - "sportInterest", - "environmentInterest", - "citizenshipInterest", - "deletedAt.$date", - "createdAt.$date", - "updatedAt.$date", - "__v", -]; - -const ligneBusSchemaFields = [ - "_id.$oid", - "cohort", - "cohortId", - "busId", - "departuredDate.$date", - "returnDate.$date", - "youngCapacity", - "totalCapacity", - "followerCapacity", - "youngSeatsTaken", - "travelTime", - "lunchBreak", - "lunchBreakReturn", - "sessionId", - "centerId", - "centerArrivalTime", - "centerDepartureTime", - "classeId", - "meetingPointsIds", - "team", - "delayedForth", - "delayedBack", - "mergedBusIds", - "createdAt.$date", - "updatedAt.$date", - "deletedAt.$date", - "__v", -]; - -const modificationBusSchemaFields = [ - "_id.$oid", - "cohort", - "cohortId", - "lineId", - "lineName", - "requestMessage", - "requestUserId", - "requestUserName", - "requestUserRole", - "tagIds", - "status", - "statusUserId", - "statusUserName", - "statusDate.$date", - "opinion", - "opinionUserId", - "opinionUserName", - "opinionDate.$date", - "messages.message", - "messages.userId", - "messages.userName", - "messages.date.$date", - "createdAt.$date", - "updatedAt.$date", - "deletedAt.$date", - "__v", -]; - function getNestedValue(obj, path) { return path.split(".").reduce((acc, part) => acc && acc[part], obj); } @@ -811,17 +22,19 @@ function getAllPaths(obj, parentPath = "") { return paths; } -function anonymizeNewFields(item, knownFields, schemaFields) { +function anonymizeNonDeclaredFields(item, whitelist) { const allPaths = getAllPaths(item); for (const path of allPaths) { - if (!knownFields.includes(path) && !schemaFields.includes(path)) { + if (!whitelist.includes(path)) { const value = getNestedValue(item, path); + // Si le champ est un tableau d'objets, on le laisse intact if (Array.isArray(value) && value.length > 0 && typeof value[0] === "object") { continue; } + // Anonymisation des champs non listés dans la whitelist if (value !== undefined) { if (Array.isArray(value)) { setNestedValue(item, path, []); @@ -839,19 +52,5 @@ function anonymizeNewFields(item, knownFields, schemaFields) { } module.exports = { - applicationSchemaFields, - cohesionCenterSchemaFields, - contractSchemaFields, - departmentServiceSchemaFields, - meetingPointSchemaFields, - missionSchemaFields, - missionEquivalenceSchemaFields, - referentSchemaFields, - sessionPhase1SchemaFields, - structureSchemaFields, - waitingListSchemaFields, - youngSchemaFields, - ligneBusSchemaFields, - modificationBusSchemaFields, - anonymizeNewFields, + anonymizeNonDeclaredFields, }; From 638ef74a90cc8c5a7f9576f512a06f5383a7670b Mon Sep 17 00:00:00 2001 From: Michael Garcia Date: Mon, 30 Sep 2024 15:27:41 +0200 Subject: [PATCH 05/14] up --- api/src/anonymization/application.js | 1 + api/src/anonymization/waitingList.js | 2 +- api/src/utils/anonymise-model-fields.js | 1 - 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/anonymization/application.js b/api/src/anonymization/application.js index fd5cefb82e..ae6bd930db 100644 --- a/api/src/anonymization/application.js +++ b/api/src/anonymization/application.js @@ -36,6 +36,7 @@ function anonymize(itemToAnonymize) { "filesType", "createdAt.$date", "updatedAt.$date", + "__v", ]; const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); diff --git a/api/src/anonymization/waitingList.js b/api/src/anonymization/waitingList.js index fdd1712554..eea2ad1eab 100644 --- a/api/src/anonymization/waitingList.js +++ b/api/src/anonymization/waitingList.js @@ -2,7 +2,7 @@ const { generateRandomEmail, generateBirthdate } = require("../utils/anonymise") const { anonymizeNonDeclaredFields } = require("../utils/anonymise-model-fields"); function anonymize(itemToAnonymize) { - const whitelist = ["_id.$oid", "zip", "mail", "birthdateAt", "createdAt.$date", "updatedAt.$date"]; + const whitelist = ["_id.$oid", "zip", "mail", "birthdateAt", "createdAt.$date", "updatedAt.$date", "__v"]; const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); diff --git a/api/src/utils/anonymise-model-fields.js b/api/src/utils/anonymise-model-fields.js index 9f83364ded..90305d5888 100644 --- a/api/src/utils/anonymise-model-fields.js +++ b/api/src/utils/anonymise-model-fields.js @@ -47,7 +47,6 @@ function anonymizeNonDeclaredFields(item, whitelist) { } } - console.log("Anonymized Item:", item); return item; } From 9e31ba4b3c5b51d263c648f34c4c73915d83ca14 Mon Sep 17 00:00:00 2001 From: Michael Garcia Date: Tue, 1 Oct 2024 15:41:04 +0200 Subject: [PATCH 06/14] up --- api/package.json | 2 +- package-lock.json | 33 +++++++++++++++++---------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/api/package.json b/api/package.json index 1769dcdb1e..721e9c6a07 100644 --- a/api/package.json +++ b/api/package.json @@ -45,7 +45,7 @@ "aws-sdk": "^2.1248.0", "bcryptjs": "^2.4.3", "body-parser": "^1.20.3", - "bson": "^6.7.0", + "bson": "^4.7.2", "bullmq": "^5.10.0", "config": "^3.3.11", "cookie-parser": "^1.4.6", diff --git a/package-lock.json b/package-lock.json index a27570390e..6ed9d9ff0d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -852,7 +852,7 @@ "aws-sdk": "^2.1248.0", "bcryptjs": "^2.4.3", "body-parser": "^1.20.3", - "bson": "^6.7.0", + "bson": "^4.7.2", "bullmq": "^5.10.0", "config": "^3.3.11", "cookie-parser": "^1.4.6", @@ -1086,11 +1086,14 @@ } }, "api/node_modules/bson": { - "version": "6.7.0", - "resolved": "https://registry.npmjs.org/bson/-/bson-6.7.0.tgz", - "integrity": "sha512-w2IquM5mYzYZv6rs3uN2DZTOBe2a0zXLj53TGDqwF4l6Sz/XsISrisXOJihArF9+BZ6Cq/GjVht7Sjfmri7ytQ==", + "version": "4.7.2", + "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.2.tgz", + "integrity": "sha512-Ry9wCtIZ5kGqkJoi6aD8KjxFZEx78guTQDnpXWiNthsxzrxAK/i8E6pCHAIZTbaEFWcOCvbecMukfK7XUvyLpQ==", + "dependencies": { + "buffer": "^5.6.0" + }, "engines": { - "node": ">=16.20.1" + "node": ">=6.9.0" } }, "api/node_modules/buffer": { @@ -1381,6 +1384,15 @@ } } }, + "api/node_modules/mongodb/node_modules/bson": { + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-6.8.0.tgz", + "integrity": "sha512-iOJg8pr7wq2tg/zSlCCHMi3hMm5JTOxLTagf3zxhcenHsFp+c6uOs6K7W5UE7A4QIJGtqh/ZovFNMP4mOPJynQ==", + "peer": true, + "engines": { + "node": ">=16.20.1" + } + }, "api/node_modules/mongoose": { "version": "6.13.0", "resolved": "https://registry.npmjs.org/mongoose/-/mongoose-6.13.0.tgz", @@ -1402,17 +1414,6 @@ "url": "https://opencollective.com/mongoose" } }, - "api/node_modules/mongoose/node_modules/bson": { - "version": "4.7.2", - "resolved": "https://registry.npmjs.org/bson/-/bson-4.7.2.tgz", - "integrity": "sha512-Ry9wCtIZ5kGqkJoi6aD8KjxFZEx78guTQDnpXWiNthsxzrxAK/i8E6pCHAIZTbaEFWcOCvbecMukfK7XUvyLpQ==", - "dependencies": { - "buffer": "^5.6.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, "api/node_modules/mongoose/node_modules/mongodb": { "version": "4.17.2", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-4.17.2.tgz", From 9430c3c595260cb9f1f0445b3c63735c4841c7a6 Mon Sep 17 00:00:00 2001 From: Michael Garcia Date: Tue, 1 Oct 2024 15:44:41 +0200 Subject: [PATCH 07/14] up --- api/src/utils/anonymise-model-fields.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/src/utils/anonymise-model-fields.js b/api/src/utils/anonymise-model-fields.js index 90305d5888..22eb308ebf 100644 --- a/api/src/utils/anonymise-model-fields.js +++ b/api/src/utils/anonymise-model-fields.js @@ -29,12 +29,10 @@ function anonymizeNonDeclaredFields(item, whitelist) { if (!whitelist.includes(path)) { const value = getNestedValue(item, path); - // Si le champ est un tableau d'objets, on le laisse intact if (Array.isArray(value) && value.length > 0 && typeof value[0] === "object") { continue; } - // Anonymisation des champs non listés dans la whitelist if (value !== undefined) { if (Array.isArray(value)) { setNestedValue(item, path, []); From 4a8016b2b8338e95b3bc40c9a75bbd8bedc6eaab Mon Sep 17 00:00:00 2001 From: Michael Garcia Date: Wed, 2 Oct 2024 16:42:26 +0200 Subject: [PATCH 08/14] up --- api/src/utils/anonymise-model-fields.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/api/src/utils/anonymise-model-fields.js b/api/src/utils/anonymise-model-fields.js index 22eb308ebf..eaf035b085 100644 --- a/api/src/utils/anonymise-model-fields.js +++ b/api/src/utils/anonymise-model-fields.js @@ -29,7 +29,14 @@ function anonymizeNonDeclaredFields(item, whitelist) { if (!whitelist.includes(path)) { const value = getNestedValue(item, path); - if (Array.isArray(value) && value.length > 0 && typeof value[0] === "object") { + if (Array.isArray(value) && value.length > 0) { + if (typeof value[0] === "object") { + value.forEach((element, index) => { + value[index] = anonymizeNonDeclaredFields(element, whitelist); + }); + } else { + setNestedValue(item, path, []); + } continue; } From bcb58f3a8ac187fd3d02974dc10bd0a3b8d203b7 Mon Sep 17 00:00:00 2001 From: Michael Garcia Date: Fri, 4 Oct 2024 15:53:49 +0200 Subject: [PATCH 09/14] up --- api/src/utils/anonymise-model-fields.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/api/src/utils/anonymise-model-fields.js b/api/src/utils/anonymise-model-fields.js index eaf035b085..5f59f26a39 100644 --- a/api/src/utils/anonymise-model-fields.js +++ b/api/src/utils/anonymise-model-fields.js @@ -1,7 +1,9 @@ +// Récupère la valeur d'un champ imbriqué dans un objet en suivant un chemin donné (ex: schoolLocation.lat) function getNestedValue(obj, path) { return path.split(".").reduce((acc, part) => acc && acc[part], obj); } +// Définit une valeur à un champ imbriqué dans un objet en suivant un chemin donné (ex: schoolLocation.lat) function setNestedValue(obj, path, value) { const keys = path.split("."); const lastKey = keys.pop(); @@ -9,6 +11,7 @@ function setNestedValue(obj, path, value) { deepObj[lastKey] = value; } +// Génère tous les chemins de propriétés possibles dans un objet, y compris les objets imbriqués function getAllPaths(obj, parentPath = "") { let paths = []; for (let key in obj) { @@ -22,6 +25,7 @@ function getAllPaths(obj, parentPath = "") { return paths; } +// Anonymise tous les champs d'un objet qui ne sont pas dans la whitelist function anonymizeNonDeclaredFields(item, whitelist) { const allPaths = getAllPaths(item); @@ -30,6 +34,7 @@ function anonymizeNonDeclaredFields(item, whitelist) { const value = getNestedValue(item, path); if (Array.isArray(value) && value.length > 0) { + // Si le tableau contient des objets, anonymise chaque objet if (typeof value[0] === "object") { value.forEach((element, index) => { value[index] = anonymizeNonDeclaredFields(element, whitelist); @@ -43,6 +48,8 @@ function anonymizeNonDeclaredFields(item, whitelist) { if (value !== undefined) { if (Array.isArray(value)) { setNestedValue(item, path, []); + } else if (path.includes("$date")) { + setNestedValue(item, path, new Date()); } else if (typeof value === "string") { setNestedValue(item, path, ""); } else { From ec8a144fc78ba7b5290b521b8815aa19436e10ad Mon Sep 17 00:00:00 2001 From: Michael Garcia Date: Fri, 4 Oct 2024 15:56:45 +0200 Subject: [PATCH 10/14] modification pour pouvoir test les jeunes --- api/src/anonymization/young.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/src/anonymization/young.js b/api/src/anonymization/young.js index 372fdfa168..f2746e821e 100644 --- a/api/src/anonymization/young.js +++ b/api/src/anonymization/young.js @@ -1,5 +1,6 @@ const { generateAddress, generateRandomName, generateRandomEmail, generateBirthdate, getYoungLocation, generateNewPhoneNumber, starify } = require("../utils/anonymise"); const { anonymizeNonDeclaredFields } = require("../utils/anonymise-model-fields"); +const crypto = require("crypto"); function anonymize(itemToAnonymize) { const whitelist = [ @@ -391,8 +392,8 @@ function anonymize(itemToAnonymize) { item.forgotPasswordResetToken = ""; item.invitationToken = ""; item.phase3Token = ""; - item.parent1Inscription2023Token = ""; - item.parent2Inscription2023Token = ""; + item.parent1Inscription2023Token = crypto.randomBytes(20).toString("hex"); + item.parent2Inscription2023Token = crypto.randomBytes(20).toString("hex"); return item; } From c7136372848bfd3ca8f5d29eec3f581bf96c04d0 Mon Sep 17 00:00:00 2001 From: Michael Garcia Date: Thu, 10 Oct 2024 20:05:45 +0200 Subject: [PATCH 11/14] up --- .../anonymization/PlanDeTransport/ligneBus.js | 11 ++-- .../PlanDeTransport/modificationBus.js | 13 ++-- api/src/anonymization/application.js | 4 +- api/src/anonymization/cohesionCenter.js | 4 +- api/src/anonymization/contract.js | 12 ++-- api/src/anonymization/departmentService.js | 6 +- api/src/anonymization/meetingPoint.js | 10 +-- api/src/anonymization/mission.js | 11 ++-- api/src/anonymization/missionEquivalence.js | 9 +-- api/src/anonymization/referent.js | 27 ++++---- api/src/anonymization/sessionPhase1.js | 11 ++-- api/src/anonymization/structure.js | 6 +- api/src/anonymization/waitingList.js | 2 +- api/src/anonymization/young.js | 63 ++++++++++--------- api/src/utils/anonymise-model-fields.js | 14 ++++- 15 files changed, 116 insertions(+), 87 deletions(-) diff --git a/api/src/anonymization/PlanDeTransport/ligneBus.js b/api/src/anonymization/PlanDeTransport/ligneBus.js index 8b006b6f20..a7613de6af 100644 --- a/api/src/anonymization/PlanDeTransport/ligneBus.js +++ b/api/src/anonymization/PlanDeTransport/ligneBus.js @@ -7,8 +7,8 @@ function anonymize(itemToAnonymize) { "cohort", "cohortId", "busId", - "departuredDate.$date", - "returnDate.$date", + "departuredDate", + "returnDate", "youngCapacity", "totalCapacity", "followerCapacity", @@ -22,6 +22,7 @@ function anonymize(itemToAnonymize) { "centerDepartureTime", "classeId", "meetingPointsIds", + "team", "team.lastName", "team.firstName", "team.birthdate", @@ -30,9 +31,9 @@ function anonymize(itemToAnonymize) { "delayedForth", "delayedBack", "mergedBusIds", - "createdAt.$date", - "updatedAt.$date", - "deletedAt.$date", + "createdAt", + "updatedAt", + "deletedAt", "__v", ]; const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); diff --git a/api/src/anonymization/PlanDeTransport/modificationBus.js b/api/src/anonymization/PlanDeTransport/modificationBus.js index 4ee8be6912..3bc2f47317 100644 --- a/api/src/anonymization/PlanDeTransport/modificationBus.js +++ b/api/src/anonymization/PlanDeTransport/modificationBus.js @@ -16,18 +16,19 @@ function anonymize(itemToAnonymize) { "status", "statusUserId", "statusUserName", - "statusDate.$date", + "statusDate", "opinion", "opinionUserId", "opinionUserName", - "opinionDate.$date", + "opinionDate", + "messages", "messages.message", "messages.userId", "messages.userName", - "messages.date.$date", - "createdAt.$date", - "updatedAt.$date", - "deletedAt.$date", + "messages.date", + "createdAt", + "updatedAt", + "deletedAt", "__v", ]; const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); diff --git a/api/src/anonymization/application.js b/api/src/anonymization/application.js index ae6bd930db..b7787f8635 100644 --- a/api/src/anonymization/application.js +++ b/api/src/anonymization/application.js @@ -34,8 +34,8 @@ function anonymize(itemToAnonymize) { "feedBackExperienceFiles", "othersFiles", "filesType", - "createdAt.$date", - "updatedAt.$date", + "createdAt", + "updatedAt", "__v", ]; diff --git a/api/src/anonymization/cohesionCenter.js b/api/src/anonymization/cohesionCenter.js index 4d75c09105..bea4c1e078 100644 --- a/api/src/anonymization/cohesionCenter.js +++ b/api/src/anonymization/cohesionCenter.js @@ -29,8 +29,8 @@ function anonymize(itemToAnonymize) { "country", "departmentCode", "sessionStatus", - "createdAt.$date", - "updatedAt.$date", + "createdAt", + "updatedAt", "__v", ]; diff --git a/api/src/anonymization/contract.js b/api/src/anonymization/contract.js index 402ef17e72..6f85ba34ba 100644 --- a/api/src/anonymization/contract.js +++ b/api/src/anonymization/contract.js @@ -20,10 +20,10 @@ function anonymize(itemToAnonymize) { "structureManagerStatus", "parent2Status", "youngContractStatus", - "parent1ValidationDate.$date", - "projectManagerValidationDate.$date", - "structureManagerValidationDate.$date", - "parent2ValidationDate.$date", + "parent1ValidationDate", + "projectManagerValidationDate", + "structureManagerValidationDate", + "parent2ValidationDate", "youngContractValidationDate", "invitationSent", "youngFirstName", @@ -73,8 +73,8 @@ function anonymize(itemToAnonymize) { "tutorEmail", "structureSiret", "structureName", - "createdAt.$date", - "updatedAt.$date", + "createdAt", + "updatedAt", "__v", ]; const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); diff --git a/api/src/anonymization/departmentService.js b/api/src/anonymization/departmentService.js index ecb7b04f6c..c1edd734b2 100644 --- a/api/src/anonymization/departmentService.js +++ b/api/src/anonymization/departmentService.js @@ -4,6 +4,7 @@ const { anonymizeNonDeclaredFields } = require("../utils/anonymise-model-fields" function anonymize(itemToAnonymize) { const whitelist = [ "_id.$oid", + "contacts", "contacts.cohort", "contacts.cohortId", "contacts.contactName", @@ -22,13 +23,14 @@ function anonymize(itemToAnonymize) { "contactName", "contactPhone", "contactMail", + "representantEtat", "representantEtat.firstName", "representantEtat.lastName", "representantEtat.mobile", "representantEtat.email", "representantEtat.role", - "createdAt.$date", - "updatedAt.$date", + "createdAt", + "updatedAt", "__v", ]; const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); diff --git a/api/src/anonymization/meetingPoint.js b/api/src/anonymization/meetingPoint.js index cce21349c2..20ea0af79c 100644 --- a/api/src/anonymization/meetingPoint.js +++ b/api/src/anonymization/meetingPoint.js @@ -17,15 +17,15 @@ function anonymize(itemToAnonymize) { "departureDepartment", "departureRegion", "hideDepartmentInConvocation", - "departureAt.$date", + "departureAt", "departureAtString", "realDepartureAtString", - "returnAt.$date", + "returnAt", "returnAtString", "realReturnAtString", - "createdAt.$date", - "updatedAt.$date", - "deletedAt.$date", + "createdAt", + "updatedAt", + "deletedAt", "__v", ]; const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); diff --git a/api/src/anonymization/mission.js b/api/src/anonymization/mission.js index f595bfe9f7..f64b93c4c9 100644 --- a/api/src/anonymization/mission.js +++ b/api/src/anonymization/mission.js @@ -7,8 +7,8 @@ function anonymize(itemToAnonymize) { "name", "domains", "mainDomain", - "startAt.$date", - "endAt.$date", + "startAt", + "endAt", "duration", "format", "frequence", @@ -36,14 +36,15 @@ function anonymize(itemToAnonymize) { "department", "region", "country", + "location", "location.lat", "location.lon", "addressVerified", "remote", "isMilitaryPreparation", - "createdAt.$date", - "updatedAt.$date", - "lastSyncAt.$date", + "createdAt", + "updatedAt", + "lastSyncAt", "isJvaMission", "jvaMissionId", "apiEngagementId", diff --git a/api/src/anonymization/missionEquivalence.js b/api/src/anonymization/missionEquivalence.js index 0846291a28..4806ef5f7b 100644 --- a/api/src/anonymization/missionEquivalence.js +++ b/api/src/anonymization/missionEquivalence.js @@ -13,8 +13,9 @@ function anonymize(itemToAnonymize) { "address", "zip", "city", - "startDate.$date", - "endDate.$date", + "startDate", + "endDate", + "frequency", "frequency.nombre", "frequency.duree", "frequency.frequence", @@ -23,8 +24,8 @@ function anonymize(itemToAnonymize) { "contactEmail", "files", "message", - "createdAt.$date", - "updatedAt.$date", + "createdAt", + "updatedAt", "__v", ]; const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); diff --git a/api/src/anonymization/referent.js b/api/src/anonymization/referent.js index f167f25a59..e1cc3b7a91 100644 --- a/api/src/anonymization/referent.js +++ b/api/src/anonymization/referent.js @@ -7,24 +7,24 @@ function anonymize(itemToAnonymize) { "firstName", "lastName", "email", - "emailValidatedAt.$date", + "emailValidatedAt", "emailWaitingValidation", "password", "loginAttempts", "token2FA", - "token2FAExpires.$date", + "token2FAExpires", "attempts2FA", "acceptCGU", - "lastLoginAt.$date", - "lastActivityAt.$date", - "lastLogoutAt.$date", - "passwordChangedAt.$date", - "registredAt.$date", - "nextLoginAttemptIn.$date", + "lastLoginAt", + "lastActivityAt", + "lastLogoutAt", + "passwordChangedAt", + "registredAt", + "nextLoginAttemptIn", "forgotPasswordResetToken", - "forgotPasswordResetExpires.$date", + "forgotPasswordResetExpires", "invitationToken", - "invitationExpires.$date", + "invitationExpires", "role", "subRole", "region", @@ -37,10 +37,11 @@ function anonymize(itemToAnonymize) { "cohesionCenterName", "phone", "mobile", + "metadata", "metadata._id.$oid", - "deletedAt.$date", - "createdAt.$date", - "updatedAt.$date", + "deletedAt", + "createdAt", + "updatedAt", "__v", ]; const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); diff --git a/api/src/anonymization/sessionPhase1.js b/api/src/anonymization/sessionPhase1.js index 9ad549ecb6..0f723d70a5 100644 --- a/api/src/anonymization/sessionPhase1.js +++ b/api/src/anonymization/sessionPhase1.js @@ -14,6 +14,7 @@ function anonymize(itemToAnonymize) { "zipCentre", "cityCentre", "headCenterId", + "team", "team.firstName", "team.lastName", "team.role", @@ -22,24 +23,26 @@ function anonymize(itemToAnonymize) { "waitingList", "placesTotal", "placesLeft", + "timeScheduleFiles", "timeScheduleFiles._id", "timeScheduleFiles.name", "timeScheduleFiles.uploadedAt", "timeScheduleFiles.size", "timeScheduleFiles.mimetype", "hasTimeSchedule", + "pedagoProjectFiles", "pedagoProjectFiles._id", "pedagoProjectFiles.name", "pedagoProjectFiles.uploadedAt", "pedagoProjectFiles.size", "pedagoProjectFiles.mimetype", "hasPedagoProject", - "dateStart.$date", - "dateEnd.$date", + "dateStart", + "dateEnd", "status", "sanitaryContactEmail", - "createdAt.$date", - "updatedAt.$date", + "createdAt", + "updatedAt", "__v", ]; const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); diff --git a/api/src/anonymization/structure.js b/api/src/anonymization/structure.js index f10bc08801..0673bbcd06 100644 --- a/api/src/anonymization/structure.js +++ b/api/src/anonymization/structure.js @@ -28,6 +28,7 @@ function anonymize(itemToAnonymize) { "department", "region", "country", + "location", "location.lon", "location.lat", "addressVerified", @@ -36,13 +37,14 @@ function anonymize(itemToAnonymize) { "isJvaStructure", "jvaStructureId", "jvaRawData", + "structureManager", "structureManager.firstName", "structureManager.lastName", "structureManager.mobile", "structureManager.email", "structureManager.role", - "createdAt.$date", - "updatedAt.$date", + "createdAt", + "updatedAt", "__v", ]; const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); diff --git a/api/src/anonymization/waitingList.js b/api/src/anonymization/waitingList.js index eea2ad1eab..862583837b 100644 --- a/api/src/anonymization/waitingList.js +++ b/api/src/anonymization/waitingList.js @@ -2,7 +2,7 @@ const { generateRandomEmail, generateBirthdate } = require("../utils/anonymise") const { anonymizeNonDeclaredFields } = require("../utils/anonymise-model-fields"); function anonymize(itemToAnonymize) { - const whitelist = ["_id.$oid", "zip", "mail", "birthdateAt", "createdAt.$date", "updatedAt.$date", "__v"]; + const whitelist = ["_id.$oid", "zip", "mail", "birthdateAt", "createdAt", "updatedAt", "__v"]; const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); diff --git a/api/src/anonymization/young.js b/api/src/anonymization/young.js index f2746e821e..20d6d04a91 100644 --- a/api/src/anonymization/young.js +++ b/api/src/anonymization/young.js @@ -17,7 +17,7 @@ function anonymize(itemToAnonymize) { "phone", "phoneZone", "gender", - "birthdateAt.$date", + "birthdateAt", "cohort", "cohortId", "originalCohort", @@ -31,42 +31,42 @@ function anonymize(itemToAnonymize) { "statusPhase1Motif", "statusPhase1MotifDetail", "statusPhase2", - "statusPhase2updatedAt.$date", - "statusPhase2OpenedAt.$date", - "statusPhase2ValidatedAt.$date", + "statusPhase2updatedAt", + "statusPhase2OpenedAt", + "statusPhase2ValidatedAt", "statusPhase2Contract", "statusPhase3", - "statusPhase3updatedAt.$date", - "statusPhase3ValidatedAt.$date", - "lastStatusAt.$date", + "statusPhase3updatedAt", + "statusPhase3ValidatedAt", + "lastStatusAt", "withdrawnReason", "withdrawnMessage", "hasStartedReinscription", "reinscriptionStep2023", "inscriptionStep2023", "inscriptionStep", - "inscriptionDoneDate.$date", + "inscriptionDoneDate", "cohesion2020Step", "inscriptionCorrectionMessage", "inscriptionRefusedMessage", "historic", "password", "token2FA", - "token2FAExpires.$date", + "token2FAExpires", "attempts2FA", "tokenEmailValidation", - "tokenEmailValidationExpires.$date", + "tokenEmailValidationExpires", "attemptsEmailValidation", "loginAttempts", - "lastLoginAt.$date", - "lastActivityAt.$date", - "lastLogoutAt.$date", - "passwordChangedAt.$date", - "nextLoginAttemptIn.$date", + "lastLoginAt", + "lastActivityAt", + "lastLogoutAt", + "passwordChangedAt", + "nextLoginAttemptIn", "forgotPasswordResetToken", - "forgotPasswordResetExpires.$date", + "forgotPasswordResetExpires", "invitationToken", - "invitationExpires.$date", + "invitationExpires", "acceptCGU", "acceptRI", "cniFiles", @@ -74,7 +74,7 @@ function anonymize(itemToAnonymize) { "presenceJDM", "cohesionStayMedicalFileReceived", "departInform", - "departSejourAt.$date", + "departSejourAt", "departSejourMotif", "departSejourMotifComment", "cohesionStayMedicalFileDownload", @@ -96,7 +96,7 @@ function anonymize(itemToAnonymize) { "cohesionCenterName", "cohesionCenterZip", "cohesionCenterCity", - "autoAffectationPhase1ExpiresAt.$date", + "autoAffectationPhase1ExpiresAt", "phase2ApplicationStatus", "phase2ApplicationFilesType", "phase2NumberHoursDone", @@ -104,8 +104,8 @@ function anonymize(itemToAnonymize) { "phase3StructureName", "phase3MissionDomain", "phase3MissionDescription", - "phase3MissionStartAt.$date", - "phase3MissionEndAt.$date", + "phase3MissionStartAt", + "phase3MissionEndAt", "phase3TutorFirstName", "phase3TutorLastName", "phase3TutorEmail", @@ -124,6 +124,7 @@ function anonymize(itemToAnonymize) { "department", "region", "country", + "location", "location.lat", "location.lon", "qpv", @@ -145,6 +146,7 @@ function anonymize(itemToAnonymize) { "schoolDepartment", "schoolRegion", "schoolCountry", + "schoolLocation", "schoolLocation.lat", "schoolLocation.lon", "schoolId", @@ -166,6 +168,7 @@ function anonymize(itemToAnonymize) { "parent1Department", "parent1Region", "parent1Country", + "parent1Location", "parent1Location.lat", "parent1Location.lon", "parent1FromFranceConnect", @@ -191,6 +194,7 @@ function anonymize(itemToAnonymize) { "parent2Department", "parent2Region", "parent2Country", + "parent2Location", "parent2Location.lat", "parent2Location.lon", "parent2FromFranceConnect", @@ -220,6 +224,7 @@ function anonymize(itemToAnonymize) { "medicosocialStructureCity", "medicosocialStructureDepartment", "medicosocialStructureRegion", + "medicosocialStructureLocation", "medicosocialStructureLocation.lat", "medicosocialStructureLocation.lon", "engagedStructure", @@ -233,8 +238,8 @@ function anonymize(itemToAnonymize) { "parentAllowSNU", "parent1AllowSNU", "parent2AllowSNU", - "parent1ValidationDate.$date", - "parent2ValidationDate.$date", + "parent1ValidationDate", + "parent2ValidationDate", "parent2RejectSNUComment", "dataProcessingConsentmentFiles", "parentConsentment", @@ -285,6 +290,7 @@ function anonymize(itemToAnonymize) { "militaryPreparationFilesCertificate", "statusMilitaryPreparationFiles", "militaryPreparationCorrectionMessage", + "files", "files.cniFiles", "files.highSkilledActivityProofFiles", "files.dataProcessingConsentmentFiles", @@ -296,11 +302,12 @@ function anonymize(itemToAnonymize) { "files.militaryPreparationFilesCensus", "files.militaryPreparationFilesAuthorization", "files.militaryPreparationFilesCertificate", - "latestCNIFileExpirationDate.$date", + "latestCNIFileExpirationDate", "CNIFileNotValidOnStart", "latestCNIFileCategory", + "missionsInMail", "missionsInMail.missionId", - "missionsInMail.date.$date", + "missionsInMail.date", "youngPhase1Agreement", "status_equivalence", "correctionRequests", @@ -319,9 +326,9 @@ function anonymize(itemToAnonymize) { "sportInterest", "environmentInterest", "citizenshipInterest", - "deletedAt.$date", - "createdAt.$date", - "updatedAt.$date", + "deletedAt", + "createdAt", + "updatedAt", "__v", ]; const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); diff --git a/api/src/utils/anonymise-model-fields.js b/api/src/utils/anonymise-model-fields.js index 5f59f26a39..36d36d1c1d 100644 --- a/api/src/utils/anonymise-model-fields.js +++ b/api/src/utils/anonymise-model-fields.js @@ -16,6 +16,10 @@ function getAllPaths(obj, parentPath = "") { let paths = []; for (let key in obj) { const currentPath = parentPath ? `${parentPath}.${key}` : key; + if (obj[key] instanceof Date) { + paths.push(currentPath); + continue; + } if (typeof obj[key] === "object" && obj[key] !== null && !Array.isArray(obj[key])) { paths = paths.concat(getAllPaths(obj[key], currentPath)); } else { @@ -25,12 +29,18 @@ function getAllPaths(obj, parentPath = "") { return paths; } +function isWhitelisted(path, whitelist) { + return whitelist.some((whitelistedPath) => { + return path === whitelistedPath || path.startsWith(whitelistedPath + "."); + }); +} + // Anonymise tous les champs d'un objet qui ne sont pas dans la whitelist function anonymizeNonDeclaredFields(item, whitelist) { const allPaths = getAllPaths(item); for (const path of allPaths) { - if (!whitelist.includes(path)) { + if (!isWhitelisted(path, whitelist)) { const value = getNestedValue(item, path); if (Array.isArray(value) && value.length > 0) { @@ -48,7 +58,7 @@ function anonymizeNonDeclaredFields(item, whitelist) { if (value !== undefined) { if (Array.isArray(value)) { setNestedValue(item, path, []); - } else if (path.includes("$date")) { + } else if (value instanceof Date) { setNestedValue(item, path, new Date()); } else if (typeof value === "string") { setNestedValue(item, path, ""); From 0bfb577fe97e835fad2a2a2d263c0734fcd615d7 Mon Sep 17 00:00:00 2001 From: Michael Garcia Date: Tue, 22 Oct 2024 14:45:51 +0200 Subject: [PATCH 12/14] up --- api/src/anonymization/young.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/src/anonymization/young.js b/api/src/anonymization/young.js index 20d6d04a91..0fb92bf956 100644 --- a/api/src/anonymization/young.js +++ b/api/src/anonymization/young.js @@ -1,6 +1,6 @@ const { generateAddress, generateRandomName, generateRandomEmail, generateBirthdate, getYoungLocation, generateNewPhoneNumber, starify } = require("../utils/anonymise"); const { anonymizeNonDeclaredFields } = require("../utils/anonymise-model-fields"); -const crypto = require("crypto"); +const { v4: uuid } = require("uuid"); function anonymize(itemToAnonymize) { const whitelist = [ @@ -399,8 +399,8 @@ function anonymize(itemToAnonymize) { item.forgotPasswordResetToken = ""; item.invitationToken = ""; item.phase3Token = ""; - item.parent1Inscription2023Token = crypto.randomBytes(20).toString("hex"); - item.parent2Inscription2023Token = crypto.randomBytes(20).toString("hex"); + item.parent1Inscription2023Token = uuid(); + item.parent2Inscription2023Token = uuid(); return item; } From 1346683e58f035ac81e913356d39860e9c51d27c Mon Sep 17 00:00:00 2001 From: Michael Garcia Date: Fri, 8 Nov 2024 11:54:16 +0100 Subject: [PATCH 13/14] up --- package-lock.json | 1 - 1 file changed, 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index b1d44a109b..6d1e6235af 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1183,7 +1183,6 @@ "node": ">=14.20.1" } }, - "api/node_modules/mongoose/node_modules/mongodb": { "version": "5.9.2", "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-5.9.2.tgz", From 447d032c56f2b64bcd96d47428c270b4ea113ddb Mon Sep 17 00:00:00 2001 From: Michael Garcia Date: Fri, 8 Nov 2024 12:01:44 +0100 Subject: [PATCH 14/14] upp --- api/src/anonymization/sessionPhase1.js | 1 - 1 file changed, 1 deletion(-) diff --git a/api/src/anonymization/sessionPhase1.js b/api/src/anonymization/sessionPhase1.js index 6dd3c59e8b..72a3fb0de7 100644 --- a/api/src/anonymization/sessionPhase1.js +++ b/api/src/anonymization/sessionPhase1.js @@ -47,7 +47,6 @@ function anonymize(itemToAnonymize) { ]; const item = anonymizeNonDeclaredFields(itemToAnonymize, whitelist); -function anonymize(item) { if (!["VALIDATED", "WAITING_VALIDATION"].includes(item.status)) item.status = "WAITING_VALIDATION"; item.team && (item.team = item.team.map((member) => {