From aff2b3e19c9941242286dacc47fe6e17866768ec Mon Sep 17 00:00:00 2001 From: rthebaud Date: Wed, 9 Oct 2024 08:46:16 +0200 Subject: [PATCH] chore(misc) : 3404 - monstache integration POC (#4377) Co-authored-by: Anselme --- api/config/default.js | 1 + api/config/production.js | 1 + api/src/models/PlanDeTransport/ligneBus.ts | 5 +- .../models/PlanDeTransport/ligneToPoint.ts | 5 +- .../models/PlanDeTransport/modificationBus.ts | 7 +- .../models/PlanDeTransport/planTransport.ts | 7 +- .../PlanDeTransport/pointDeRassemblement.ts | 5 +- .../PlanDeTransport/tableDeRepartition.ts | 5 +- api/src/models/alerteMessage.ts | 5 +- api/src/models/application.ts | 6 +- api/src/models/cle/classe.ts | 5 +- api/src/models/cle/etablissement.ts | 5 +- api/src/models/cohesionCenter.ts | 5 +- api/src/models/contract.ts | 5 +- api/src/models/departmentService.ts | 5 +- api/src/models/email.ts | 5 +- api/src/models/inscriptionGoal.ts | 5 +- api/src/models/mission.ts | 6 +- api/src/models/missionAPI.ts | 5 +- api/src/models/referent.ts | 47 +- api/src/models/schoolRAMSES.ts | 6 +- api/src/models/sessionPhase1.ts | 5 +- api/src/models/structure.ts | 6 +- api/src/models/young.ts | 56 +- .../scripts/reindex_es_all_models/index.js | 17 +- infra/ES_datariver/Dockerfile | 16 + infra/ES_datariver/collections.csv | 22 + .../config/index_templates/README.txt | 1 + .../config/index_templates/application.json | 267 +++ .../index_templates/cohesioncenter.json | 213 ++ .../index_templates/departmentservice.json | 214 ++ .../config/index_templates/lignebus.json | 120 + .../config/index_templates/mission.json | 386 ++++ .../config/index_templates/missionapi.json | 192 ++ .../index_templates/modificationbus.json | 182 ++ .../index_templates/plandetransport.json | 478 ++++ .../index_templates/pointderassemblement.json | 130 ++ .../config/index_templates/referent.json | 179 ++ .../config/index_templates/schoolramses.json | 138 ++ .../config/index_templates/sessionphase1.json | 82 + .../config/index_templates/structure.json | 330 +++ .../config/index_templates/young.json | 1941 +++++++++++++++++ infra/ES_datariver/config/index_whitelist.txt | 21 + .../config/ingest_pipelines/README.txt | 1 + .../config/ingest_pipelines/mission.json | 11 + .../config/ingest_pipelines/referent.json | 26 + .../config/ingest_pipelines/schoolramses.json | 11 + .../ingest_pipelines/sessionphase1.json | 11 + .../config/ingest_pipelines/structure.json | 11 + .../config/ingest_pipelines/young.json | 30 + infra/ES_datariver/docker_start.sh | 12 + infra/ES_datariver/generate_config.sh | 59 + infra/ES_datariver/scripts/install.sh | 38 + infra/ES_datariver/scripts/uninstall.sh | 53 + 54 files changed, 5328 insertions(+), 77 deletions(-) create mode 100644 infra/ES_datariver/Dockerfile create mode 100644 infra/ES_datariver/collections.csv create mode 100644 infra/ES_datariver/config/index_templates/README.txt create mode 100644 infra/ES_datariver/config/index_templates/application.json create mode 100644 infra/ES_datariver/config/index_templates/cohesioncenter.json create mode 100644 infra/ES_datariver/config/index_templates/departmentservice.json create mode 100644 infra/ES_datariver/config/index_templates/lignebus.json create mode 100644 infra/ES_datariver/config/index_templates/mission.json create mode 100644 infra/ES_datariver/config/index_templates/missionapi.json create mode 100644 infra/ES_datariver/config/index_templates/modificationbus.json create mode 100644 infra/ES_datariver/config/index_templates/plandetransport.json create mode 100644 infra/ES_datariver/config/index_templates/pointderassemblement.json create mode 100644 infra/ES_datariver/config/index_templates/referent.json create mode 100644 infra/ES_datariver/config/index_templates/schoolramses.json create mode 100644 infra/ES_datariver/config/index_templates/sessionphase1.json create mode 100644 infra/ES_datariver/config/index_templates/structure.json create mode 100644 infra/ES_datariver/config/index_templates/young.json create mode 100644 infra/ES_datariver/config/index_whitelist.txt create mode 100644 infra/ES_datariver/config/ingest_pipelines/README.txt create mode 100644 infra/ES_datariver/config/ingest_pipelines/mission.json create mode 100644 infra/ES_datariver/config/ingest_pipelines/referent.json create mode 100644 infra/ES_datariver/config/ingest_pipelines/schoolramses.json create mode 100644 infra/ES_datariver/config/ingest_pipelines/sessionphase1.json create mode 100644 infra/ES_datariver/config/ingest_pipelines/structure.json create mode 100644 infra/ES_datariver/config/ingest_pipelines/young.json create mode 100755 infra/ES_datariver/docker_start.sh create mode 100755 infra/ES_datariver/generate_config.sh create mode 100755 infra/ES_datariver/scripts/install.sh create mode 100755 infra/ES_datariver/scripts/uninstall.sh diff --git a/api/config/default.js b/api/config/default.js index 831ab5dd56..6ea4d87fc2 100644 --- a/api/config/default.js +++ b/api/config/default.js @@ -68,4 +68,5 @@ module.exports = { ENABLE_2FA: false, LOG_LEVEL: "info", // error, warn, info, http, debug DO_MIGRATION: true, + ENABLE_MONGOOSE_ELASTIC: false, }; diff --git a/api/config/production.js b/api/config/production.js index 7a4c758dbe..cc93973fd4 100644 --- a/api/config/production.js +++ b/api/config/production.js @@ -20,5 +20,6 @@ module.exports = { TASK_QUEUE_PREFIX: "production", MAIL_TRANSPORT: "BREVO", ENABLE_2FA: true, + ENABLE_MONGOOSE_ELASTIC: true, ...secrets, }; diff --git a/api/src/models/PlanDeTransport/ligneBus.ts b/api/src/models/PlanDeTransport/ligneBus.ts index 6705b794dd..97edfa78e0 100644 --- a/api/src/models/PlanDeTransport/ligneBus.ts +++ b/api/src/models/PlanDeTransport/ligneBus.ts @@ -1,3 +1,4 @@ +import config from "config"; import mongoose, { Schema, InferSchemaType } from "mongoose"; import mongooseElastic from "@selego/mongoose-elastic"; import patchHistory from "mongoose-patch-history"; @@ -46,7 +47,9 @@ schema.plugin(patchHistory, { excludes: ["/updatedAt"], }); -schema.plugin(mongooseElastic(esClient), MODELNAME); +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin(mongooseElastic(esClient), MODELNAME); +} schema.index({ cohort: 1 }); schema.index({ cohort: 1, busId: 1 }, { unique: true }); diff --git a/api/src/models/PlanDeTransport/ligneToPoint.ts b/api/src/models/PlanDeTransport/ligneToPoint.ts index 089c9f7c4e..9d34c71ba2 100644 --- a/api/src/models/PlanDeTransport/ligneToPoint.ts +++ b/api/src/models/PlanDeTransport/ligneToPoint.ts @@ -1,3 +1,4 @@ +import config from "config"; import mongoose, { Schema, InferSchemaType } from "mongoose"; import patchHistory from "mongoose-patch-history"; import mongooseElastic from "@selego/mongoose-elastic"; @@ -35,7 +36,9 @@ schema.plugin(patchHistory, { excludes: ["/updatedAt"], }); -schema.plugin(mongooseElastic(esClient), MODELNAME); +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin(mongooseElastic(esClient), MODELNAME); +} type LigneToPointType = InterfaceExtended>; export type LigneToPointDocument = DocumentExtended; diff --git a/api/src/models/PlanDeTransport/modificationBus.ts b/api/src/models/PlanDeTransport/modificationBus.ts index ae4d546d3c..8d8dccfd9b 100644 --- a/api/src/models/PlanDeTransport/modificationBus.ts +++ b/api/src/models/PlanDeTransport/modificationBus.ts @@ -1,4 +1,5 @@ -import mongoose, { InferSchemaType, Schema } from "mongoose"; +import config from "config"; +import mongoose, { Schema, InferSchemaType } from "mongoose"; import patchHistory from "mongoose-patch-history"; import mongooseElastic from "@selego/mongoose-elastic"; @@ -40,7 +41,9 @@ schema.plugin(patchHistory, { excludes: ["/updatedAt"], }); -schema.plugin(mongooseElastic(esClient), MODELNAME); +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin(mongooseElastic(esClient), MODELNAME); +} type ModificationBusType = InterfaceExtended>; export type ModificationBusDocument = DocumentExtended; diff --git a/api/src/models/PlanDeTransport/planTransport.ts b/api/src/models/PlanDeTransport/planTransport.ts index cfd57cf62c..d9bf16645d 100644 --- a/api/src/models/PlanDeTransport/planTransport.ts +++ b/api/src/models/PlanDeTransport/planTransport.ts @@ -1,4 +1,5 @@ -import mongoose, { InferSchemaType, Schema } from "mongoose"; +import config from "config"; +import mongoose, { Schema, InferSchemaType } from "mongoose"; import patchHistory from "mongoose-patch-history"; import mongooseElastic from "@selego/mongoose-elastic"; @@ -49,7 +50,9 @@ schema.plugin(patchHistory, { excludes: ["/updatedAt"], }); -schema.plugin(mongooseElastic(esClient), MODELNAME); +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin(mongooseElastic(esClient), MODELNAME); +} schema.index({ ligneDeBusId: 1 }); diff --git a/api/src/models/PlanDeTransport/pointDeRassemblement.ts b/api/src/models/PlanDeTransport/pointDeRassemblement.ts index bcfed55f36..1396dd3a64 100644 --- a/api/src/models/PlanDeTransport/pointDeRassemblement.ts +++ b/api/src/models/PlanDeTransport/pointDeRassemblement.ts @@ -1,3 +1,4 @@ +import config from "config"; import mongoose, { InferSchemaType, Schema } from "mongoose"; import patchHistory from "mongoose-patch-history"; import mongooseElastic from "@selego/mongoose-elastic"; @@ -35,7 +36,9 @@ schema.plugin(patchHistory, { excludes: ["/updatedAt"], }); -schema.plugin(mongooseElastic(esClient), MODELNAME); +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin(mongooseElastic(esClient), MODELNAME); +} type PointDeRassemblementType = InterfaceExtended>; export type PointDeRassemblementDocument = DocumentExtended; diff --git a/api/src/models/PlanDeTransport/tableDeRepartition.ts b/api/src/models/PlanDeTransport/tableDeRepartition.ts index 34ef334145..5896a92e42 100644 --- a/api/src/models/PlanDeTransport/tableDeRepartition.ts +++ b/api/src/models/PlanDeTransport/tableDeRepartition.ts @@ -1,3 +1,4 @@ +import config from "config"; import mongoose, { Schema, InferSchemaType } from "mongoose"; import mongooseElastic from "@selego/mongoose-elastic"; import esClient from "../../es"; @@ -34,7 +35,9 @@ schema.plugin(patchHistory, { excludes: ["/updatedAt"], }); -schema.plugin(mongooseElastic(esClient), MODELNAME); +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin(mongooseElastic(esClient), MODELNAME); +} type TableDeRepartitionType = InterfaceExtended>; export type TableDeRepartitionDocument = DocumentExtended; diff --git a/api/src/models/alerteMessage.ts b/api/src/models/alerteMessage.ts index 922d2029f7..75fb6e14e6 100644 --- a/api/src/models/alerteMessage.ts +++ b/api/src/models/alerteMessage.ts @@ -1,3 +1,4 @@ +import config from "config"; import mongoose, { Schema } from "mongoose"; import mongooseElastic from "@selego/mongoose-elastic"; import esClient from "../es"; @@ -35,7 +36,9 @@ schema.plugin(patchHistory, { excludes: ["/updatedAt"], }); -schema.plugin(mongooseElastic(esClient), MODELNAME); +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin(mongooseElastic(esClient), MODELNAME); +} export type AlerteMessageDocument = DocumentExtended; type SchemaExtended = AlerteMessageDocument & UserExtension; diff --git a/api/src/models/application.ts b/api/src/models/application.ts index db38c258fc..902b4675c3 100644 --- a/api/src/models/application.ts +++ b/api/src/models/application.ts @@ -1,3 +1,4 @@ +import config from "config"; import mongoose, { Schema, InferSchemaType } from "mongoose"; import patchHistory from "mongoose-patch-history"; import mongooseElastic from "@selego/mongoose-elastic"; @@ -45,7 +46,10 @@ schema.plugin(patchHistory, { }, excludes: ["/updatedAt"], }); -schema.plugin(mongooseElastic(esClient), MODELNAME); + +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin(mongooseElastic(esClient), MODELNAME); +} type ApplicationType = InterfaceExtended>; export type ApplicationDocument = DocumentExtended; diff --git a/api/src/models/cle/classe.ts b/api/src/models/cle/classe.ts index ce51b183d8..0e1c1b4580 100644 --- a/api/src/models/cle/classe.ts +++ b/api/src/models/cle/classe.ts @@ -1,3 +1,4 @@ +import config from "config"; import mongoose from "mongoose"; import patchHistory from "mongoose-patch-history"; @@ -93,7 +94,9 @@ schema.plugin(patchHistory, { excludes: ["/updatedAt"], }); -schema.plugin(mongooseElastic(esClient), MODELNAME); +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin(mongooseElastic(esClient), MODELNAME); +} export type ClasseDocument = DocumentExtended< ClasseType & { diff --git a/api/src/models/cle/etablissement.ts b/api/src/models/cle/etablissement.ts index 4cea4d53da..900ca7c494 100644 --- a/api/src/models/cle/etablissement.ts +++ b/api/src/models/cle/etablissement.ts @@ -1,3 +1,4 @@ +import config from "config"; import mongoose, { Schema } from "mongoose"; import patchHistory from "mongoose-patch-history"; @@ -51,7 +52,9 @@ schema.plugin(patchHistory, { excludes: ["/updatedAt"], }); -schema.plugin(mongooseElastic(esClient), MODELNAME); +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin(mongooseElastic(esClient), MODELNAME); +} export type EtablissementDocument = DocumentExtended; type SchemaExtended = EtablissementDocument & UserExtension; diff --git a/api/src/models/cohesionCenter.ts b/api/src/models/cohesionCenter.ts index a337cf59fa..c6c9142fcd 100644 --- a/api/src/models/cohesionCenter.ts +++ b/api/src/models/cohesionCenter.ts @@ -1,3 +1,4 @@ +import config from "config"; import mongoose, { Schema } from "mongoose"; import patchHistory from "mongoose-patch-history"; import mongooseElastic from "@selego/mongoose-elastic"; @@ -40,7 +41,9 @@ schema.plugin(patchHistory, { excludes: ["/updatedAt"], }); -schema.plugin(mongooseElastic(esClient), MODELNAME); +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin(mongooseElastic(esClient), MODELNAME); +} export type CohesionCenterDocument = DocumentExtended; type SchemaExtended = CohesionCenterDocument & UserExtension; diff --git a/api/src/models/contract.ts b/api/src/models/contract.ts index 75b085a89c..288edc0b69 100644 --- a/api/src/models/contract.ts +++ b/api/src/models/contract.ts @@ -1,3 +1,4 @@ +import config from "config"; import mongoose, { Schema, InferSchemaType } from "mongoose"; import patchHistory from "mongoose-patch-history"; import mongooseElastic from "@selego/mongoose-elastic"; @@ -40,7 +41,9 @@ schema.plugin(patchHistory, { excludes: ["/parent1Token", "/projectManagerToken", "/structureManagerToken", "/parent2Token", "/youngContractToken", "/updatedAt"], }); -schema.plugin(mongooseElastic(esClient), MODELNAME); +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin(mongooseElastic(esClient), MODELNAME); +} type ContractType = InterfaceExtended>; export type ContractDocument = DocumentExtended; diff --git a/api/src/models/departmentService.ts b/api/src/models/departmentService.ts index 312cb8f666..422596d793 100644 --- a/api/src/models/departmentService.ts +++ b/api/src/models/departmentService.ts @@ -1,3 +1,4 @@ +import config from "config"; import mongoose, { Schema, InferSchemaType } from "mongoose"; import patchHistory from "mongoose-patch-history"; @@ -41,7 +42,9 @@ schema.plugin(patchHistory, { excludes: ["/updatedAt"], }); -schema.plugin(mongooseElastic(esClient), MODELNAME); +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin(mongooseElastic(esClient), MODELNAME); +} type DepartmentServiceType = InterfaceExtended>; export type DepartmentServiceDocument = DocumentExtended; diff --git a/api/src/models/email.ts b/api/src/models/email.ts index ea5cbd6a9e..1b7c7ce3fb 100644 --- a/api/src/models/email.ts +++ b/api/src/models/email.ts @@ -1,3 +1,4 @@ +import config from "config"; import mongoose, { Schema, InferSchemaType } from "mongoose"; import mongooseElastic from "@selego/mongoose-elastic"; import esClient from "../es"; @@ -11,7 +12,9 @@ const MODELNAME = "email"; const schema = new Schema(EmailSchema); -schema.plugin(mongooseElastic(esClient), MODELNAME); +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin(mongooseElastic(esClient), MODELNAME); +} type EmailType = InterfaceExtended>; export type EmailDocument = DocumentExtended; diff --git a/api/src/models/inscriptionGoal.ts b/api/src/models/inscriptionGoal.ts index 61c5933f8e..4f832e9436 100644 --- a/api/src/models/inscriptionGoal.ts +++ b/api/src/models/inscriptionGoal.ts @@ -1,3 +1,4 @@ +import config from "config"; import mongoose, { Schema, InferSchemaType } from "mongoose"; import patchHistory from "mongoose-patch-history"; import mongooseElastic from "@selego/mongoose-elastic"; @@ -35,7 +36,9 @@ schema.plugin(patchHistory, { excludes: ["/updatedAt"], }); -schema.plugin(mongooseElastic(esClient), MODELNAME); +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin(mongooseElastic(esClient), MODELNAME); +} type InscriptionGoalType = InterfaceExtended>; export type InscriptionGoalDocument = DocumentExtended; diff --git a/api/src/models/mission.ts b/api/src/models/mission.ts index 2c0fbd4160..94497c1af9 100644 --- a/api/src/models/mission.ts +++ b/api/src/models/mission.ts @@ -1,3 +1,4 @@ +import config from "config"; import mongoose, { InferSchemaType } from "mongoose"; import mongooseElastic from "@selego/mongoose-elastic"; import patchHistory from "mongoose-patch-history"; @@ -40,7 +41,10 @@ schema.plugin(patchHistory, { }, excludes: ["/updatedAt"], }); -schema.plugin(mongooseElastic(esClient, { selectiveIndexing: true, ignore: ["jvaRawData"] }), MODELNAME); + +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin(mongooseElastic(esClient, { selectiveIndexing: true, ignore: ["jvaRawData"] }), MODELNAME); +} type MissionType = InterfaceExtended>; export type MissionDocument = DocumentExtended; diff --git a/api/src/models/missionAPI.ts b/api/src/models/missionAPI.ts index de2e224e23..ec94368eb5 100644 --- a/api/src/models/missionAPI.ts +++ b/api/src/models/missionAPI.ts @@ -1,3 +1,4 @@ +import config from "config"; import mongoose, { Schema, InferSchemaType } from "mongoose"; import patchHistory from "mongoose-patch-history"; import mongooseElastic from "@selego/mongoose-elastic"; @@ -35,7 +36,9 @@ schema.plugin(patchHistory, { excludes: ["/updatedAt"], }); -schema.plugin(mongooseElastic(esClient), MODELNAME); +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin(mongooseElastic(esClient), MODELNAME); +} type MissionAPIType = InterfaceExtended>; export type MissionAPIDocument = DocumentExtended; diff --git a/api/src/models/referent.ts b/api/src/models/referent.ts index f12922579b..0e33f91083 100644 --- a/api/src/models/referent.ts +++ b/api/src/models/referent.ts @@ -1,3 +1,4 @@ +import config from "config"; import mongoose, { Schema } from "mongoose"; import patchHistory from "mongoose-patch-history"; import bcrypt from "bcryptjs"; @@ -88,28 +89,30 @@ schema.plugin(patchHistory, { ], }); -schema.plugin( - mongooseElastic(esClient, { - selectiveIndexing: true, - ignore: [ - "password", - "lastLogoutAt", - "passwordChangedAt", - "nextLoginAttemptIn", - "forgotPasswordResetToken", - "forgotPasswordResetExpires", - "invitationToken", - "invitationExpires", - "loginAttempts", - "attempts2FA", - "updatedAt", - "lastActivityAt", - "token2FA", - "token2FAExpires", - ], - }), - MODELNAME, -); +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin( + mongooseElastic(esClient, { + selectiveIndexing: true, + ignore: [ + "password", + "lastLogoutAt", + "passwordChangedAt", + "nextLoginAttemptIn", + "forgotPasswordResetToken", + "forgotPasswordResetExpires", + "invitationToken", + "invitationExpires", + "loginAttempts", + "attempts2FA", + "updatedAt", + "lastActivityAt", + "token2FA", + "token2FAExpires", + ], + }), + MODELNAME, + ); +} schema.set("toObject", { virtuals: true }); schema.set("toJSON", { virtuals: true }); diff --git a/api/src/models/schoolRAMSES.ts b/api/src/models/schoolRAMSES.ts index 6077862b14..1f1ccd8bdb 100644 --- a/api/src/models/schoolRAMSES.ts +++ b/api/src/models/schoolRAMSES.ts @@ -1,3 +1,4 @@ +import config from "config"; import mongoose, { Schema, InferSchemaType } from "mongoose"; import patchHistory from "mongoose-patch-history"; import mongooseElastic from "@selego/mongoose-elastic"; @@ -34,8 +35,9 @@ schema.plugin(patchHistory, { }, excludes: ["/updatedAt"], }); - -schema.plugin(mongooseElastic(esClient, { selectiveIndexing: true, ignore: ["raw_data"] }), MODELNAME); +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin(mongooseElastic(esClient, { selectiveIndexing: true, ignore: ["raw_data"] }), MODELNAME); +} type SchoolRAMSESType = InterfaceExtended>; export type SchoolRAMSESDocument = DocumentExtended; diff --git a/api/src/models/sessionPhase1.ts b/api/src/models/sessionPhase1.ts index e138325cbb..d84b50f33e 100644 --- a/api/src/models/sessionPhase1.ts +++ b/api/src/models/sessionPhase1.ts @@ -1,3 +1,4 @@ +import config from "config"; import mongoose, { Schema } from "mongoose"; import mongooseElastic from "@selego/mongoose-elastic"; import patchHistory from "mongoose-patch-history"; @@ -58,7 +59,9 @@ schema.plugin(patchHistory, { excludes: ["/updatedAt"], }); -schema.plugin(mongooseElastic(esClient, { selectiveIndexing: true, ignore: ["team"] }), MODELNAME); +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin(mongooseElastic(esClient, { selectiveIndexing: true, ignore: ["team"] }), MODELNAME); +} schema.index({ cohesionCenterId: 1 }); diff --git a/api/src/models/structure.ts b/api/src/models/structure.ts index cbeeb1bb30..61491dc12d 100644 --- a/api/src/models/structure.ts +++ b/api/src/models/structure.ts @@ -1,3 +1,4 @@ +import config from "config"; import mongoose, { Schema, InferSchemaType } from "mongoose"; import patchHistory from "mongoose-patch-history"; import mongooseElastic from "@selego/mongoose-elastic"; @@ -40,7 +41,10 @@ schema.plugin(patchHistory, { }, excludes: ["/updatedAt"], }); -schema.plugin(mongooseElastic(esClient, { selectiveIndexing: true, ignore: ["jvaRawData"] }), MODELNAME); + +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin(mongooseElastic(esClient, { selectiveIndexing: true, ignore: ["jvaRawData"] }), MODELNAME); +} type StructureType = InterfaceExtended>; export type StructureDocument = DocumentExtended; diff --git a/api/src/models/young.ts b/api/src/models/young.ts index f9e71f0797..6df1b61268 100644 --- a/api/src/models/young.ts +++ b/api/src/models/young.ts @@ -1,8 +1,8 @@ +import config from "config"; import mongoose, { Schema } from "mongoose"; import bcrypt from "bcryptjs"; import mongooseElastic from "@selego/mongoose-elastic"; import patchHistory from "mongoose-patch-history"; -import config from "config"; import { YOUNG_SOURCE, YOUNG_STATUS, YoungSchema, YoungSchemaCorrectionRequest, YoungSchemaFile, YoungSchemaNote, YoungType } from "snu-lib"; @@ -115,32 +115,34 @@ schema.plugin(patchHistory, { ], }); -schema.plugin( - mongooseElastic(esClient, { - selectiveIndexing: true, - ignore: [ - "historic", - "missionsInMail", - "password", - "lastLogoutAt", - "passwordChangedAt", - "nextLoginAttemptIn", - "forgotPasswordResetToken", - "forgotPasswordResetExpires", - "invitationExpires", - "phase3Token", - "loginAttempts", - "parent1Inscription2023Token", - "parent2Inscription2023Token", - "updatedAt", - "lastActivityAt", - "userIps", - "token2FA", - "token2FAExpires", - ], - }), - MODELNAME, -); +if (config.get("ENABLE_MONGOOSE_ELASTIC")) { + schema.plugin( + mongooseElastic(esClient, { + selectiveIndexing: true, + ignore: [ + "historic", + "missionsInMail", + "password", + "lastLogoutAt", + "passwordChangedAt", + "nextLoginAttemptIn", + "forgotPasswordResetToken", + "forgotPasswordResetExpires", + "invitationExpires", + "phase3Token", + "loginAttempts", + "parent1Inscription2023Token", + "parent2Inscription2023Token", + "updatedAt", + "lastActivityAt", + "userIps", + "token2FA", + "token2FAExpires", + ], + }), + MODELNAME, + ); +} schema.index({ ligneId: 1 }); schema.index({ sessionPhase1Id: 1 }); diff --git a/api/src/scripts/reindex_es_all_models/index.js b/api/src/scripts/reindex_es_all_models/index.js index 5f3354df32..ea164b77e4 100644 --- a/api/src/scripts/reindex_es_all_models/index.js +++ b/api/src/scripts/reindex_es_all_models/index.js @@ -9,7 +9,6 @@ const esClient = require("../../es"); const { ApplicationModel, - CohortModel, CohesionCenterModel, ContractModel, DepartmentServiceModel, @@ -31,10 +30,10 @@ const { PlanTransportModel, LigneBusModel, LigneToPointModel, - EmailModel, ClasseModel, EtablissementModel, SchoolRAMSESModel, + EmailModel, } = require("../../models"); const StatsYoungCenterModel = require("../../models/legacy/stats-young-center"); @@ -48,6 +47,12 @@ if (args.length > 0) { } async function reindexESAllModels() { + if (!config.get(ENABLE_MONGOOSE_ELASTIC)) { + console.log("MongooseElastic is disabled (see ENABLE_MONGOOSE_ELASTIC). Synchronization is performed by listening to Mongo changestream (see ESDatariver)"); + console.log("Aborting"); + return; + } + try { const all_models = [ ApplicationModel, @@ -77,6 +82,10 @@ async function reindexESAllModels() { ModificationBusModel, LigneBusModel, LigneToPointModel, + PlanTransportModel, + EmailModel, + ClasseModel, + EtablissementModel, SchoolRAMSESModel, ]; @@ -98,7 +107,6 @@ async function reindexESAllModels() { LigneBusModel, PlanTransportModel, EmailModel, - CohortModel, ClasseModel, EtablissementModel, ]; @@ -107,7 +115,6 @@ async function reindexESAllModels() { [ClasseModel.modelName]: ["etablissement"], }; - // const models_indexed = [CohortModel]; // useful_models; let models_indexed = useful_models; if (groupName === "all") { @@ -180,7 +187,7 @@ async function reindexESAllModels() { try { console.log(`Indexing ${index} ${i + 1}/${total}`); bulk.push(doc); - if (bulk.length >= 1000) await flush(); + if (bulk.length >= 10000) await flush(); } catch (e) { console.log("Error", e); } diff --git a/infra/ES_datariver/Dockerfile b/infra/ES_datariver/Dockerfile new file mode 100644 index 0000000000..27e5f61cc3 --- /dev/null +++ b/infra/ES_datariver/Dockerfile @@ -0,0 +1,16 @@ +# Force version to 6.7.14 cause we dont want backoff indexing coming in 6.7.15 +FROM rwynn/monstache:6.7.14 + +ENV MONSTACHE_ES_URLS="" +ENV MONSTACHE_MONGO_URL="" + +ARG MONGO_DB_NAME="" +ENV MONGO_DB_NAME=${MONGO_DB_NAME} + +COPY collections.csv . +COPY generate_config.sh . +COPY docker_start.sh . + +RUN sh generate_config.sh $MONGO_DB_NAME collections.csv > monstache.toml + +ENTRYPOINT ["./docker_start.sh"] diff --git a/infra/ES_datariver/collections.csv b/infra/ES_datariver/collections.csv new file mode 100644 index 0000000000..cc886a462c --- /dev/null +++ b/infra/ES_datariver/collections.csv @@ -0,0 +1,22 @@ +db_collection_name,es_index_name +applications,application +classes,classe +cohesioncenters,cohesioncenter +contracts,contract +departmentservices,departmentservice +emails,email +etablissements,etablissement +inscriptiongoals,inscriptiongoal +lignebuses,lignebus +lignetopoints,lignetopoint +missions,mission +missionapis,missionapi +modificationbuses,modificationbus +plandetransports,plandetransport +pointderassemblements,pointderassemblement +referents,referent +schoolramses,schoolramses +sessionphase1,sessionphase1 +structures,structure +tablederepartitions,tablederepartition +youngs,young diff --git a/infra/ES_datariver/config/index_templates/README.txt b/infra/ES_datariver/config/index_templates/README.txt new file mode 100644 index 0000000000..e52e14198a --- /dev/null +++ b/infra/ES_datariver/config/index_templates/README.txt @@ -0,0 +1 @@ +https://www.elastic.co/guide/en/elasticsearch/reference/current/indices.html diff --git a/infra/ES_datariver/config/index_templates/application.json b/infra/ES_datariver/config/index_templates/application.json new file mode 100644 index 0000000000..e582ab46c9 --- /dev/null +++ b/infra/ES_datariver/config/index_templates/application.json @@ -0,0 +1,267 @@ +{ + "index_patterns": ["application"], + "template": { + "settings": { + "index": { + "number_of_replicas": "0" + } + }, + "mappings": { + "properties": { + "sqlId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "youngBirthdateAt": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "hidden": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "missionRegion": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "createdAt": { + "type": "date" + }, + "youngCohort": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "feedBackExperienceFiles": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "youngLastName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "__v": { + "type": "long" + }, + "youngCity": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "missionName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "youngEmail": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "justificatifsFiles": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "statusComment": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "contractAvenantFiles": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "updatedAt": { + "type": "date" + }, + "youngFirstName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "tutorId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "missionId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "isJvaMission": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "structureId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "priority": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "othersFiles": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "youngDepartment": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "tutorName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "youngId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "contractId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "missionDuration": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "missionDepartment": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "status": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + } + } + } + }, + "composed_of": [] +} diff --git a/infra/ES_datariver/config/index_templates/cohesioncenter.json b/infra/ES_datariver/config/index_templates/cohesioncenter.json new file mode 100644 index 0000000000..51527408b1 --- /dev/null +++ b/infra/ES_datariver/config/index_templates/cohesioncenter.json @@ -0,0 +1,213 @@ +{ + "index_patterns": ["cohesioncenter"], + "template": { + "settings": { + "index": { + "analysis": { + "analyzer": { + "folding": { + "filter": ["lowercase", "asciifolding"], + "tokenizer": "standard" + } + } + }, + "number_of_replicas": "0" + } + }, + "mappings": { + "properties": { + "zip": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "country": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "lastName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + }, + "folded": { + "analyzer": "folding", + "type": "text" + } + } + }, + "address": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "code": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "waitingList": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "city": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "COR": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "departmentCode": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "sessionStatus": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "placesTotal": { + "type": "long" + }, + "createdAt": { + "type": "date" + }, + "pmr": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "__v": { + "type": "long" + }, + "observations": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "code2022": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "addressVerified": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "department": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "region": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "outfitDelivered": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "cohorts": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "placesLeft": { + "type": "long" + }, + "updatedAt": { + "type": "date" + } + } + } + }, + "composed_of": [] +} diff --git a/infra/ES_datariver/config/index_templates/departmentservice.json b/infra/ES_datariver/config/index_templates/departmentservice.json new file mode 100644 index 0000000000..978fa4bd0b --- /dev/null +++ b/infra/ES_datariver/config/index_templates/departmentservice.json @@ -0,0 +1,214 @@ +{ + "index_patterns": ["departmentservice"], + "template": { + "settings": { + "index": { + "number_of_replicas": "0" + } + }, + "mappings": { + "properties": { + "zip": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "address": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "city": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "contactName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "directionName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "createdAt": { + "type": "date" + }, + "complementAddress": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "__v": { + "type": "long" + }, + "representantEtat": { + "type": "nested", + "properties": { + "firstName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "lastName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "role": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "mobile": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "email": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + } + } + }, + "contactMail": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "contactPhone": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "department": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "anonymised": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "contacts": { + "type": "nested", + "properties": { + "contactName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "cohort": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "_id": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "contactMail": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "contactPhone": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + } + } + }, + "updatedAt": { + "type": "date" + } + } + } + }, + "composed_of": [] +} diff --git a/infra/ES_datariver/config/index_templates/lignebus.json b/infra/ES_datariver/config/index_templates/lignebus.json new file mode 100644 index 0000000000..bff4b099f5 --- /dev/null +++ b/infra/ES_datariver/config/index_templates/lignebus.json @@ -0,0 +1,120 @@ +{ + "index_patterns": ["lignebus"], + "template": { + "settings": { + "index": { + "number_of_replicas": "0" + } + }, + "mappings": { + "properties": { + "busId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "meetingPointsIds": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "centerId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "km": { + "type": "long" + }, + "travelTime": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "departuredDate": { + "type": "date" + }, + "youngSeatsTaken": { + "type": "long" + }, + "cohort": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "followerCapacity": { + "type": "long" + }, + "lunchBreak": { + "type": "boolean" + }, + "sessionId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "centerDepartureTime": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "createdAt": { + "type": "date" + }, + "returnDate": { + "type": "date" + }, + "youngCapacity": { + "type": "long" + }, + "totalCapacity": { + "type": "long" + }, + "centerArrivalTime": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "lunchBreakReturn": { + "type": "boolean" + }, + "updatedAt": { + "type": "date" + } + } + } + }, + "composed_of": [] +} diff --git a/infra/ES_datariver/config/index_templates/mission.json b/infra/ES_datariver/config/index_templates/mission.json new file mode 100644 index 0000000000..3f14028926 --- /dev/null +++ b/infra/ES_datariver/config/index_templates/mission.json @@ -0,0 +1,386 @@ +{ + "index_patterns": ["mission"], + "template": { + "settings": { + "index": { + "analysis": { + "analyzer": { + "folding": { + "filter": ["lowercase", "asciifolding"], + "tokenizer": "standard" + } + } + }, + "number_of_replicas": "0", + "default_pipeline": "mission" + } + }, + "mappings": { + "properties": { + "sqlId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "country": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "lastName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + }, + "folded": { + "analyzer": "folding", + "type": "text" + } + } + }, + "city": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + }, + "folded": { + "analyzer": "folding", + "type": "text" + } + } + }, + "justifications": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "lastSyncAt": { + "type": "date" + }, + "description": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "mainDomain": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "isMilitaryPreparation": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "remote": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "duration": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "createdAt": { + "type": "date" + }, + "jvaMissionId": { + "type": "long" + }, + "__v": { + "type": "long" + }, + "sqlStructureId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "frequence": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "statusComment": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "addressVerified": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "department": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "contraintes": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "placesLeft": { + "type": "long" + }, + "startAt": { + "type": "date" + }, + "updatedAt": { + "type": "date" + }, + "zip": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "pendingApplications": { + "type": "long" + }, + "tutorId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "period": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "address": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "visibility": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "structureName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + }, + "folded": { + "analyzer": "folding", + "type": "text" + } + } + }, + "format": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "domains": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "isJvaMission": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "structureId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "placesTotal": { + "type": "long" + }, + "endAt": { + "type": "date" + }, + "firstName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + }, + "folded": { + "analyzer": "folding", + "type": "text" + } + } + }, + "tutorName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "subPeriod": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "placesTaken": { + "type": "long" + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + }, + "folded": { + "analyzer": "folding", + "type": "text" + } + } + }, + "sqlTutorId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "location": { + "type": "geo_point" + }, + "region": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "actions": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "status": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + } + } + } + }, + "composed_of": [] +} diff --git a/infra/ES_datariver/config/index_templates/missionapi.json b/infra/ES_datariver/config/index_templates/missionapi.json new file mode 100644 index 0000000000..c2f07d3e61 --- /dev/null +++ b/infra/ES_datariver/config/index_templates/missionapi.json @@ -0,0 +1,192 @@ +{ + "index_patterns": ["missionapi"], + "template": { + "settings": { + "index": { + "number_of_replicas": "0" + } + }, + "mappings": { + "properties": { + "country": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "city": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "lastSyncAt": { + "type": "date" + }, + "departmentCode": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "postalCode": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "description": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "remote": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "title": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "createdAt": { + "type": "date" + }, + "__v": { + "type": "long" + }, + "startAt": { + "type": "date" + }, + "updatedAt": { + "type": "date" + }, + "departmentName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "organizationName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "format": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "endAt": { + "type": "date" + }, + "publisherUrl": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "places": { + "type": "long" + }, + "publisherName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "domain": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "adresse": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "applicationUrl": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "location": { + "type": "geo_point" + }, + "region": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "status": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + } + } + } + }, + "composed_of": [] +} diff --git a/infra/ES_datariver/config/index_templates/modificationbus.json b/infra/ES_datariver/config/index_templates/modificationbus.json new file mode 100644 index 0000000000..b7bb43fc39 --- /dev/null +++ b/infra/ES_datariver/config/index_templates/modificationbus.json @@ -0,0 +1,182 @@ +{ + "index_patterns": ["modificationbus"], + "template": { + "settings": { + "index": { + "number_of_replicas": "0" + } + }, + "mappings": { + "properties": { + "date": { + "type": "date" + }, + "statusDate": { + "type": "date" + }, + "tagIds": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "lineId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "lineName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "requestUserId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "requestUserName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "opinion": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "requestMessage": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "createdAt": { + "type": "date" + }, + "statusUserId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "opinionDate": { + "type": "date" + }, + "opinionUserName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "messages": { + "type": "object", + "properties": { + "date": { + "type": "date" + }, + "message": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "userName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "userId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + } + } + }, + "opinionUserId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "statusUserName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "requestUserRole": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "status": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "updatedAt": { + "type": "date" + } + } + } + }, + "composed_of": [] +} diff --git a/infra/ES_datariver/config/index_templates/plandetransport.json b/infra/ES_datariver/config/index_templates/plandetransport.json new file mode 100644 index 0000000000..01e1eb4371 --- /dev/null +++ b/infra/ES_datariver/config/index_templates/plandetransport.json @@ -0,0 +1,478 @@ +{ + "index_patterns": ["plandetransport"], + "template": { + "settings": { + "index": { + "number_of_replicas": "0" + } + }, + "mappings": { + "properties": { + "busId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "centerId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "centerRegion": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "travelTime": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "pointDeRassemblements": { + "type": "object", + "properties": { + "zip": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "address": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "code": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "city": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "meetingHour": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "copyId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "createdAt": { + "type": "date" + }, + "__t": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "complementAddress": { + "type": "object", + "properties": { + "cohort": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "complement": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + } + } + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "location": { + "type": "object", + "properties": { + "lon": { + "type": "float" + }, + "lat": { + "type": "float" + } + } + }, + "department": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "region": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "cohorts": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "updatedAt": { + "type": "date" + } + } + }, + "modificationBuses": { + "type": "object", + "properties": { + "statusDate": { + "type": "date" + }, + "tagIds": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "cohort": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "lineId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "lineName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "requestUserId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "requestUserName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "opinion": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "requestMessage": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "createdAt": { + "type": "date" + }, + "statusUserId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "opinionDate": { + "type": "date" + }, + "opinionUserName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "messages": { + "type": "object", + "properties": { + "date": { + "type": "date" + }, + "message": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "userName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "userId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + } + } + }, + "opinionUserId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "statusUserName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "requestUserRole": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "status": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "updatedAt": { + "type": "date" + } + } + }, + "cohort": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "followerCapacity": { + "type": "long" + }, + "lunchBreak": { + "type": "boolean" + }, + "fillingRate": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "returnString": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "centerDepartureTime": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "createdAt": { + "type": "date" + }, + "youngCapacity": { + "type": "long" + }, + "centerDepartment": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "departureString": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "totalCapacity": { + "type": "long" + }, + "youngCount": { + "type": "long" + }, + "centerCode": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "centerArrivalTime": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "lunchBreakReturn": { + "type": "boolean" + }, + "centerName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "updatedAt": { + "type": "date" + } + } + } + }, + "composed_of": [] +} diff --git a/infra/ES_datariver/config/index_templates/pointderassemblement.json b/infra/ES_datariver/config/index_templates/pointderassemblement.json new file mode 100644 index 0000000000..d861dd9f06 --- /dev/null +++ b/infra/ES_datariver/config/index_templates/pointderassemblement.json @@ -0,0 +1,130 @@ +{ + "index_patterns": ["pointderassemblement"], + "template": { + "settings": { + "index": { + "number_of_replicas": "0" + } + }, + "mappings": { + "properties": { + "zip": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "address": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "code": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "city": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "createdAt": { + "type": "date" + }, + "deletedAt": { + "type": "date" + }, + "complementAddress": { + "type": "object", + "properties": { + "cohort": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "complement": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + } + } + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "location": { + "type": "object", + "properties": { + "lon": { + "type": "float" + }, + "lat": { + "type": "float" + } + } + }, + "department": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "region": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "cohorts": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "updatedAt": { + "type": "date" + } + } + } + }, + "composed_of": [] +} diff --git a/infra/ES_datariver/config/index_templates/referent.json b/infra/ES_datariver/config/index_templates/referent.json new file mode 100644 index 0000000000..7f1b39dfb8 --- /dev/null +++ b/infra/ES_datariver/config/index_templates/referent.json @@ -0,0 +1,179 @@ +{ + "index_patterns": ["referent"], + "template": { + "settings": { + "index": { + "analysis": { + "analyzer": { + "folding": { + "filter": ["lowercase", "asciifolding"], + "tokenizer": "standard" + } + } + }, + "number_of_replicas": "0", + "default_pipeline": "referent" + } + }, + "mappings": { + "properties": { + "sqlId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "lastName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + }, + "folded": { + "analyzer": "folding", + "type": "text" + } + } + }, + "role": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "registredAt": { + "type": "date" + }, + "mobile": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "structureId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "subRole": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "acceptCGU": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "createdAt": { + "type": "date" + }, + "firstName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + }, + "folded": { + "analyzer": "folding", + "type": "text" + } + } + }, + "sessionPhase1Id": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "cohesionCenterId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "lastLoginAt": { + "type": "date" + }, + "phone": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "__v": { + "type": "long" + }, + "department": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "region": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "cohesionCenterName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "email": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + } + } + } + }, + "composed_of": [] +} diff --git a/infra/ES_datariver/config/index_templates/schoolramses.json b/infra/ES_datariver/config/index_templates/schoolramses.json new file mode 100644 index 0000000000..40aba68afc --- /dev/null +++ b/infra/ES_datariver/config/index_templates/schoolramses.json @@ -0,0 +1,138 @@ +{ + "index_patterns": ["schoolramses"], + "template": { + "settings": { + "index": { + "analysis": { + "analyzer": { + "folding": { + "filter": ["lowercase", "asciifolding"], + "tokenizer": "standard" + } + } + }, + "number_of_replicas": "0", + "default_pipeline": "schoolramses" + } + }, + "mappings": { + "properties": { + "departmentName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + }, + "folded": { + "analyzer": "folding", + "type": "text" + } + } + }, + "country": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + }, + "folded": { + "analyzer": "folding", + "type": "text" + } + } + }, + "createdAt": { + "type": "date" + }, + "city": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + }, + "folded": { + "analyzer": "folding", + "type": "text" + } + } + }, + "__v": { + "type": "long" + }, + "adresse": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + }, + "folded": { + "analyzer": "folding", + "type": "text" + } + } + }, + "postcode": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "fullName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + }, + "folded": { + "analyzer": "folding", + "type": "text" + } + } + }, + "region": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + }, + "folded": { + "analyzer": "folding", + "type": "text" + } + } + }, + "type": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "uai": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "updatedAt": { + "type": "date" + } + } + } + }, + "composed_of": [] +} diff --git a/infra/ES_datariver/config/index_templates/sessionphase1.json b/infra/ES_datariver/config/index_templates/sessionphase1.json new file mode 100644 index 0000000000..18c6bfbe68 --- /dev/null +++ b/infra/ES_datariver/config/index_templates/sessionphase1.json @@ -0,0 +1,82 @@ +{ + "index_patterns": ["sessionphase1"], + "template": { + "settings": { + "index": { + "number_of_replicas": "0", + "default_pipeline": "sessionphase1" + } + }, + "mappings": { + "properties": { + "headCenterId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "pedagoProjectFiles": { + "type": "nested" + }, + "createdAt": { + "type": "date" + }, + "cohesionCenterId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "waitingList": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "__v": { + "type": "long" + }, + "timeScheduleFiles": { + "type": "nested" + }, + "cohort": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "placesTotal": { + "type": "long" + }, + "placesLeft": { + "type": "long" + }, + "status": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "updatedAt": { + "type": "date" + } + } + } + }, + "composed_of": [] +} diff --git a/infra/ES_datariver/config/index_templates/structure.json b/infra/ES_datariver/config/index_templates/structure.json new file mode 100644 index 0000000000..56e3b76c50 --- /dev/null +++ b/infra/ES_datariver/config/index_templates/structure.json @@ -0,0 +1,330 @@ +{ + "index_patterns": ["structure"], + "template": { + "settings": { + "index": { + "analysis": { + "analyzer": { + "folding": { + "filter": ["lowercase", "asciifolding"], + "tokenizer": "standard" + } + } + }, + "number_of_replicas": "0", + "default_pipeline": "structure" + } + }, + "mappings": { + "properties": { + "sqlId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "country": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "lastName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + }, + "folded": { + "analyzer": "folding", + "type": "text" + } + } + }, + "structurePriveeType": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "city": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "structurePubliqueEtatType": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "networkName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "structurePubliqueType": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "description": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "isMilitaryPreparation": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "sousType": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "instagram": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "siret": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "createdAt": { + "type": "date" + }, + "isJvaStructure": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "twitter": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "__v": { + "type": "long" + }, + "networkId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "sqlNetworkId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "state": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "department": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "updatedAt": { + "type": "date" + }, + "zip": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "associationTypes": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "jvaStructureId": { + "type": "long" + }, + "types": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "website": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "address": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "facebook": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "sqlUserId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "legalStatus": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "isNetwork": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "name": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "location": { + "type": "object", + "properties": { + "lon": { + "type": "float" + }, + "lat": { + "type": "float" + } + } + }, + "region": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "status": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + } + } + } + }, + "composed_of": [] +} diff --git a/infra/ES_datariver/config/index_templates/young.json b/infra/ES_datariver/config/index_templates/young.json new file mode 100644 index 0000000000..8fc5c26ba4 --- /dev/null +++ b/infra/ES_datariver/config/index_templates/young.json @@ -0,0 +1,1941 @@ +{ + "index_patterns": ["young"], + "template": { + "settings": { + "index": { + "max_result_window": "10000", + "default_pipeline": "young", + "analysis": { + "analyzer": { + "uax": { + "filter": ["lowercase"], + "tokenizer": "uax_url_email" + }, + "folding": { + "filter": ["lowercase", "asciifolding"], + "tokenizer": "standard" + } + } + }, + "number_of_replicas": "0" + } + }, + "mappings": { + "properties": { + "imageRightFiles": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "phase2NumberHoursEstimated": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "phase2ApplicationStatus": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "inscriptionStep": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "professionnalProject": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "defenseInterest": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent2Location": { + "type": "object", + "properties": { + "lon": { + "type": "float" + }, + "lat": { + "type": "float" + } + } + }, + "cohesionCenterName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "environmentInterest": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "healthInterest": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "hostLastName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "originalCohort": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent2FromFranceConnect": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "highSkilledActivityProofFiles": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "cohesionStayMedicalFileDownload": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent1Country": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "sessionPhase1Id": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "hostFirstName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "qpv": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "grade": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "mobilityNearSchool": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "medicosocialStructureCity": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent2OwnAddress": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "solidarityInterest": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "rulesYoung": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "mobilityNearHome": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent1Address": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "phase3MissionDomain": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "phase3MissionEndAt": { + "type": "date" + }, + "schoolName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "email": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "situation": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "meetingPointId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "rulesParent1": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "address": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "militaryPreparationFilesIdentity": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent2Zip": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "rulesParent2": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "missionFormat": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "jdc": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "phase3MissionDescription": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "codeCenterTmp": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "autoTestPCR": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "lastStatusAt": { + "type": "date" + }, + "consentment": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "country": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "notes": { + "type": "nested", + "properties": { + "phase": { + "type": "text" + }, + "createdAt": { + "type": "date" + }, + "note": { + "type": "text" + }, + "referent": { + "type": "object", + "properties": { + "firstName": { + "type": "text" + }, + "lastName": { + "type": "text" + }, + "role": { + "type": "text" + }, + "_id": { + "type": "text" + } + } + }, + "updatedAt": { + "type": "date" + } + } + }, + "statusPhase2ValidatedAt": { + "type": "date" + }, + "statusPhase2Contract": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "schoolType": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "engagedDescription": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "frenchNationality": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "createdAt": { + "type": "date" + }, + "ppsBeneficiary": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent2Phone": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "departSejourMotif": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "__v": { + "type": "long" + }, + "parentConsentment": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "birthCountry": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "rulesFilesStatus": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "handicapInSameDepartment": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "reducedMobilityAccess": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "medicosocialStructureLocation": { + "type": "object", + "properties": { + "lon": { + "type": "float" + }, + "lat": { + "type": "float" + } + } + }, + "populationDensity": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "phase3StructureName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "firstName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + }, + "folded": { + "analyzer": "folding", + "type": "text" + } + } + }, + "periodRanking": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "lastLoginAt": { + "type": "date" + }, + "withdrawnReason": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "complementAddress": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "militaryPreparationFilesCensus": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "foreignZip": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "region": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "professionnalProjectPrecision": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "allergies": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "schoolCity": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "imageRight": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "cityCode": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "cohort": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "schoolDepartment": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "dataProcessingConsentmentFiles": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "departSejourMotifComment": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "informationAccuracy": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent1FromFranceConnect": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "sessionPhase1IdTmp": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "addressVerified": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent1OwnAddress": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "autoAffectationPhase1ExpiresAt": { + "type": "date" + }, + "cohesionCenterZip": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "mobilityNearRelative": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "cultureInterest": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "mobilityNearRelativeCity": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "cohesion2020Step": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "location": { + "type": "geo_point" + }, + "statusPhase1Tmp": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "phase3TutorNote": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "birthCityZip": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "sqlId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "cohortDetailedChangeReason": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "cohesionCenterCity": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent2LastName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "hostRegion": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "medicosocialStructureRegion": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "mobilityNearRelativeName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "mobilityNearRelativeAddress": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "foreignCountry": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "schoolZip": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "autoTestPCRFiles": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "phase": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "zip": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "mobilityNearRelativeZip": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "period": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "hostZip": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "birthCity": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "schoolAddress": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "departSejourAt": { + "type": "date" + }, + "militaryPreparationFilesAuthorization": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "phase3TutorLastName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "cohesionCenterId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "phone": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent1City": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "autoTestPCRFilesStatus": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent1Region": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "status_equivalence": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "foreignAddress": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "hostCity": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "status": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "engaged": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "lastName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + }, + "folded": { + "analyzer": "folding", + "type": "text" + } + } + }, + "gender": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "city": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + }, + "folded": { + "analyzer": "folding", + "type": "text" + } + } + }, + "motivations": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "cohesionStayMedicalFileReceived": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent2Address": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent2Region": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "schoolId": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "highSkilledActivity": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "medicosocialStructure": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "inscriptionCorrectionMessage": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "medicosocialStructureName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "academy": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "highSkilledActivityType": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent2FirstName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "paiBeneficiary": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "citizenshipInterest": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "departInform": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "presenceJDM": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent2Country": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "statusPhase2UpdatedAt": { + "type": "date" + }, + "statusPhase3ValidatedAt": { + "type": "date" + }, + "hostAddress": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "phase3TutorEmail": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "autoTestPCRFilesComment": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent1Zip": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "cniFiles": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "mobilityTransport": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parentConsentmentFilesCompliantInfo": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "withdrawnMessage": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "highSkilledActivityInSameDepartment": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "employed": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "medicosocialStructureZip": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "hostRelationship": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "busTmp": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "mobilityTransportOther": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent2Department": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "phase2NumberHoursDone": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "imageRightFilesStatus": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent1FirstName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent2City": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "hostDepartment": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "handicap": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "domains": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "securityInterest": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "statusPhase2": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "acceptCGU": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "statusPhase1": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "statusPhase3": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "medicosocialStructureDepartment": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "convocationFileDownload": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "militaryPreparationFilesCertificate": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "engagedStructure": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent1Phone": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "schoolRegion": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent1Status": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "statusMilitaryPreparationFiles": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parentConsentmentFiles": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "phase3TutorFirstName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent1Department": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "specificAmenagment": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "schoolCountry": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "desiredLocation": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "phase3MissionStartAt": { + "type": "date" + }, + "phase3TutorPhone": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "imageRightFilesComment": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent2Email": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent1Email": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "cohesionStayPresence": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent2Status": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "deplacementPhase1Autonomous": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "rulesFiles": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent1Location": { + "type": "object", + "properties": { + "lon": { + "type": "float" + }, + "lat": { + "type": "float" + } + } + }, + "department": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parent1LastName": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "statusPhase3UpdatedAt": { + "type": "date" + }, + "educationInterest": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "birthdateAt": { + "type": "date" + }, + "statusPhase1Motif": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "youngPhase1Agreement": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "schooled": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "medicosocialStructureAddress": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "sportInterest": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "specificAmenagmentType": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "aknowledgmentTerminaleSessionAvailability": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "parentConsentmentFilesCompliant": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "foreignCity": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "cohortChangeReason": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + }, + "inscriptionRefusedMessage": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + } + } + } + } + }, + "composed_of": [] +} diff --git a/infra/ES_datariver/config/index_whitelist.txt b/infra/ES_datariver/config/index_whitelist.txt new file mode 100644 index 0000000000..8528ee3343 --- /dev/null +++ b/infra/ES_datariver/config/index_whitelist.txt @@ -0,0 +1,21 @@ +application +classe +cohesioncenter +contract +departmentservice +email +etablissement +inscriptiongoal +lignebus +lignetopoint +mission +missionapi +modificationbus +plandetransport +pointderassemblement +referent +schoolramses +sessionphase1 +structure +tablederepartition +young diff --git a/infra/ES_datariver/config/ingest_pipelines/README.txt b/infra/ES_datariver/config/ingest_pipelines/README.txt new file mode 100644 index 0000000000..5bd55a920b --- /dev/null +++ b/infra/ES_datariver/config/ingest_pipelines/README.txt @@ -0,0 +1 @@ +https://www.elastic.co/guide/en/elasticsearch/reference/current/ingest-apis.html diff --git a/infra/ES_datariver/config/ingest_pipelines/mission.json b/infra/ES_datariver/config/ingest_pipelines/mission.json new file mode 100644 index 0000000000..e67aaaf1c2 --- /dev/null +++ b/infra/ES_datariver/config/ingest_pipelines/mission.json @@ -0,0 +1,11 @@ +{ + "description": "mission ignore fields", + "processors": [ + { + "remove": { + "field": " jvaRawData", + "ignore_missing": true + } + } + ] +} diff --git a/infra/ES_datariver/config/ingest_pipelines/referent.json b/infra/ES_datariver/config/ingest_pipelines/referent.json new file mode 100644 index 0000000000..0be6ba1b94 --- /dev/null +++ b/infra/ES_datariver/config/ingest_pipelines/referent.json @@ -0,0 +1,26 @@ +{ + "description": "referent ignore fields", + "processors": [ + { + "remove": { + "field": [ + "password", + "lastLogoutAt", + "passwordChangedAt", + "nextLoginAttemptIn", + "forgotPasswordResetToken", + "forgotPasswordResetExpires", + "invitationToken", + "invitationExpires", + "loginAttempts", + "attempts2FA", + "updatedAt", + "lastActivityAt", + "token2FA", + "token2FAExpires" + ], + "ignore_missing": true + } + } + ] +} diff --git a/infra/ES_datariver/config/ingest_pipelines/schoolramses.json b/infra/ES_datariver/config/ingest_pipelines/schoolramses.json new file mode 100644 index 0000000000..5c8b638462 --- /dev/null +++ b/infra/ES_datariver/config/ingest_pipelines/schoolramses.json @@ -0,0 +1,11 @@ +{ + "description": "schoolramses ignore fields", + "processors": [ + { + "remove": { + "field": "raw_data", + "ignore_missing": true + } + } + ] +} diff --git a/infra/ES_datariver/config/ingest_pipelines/sessionphase1.json b/infra/ES_datariver/config/ingest_pipelines/sessionphase1.json new file mode 100644 index 0000000000..e767e7481b --- /dev/null +++ b/infra/ES_datariver/config/ingest_pipelines/sessionphase1.json @@ -0,0 +1,11 @@ +{ + "description": "sessionphase1 ignore fields", + "processors": [ + { + "remove": { + "field": "team", + "ignore_missing": true + } + } + ] +} diff --git a/infra/ES_datariver/config/ingest_pipelines/structure.json b/infra/ES_datariver/config/ingest_pipelines/structure.json new file mode 100644 index 0000000000..4c003b15c0 --- /dev/null +++ b/infra/ES_datariver/config/ingest_pipelines/structure.json @@ -0,0 +1,11 @@ +{ + "description": "structure ignore fields", + "processors": [ + { + "remove": { + "field": "jvaRawData", + "ignore_missing": true + } + } + ] +} diff --git a/infra/ES_datariver/config/ingest_pipelines/young.json b/infra/ES_datariver/config/ingest_pipelines/young.json new file mode 100644 index 0000000000..e99c94f329 --- /dev/null +++ b/infra/ES_datariver/config/ingest_pipelines/young.json @@ -0,0 +1,30 @@ +{ + "description": "young ignore fields", + "processors": [ + { + "remove": { + "field": [ + "historic", + "missionsInMail", + "password", + "lastLogoutAt", + "passwordChangedAt", + "nextLoginAttemptIn", + "forgotPasswordResetToken", + "forgotPasswordResetExpires", + "invitationExpires", + "phase3Token", + "loginAttempts", + "parent1Inscription2023Token", + "parent2Inscription2023Token", + "updatedAt", + "lastActivityAt", + "userIps", + "token2FA", + "token2FAExpires" + ], + "ignore_missing": true + } + } + ] +} diff --git a/infra/ES_datariver/docker_start.sh b/infra/ES_datariver/docker_start.sh new file mode 100755 index 0000000000..0639d29d48 --- /dev/null +++ b/infra/ES_datariver/docker_start.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +set -e + +monstache -f monstache.toml -worker default --enable-http-server & +monstache -f monstache.toml -worker young & + +# Wait for any process to exit +wait -n + +# Exit with status of process that exited first +exit $? diff --git a/infra/ES_datariver/generate_config.sh b/infra/ES_datariver/generate_config.sh new file mode 100755 index 0000000000..2ae5efde13 --- /dev/null +++ b/infra/ES_datariver/generate_config.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +set -e + + +if [ "$#" -lt 2 ]; then + echo "Generate monstache configuration" + echo "Usage $0 " + echo " db_name: Source database name" + echo " file: Collection mapping filename" + exit 1 +fi + +db_name=$1 +col_file=$2 + +if [[ $db_name == "" ]] +then + echo "You must specify the database name" + exit 1 +fi + +if [[ ! -f "$col_file" ]] +then + echo "file mut be a valid filename" + exit 1 +fi + +collections=$(tail -n +2 $col_file | cut -d "," -f 1 | sed "s/^\(.*\)$/\"$db_name.\1\",/g") + +cat <