diff --git a/docker-compose.yml b/docker-compose.yml index c758944ee..690fd4343 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -282,7 +282,7 @@ services: # AUTH_JWT_KEY: ${ENCRYPTION_KEY} # AUTH_JWT_AUD: ${FMTM_DOMAIN} ports: - - "7055:7055" + - "7055:3000" networks: - fmtm-net restart: "unless-stopped" diff --git a/src/backend/app/db/models.py b/src/backend/app/db/models.py index 6ef90816b..527397444 100644 --- a/src/backend/app/db/models.py +++ b/src/backend/app/db/models.py @@ -41,6 +41,7 @@ from app.db.enums import ( BackgroundTaskStatus, CommunityType, + EntityState, HTTPStatus, MappingLevel, MappingState, @@ -842,7 +843,7 @@ async def create( Args: db (Connection): The database connection. - project_id (int): The organisation ID. + project_id (int): The project ID. tasks (geojson.FeatureCollection): FeatureCollection of task areas. Returns: @@ -1296,6 +1297,79 @@ async def delete(cls, db: Connection, project_id: int) -> bool: ) +class DbOdkEntities(BaseModel): + """Table odk_entities. + + Mirror tracking the status of Entities in ODK. + """ + + entity_id: UUID + status: EntityState + project_id: int + task_id: int + + @classmethod + async def upsert( + cls, + db: Connection, + project_id: int, + entities: list[Self], + ) -> bool: + """Update or insert Entity data, with statuses. + + Args: + db (Connection): The database connection. + project_id (int): The project ID. + entities (list[Self]): List of DbOdkEntities objects. + + Returns: + bool: Success or failure. + """ + log.info( + f"Updating FMTM database Entities for project {project_id} " + f"with ({len(entities)}) features" + ) + + sql = """ + INSERT INTO public.odk_entities + (entity_id, status, project_id, task_id) + VALUES + """ + + # Prepare data for bulk insert + values = [] + data = {} + for index, entity in enumerate(entities): + entity_index = f"entity_{index}" + values.append( + f"(%({entity_index}_entity_id)s, " + f"%({entity_index}_status)s, " + f"%({entity_index}_project_id)s, " + f"%({entity_index}_task_id)s)" + ) + data[f"{entity_index}_entity_id"] = entity["id"] + data[f"{entity_index}_status"] = EntityState(int(entity["status"])).name + data[f"{entity_index}_project_id"] = project_id + task_id = entity["task_id"] + data[f"{entity_index}_task_id"] = int(task_id) if task_id else None + + sql += ( + ", ".join(values) + + """ + ON CONFLICT (entity_id) DO UPDATE SET + status = EXCLUDED.status, + task_id = EXCLUDED.task_id + RETURNING True; + """ + ) + + async with db.cursor() as cur: + await cur.execute(sql, data) + result = await cur.fetchall() + + return bool(result) + + class DbBackgroundTask(BaseModel): """Table background_tasks. diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 254cb1982..c33425319 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -532,8 +532,8 @@ async def generate_project_files( # Split extract by task area log.debug("Splitting data extract per task area") - # TODO in future this splitting could be removed as the task_id is - # no longer used in the XLSForm + # TODO in future this splitting could be removed if the task_id is + # no longer used in the XLSForm for the map filter task_extract_dict = await split_geojson_by_task_areas( db, feature_collection, project_id ) diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index 9e16d025d..6873a37e0 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -57,7 +57,14 @@ ProjectRole, XLSFormType, ) -from app.db.models import DbBackgroundTask, DbBasemap, DbProject, DbTask, DbUserRole +from app.db.models import ( + DbBackgroundTask, + DbBasemap, + DbOdkEntities, + DbProject, + DbTask, + DbUserRole, +) from app.db.postgis_utils import ( check_crs, featcol_keep_single_geom_type, @@ -174,14 +181,20 @@ async def get_odk_entities_geojson( response_model=list[central_schemas.EntityMappingStatus], ) async def get_odk_entities_mapping_statuses( + project_id: int, project: Annotated[DbProject, Depends(project_deps.get_project)], db: Annotated[Connection, Depends(db_conn)], ): """Get the ODK entities mapping statuses, i.e. in progress or complete.""" - return await central_crud.get_entities_data( + entities = await central_crud.get_entities_data( project.odk_credentials, project.odkid, ) + # First update the Entity statuses in the db + # FIXME this is a hack and in the long run should be replaced + # https://github.com/hotosm/fmtm/issues/1841 + await DbOdkEntities.upsert(db, project_id, entities) + return entities @router.get( diff --git a/src/backend/migrations/009-entity-table.sql b/src/backend/migrations/009-entity-table.sql new file mode 100644 index 000000000..8d893d2d9 --- /dev/null +++ b/src/backend/migrations/009-entity-table.sql @@ -0,0 +1,36 @@ +-- ## Migration to: +-- * Add a new table that syncs the ODK Entity status to FMTM. +-- * Add a primary key for entity_id field. +-- * Add two indexes on entity_id + project_id / task_id + + +-- Start a transaction +BEGIN; + +CREATE TABLE IF NOT EXISTS public.odk_entities ( + entity_id UUID NOT NULL, + status entitystate NOT NULL, + project_id integer NOT NULL, + task_id integer +); +ALTER TABLE public.odk_entities OWNER TO fmtm; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'odk_entities_pkey') THEN + ALTER TABLE ONLY public.odk_entities + ADD CONSTRAINT odk_entities_pkey PRIMARY KEY (entity_id); + END IF; +END $$; + +CREATE INDEX IF NOT EXISTS idx_entities_project_id +ON public.odk_entities USING btree ( + entity_id, project_id +); +CREATE INDEX IF NOT EXISTS idx_entities_task_id +ON public.odk_entities USING btree ( + entity_id, task_id +); + +-- Commit the transaction +COMMIT; diff --git a/src/backend/migrations/init/fmtm_base_schema.sql b/src/backend/migrations/init/fmtm_base_schema.sql index e295b56ee..d37230eac 100644 --- a/src/backend/migrations/init/fmtm_base_schema.sql +++ b/src/backend/migrations/init/fmtm_base_schema.sql @@ -160,7 +160,7 @@ SET default_table_access_method = heap; -- Tables -CREATE TABLE IF NOT EXISTS public._migrations ( +CREATE TABLE public._migrations ( script_name text, date_executed timestamp with time zone ); @@ -339,6 +339,14 @@ CREATE TABLE public.users ( ); ALTER TABLE public.users OWNER TO fmtm; +CREATE TABLE public.odk_entities ( + entity_id UUID NOT NULL, + status entitystate NOT NULL, + project_id integer NOT NULL, + task_id integer +); +ALTER TABLE public.odk_entities OWNER TO fmtm; + CREATE TABLE public.xlsforms ( id integer NOT NULL, title character varying, @@ -435,6 +443,9 @@ ADD CONSTRAINT users_pkey PRIMARY KEY (id); ALTER TABLE ONLY public.users ADD CONSTRAINT users_username_key UNIQUE (username); +ALTER TABLE ONLY public.odk_entities +ADD CONSTRAINT odk_entities_pkey PRIMARY KEY (entity_id); + ALTER TABLE ONLY public.xlsforms ADD CONSTRAINT xlsforms_pkey PRIMARY KEY (id); @@ -447,16 +458,16 @@ ADD CONSTRAINT submission_photos_pkey PRIMARY KEY (id); -- Indexing CREATE INDEX idx_projects_outline ON public.projects USING gist (outline); -CREATE INDEX IF NOT EXISTS idx_projects_mapper_level +CREATE INDEX idx_projects_mapper_level ON public.projects USING btree ( mapper_level ); -CREATE INDEX IF NOT EXISTS idx_projects_organisation_id +CREATE INDEX idx_projects_organisation_id ON public.projects USING btree ( organisation_id ); CREATE INDEX idx_tasks_outline ON public.tasks USING gist (outline); -CREATE INDEX IF NOT EXISTS idx_tasks_composite +CREATE INDEX idx_tasks_composite ON public.tasks USING btree ( id, project_id ); @@ -466,26 +477,34 @@ CREATE INDEX idx_user_roles ON public.user_roles USING btree ( CREATE INDEX idx_org_managers ON public.organisation_managers USING btree ( user_id, organisation_id ); -CREATE INDEX IF NOT EXISTS idx_task_event_composite +CREATE INDEX idx_task_event_composite ON public.task_events USING btree ( task_id, project_id ); -CREATE INDEX IF NOT EXISTS idx_task_event_project_user +CREATE INDEX idx_task_event_project_user ON public.task_events USING btree ( user_id, project_id ); -CREATE INDEX IF NOT EXISTS idx_task_event_project_id +CREATE INDEX idx_task_event_project_id ON public.task_events USING btree ( task_id, project_id ); -CREATE INDEX IF NOT EXISTS idx_task_event_user_id +CREATE INDEX idx_task_event_user_id ON public.task_events USING btree ( task_id, user_id ); -CREATE INDEX IF NOT EXISTS idx_task_history_date +CREATE INDEX idx_task_history_date ON public.task_history USING btree ( task_id, created_at ); +CREATE INDEX idx_entities_project_id +ON public.odk_entities USING btree ( + entity_id, project_id +); +CREATE INDEX idx_entities_task_id +ON public.odk_entities USING btree ( + entity_id, task_id +); -- Foreign keys diff --git a/src/mapper/src/lib/components/page/layer-switcher.svelte b/src/mapper/src/lib/components/page/layer-switcher.svelte index 22a15c412..570d93376 100644 --- a/src/mapper/src/lib/components/page/layer-switcher.svelte +++ b/src/mapper/src/lib/components/page/layer-switcher.svelte @@ -1,5 +1,5 @@ diff --git a/src/mapper/src/lib/components/page/legend.svelte b/src/mapper/src/lib/components/page/legend.svelte index 91ca4a235..8a495875d 100644 --- a/src/mapper/src/lib/components/page/legend.svelte +++ b/src/mapper/src/lib/components/page/legend.svelte @@ -1,7 +1,7 @@