From 1f4c9520c5b15bcdf09e0260b13edf76c0f710b3 Mon Sep 17 00:00:00 2001 From: Mekhti Date: Tue, 7 May 2024 11:15:46 -0700 Subject: [PATCH] Fix for PR decision indexing --- .../init_planning_review_decisions.py | 29 +++++++++++++++---- .../planning_review_decisions_insert.sql | 3 +- .../planning-review-decision.controller.ts | 26 ++++++++--------- .../planning-review-decision.service.ts | 2 +- 4 files changed, 39 insertions(+), 21 deletions(-) diff --git a/bin/migrate-oats-data/planning_review/decisions/init_planning_review_decisions.py b/bin/migrate-oats-data/planning_review/decisions/init_planning_review_decisions.py index a9bab23e14..5cc7f90a7a 100644 --- a/bin/migrate-oats-data/planning_review/decisions/init_planning_review_decisions.py +++ b/bin/migrate-oats-data/planning_review/decisions/init_planning_review_decisions.py @@ -5,6 +5,8 @@ add_timezone_and_keep_date_part, OatsToAlcsDecisionOutcomes, AlcsPlanningReviewOutcomes, + to_alcs_format, + get_now_with_offset, ) from db import inject_conn_pool from psycopg2.extras import RealDictCursor, execute_batch @@ -39,6 +41,8 @@ def process_planning_review_decisions(conn=None, batch_size=BATCH_UPLOAD_SIZE): failed_inserts_count = 0 successful_inserts_count = 0 last_planning_decision_id = 0 + last_file_number = 0 + last_decision_date = "" with open( "planning_review/sql/decisions/planning_review_decisions_insert.sql", @@ -47,10 +51,16 @@ def process_planning_review_decisions(conn=None, batch_size=BATCH_UPLOAD_SIZE): ) as sql_file: query = sql_file.read() while True: + query_condition = "" + if last_decision_date: + query_condition = f"WHERE pr.file_number::bigint >= {last_file_number} AND opd.decision_date >= {last_decision_date} AND opd.planning_decision_id > {last_planning_decision_id} ORDER BY pr.file_number::bigint, opd.decision_date, opd.planning_decision_id;" + else: + query_condition = f"WHERE pr.file_number::bigint >= {last_file_number} AND opd.planning_decision_id > {last_planning_decision_id} ORDER BY pr.file_number::bigint, opd.decision_date, opd.planning_decision_id;" + cursor.execute( f""" {query} - WHERE opd.planning_decision_id > {last_planning_decision_id} ORDER BY opd.planning_decision_id; + {query_condition} """ ) @@ -64,9 +74,10 @@ def process_planning_review_decisions(conn=None, batch_size=BATCH_UPLOAD_SIZE): successful_inserts_count = successful_inserts_count + len( inserted_data ) - last_planning_decision_id = dict(inserted_data[-1])[ - "planning_decision_id" - ] + last_record = dict(inserted_data[-1]) + last_planning_decision_id = last_record["planning_decision_id"] + last_file_number = last_record["file_number"] + last_decision_date = last_record["decision_date"] logger.debug( f"Retrieved/updated items count: {len(inserted_data)}; total successfully inserted planning referral so far {successful_inserts_count}; last updated planning_decision_id: {last_planning_decision_id}" @@ -106,7 +117,8 @@ def _insert_base_fields(conn, batch_size, cursor, rows): outcome_code, resolution_number, resolution_year, - date + date, + created_at ) VALUES ( %(review_uuid)s, @@ -116,13 +128,16 @@ def _insert_base_fields(conn, batch_size, cursor, rows): %(outcome_code)s, %(resolution_number)s, %(resolution_year)s, - %(date)s + %(date)s, + %(created_at)s ) + ON CONFLICT DO NOTHING; """ def _prepare_oats_planning_review_data(row_data_list): mapped_data_list = [] + insert_index = 0 for row in row_data_list: mapped_data_list.append( { @@ -134,8 +149,10 @@ def _prepare_oats_planning_review_data(row_data_list): "outcome_code": _map_outcome_code(row), "resolution_number": row["resolution_number"], "resolution_year": _map_resolution_year(row), + "created_at": to_alcs_format(get_now_with_offset(insert_index)), } ) + insert_index += 1 return mapped_data_list diff --git a/bin/migrate-oats-data/planning_review/sql/decisions/planning_review_decisions_insert.sql b/bin/migrate-oats-data/planning_review/sql/decisions/planning_review_decisions_insert.sql index cbd40c3352..321fecc8cf 100644 --- a/bin/migrate-oats-data/planning_review/sql/decisions/planning_review_decisions_insert.sql +++ b/bin/migrate-oats-data/planning_review/sql/decisions/planning_review_decisions_insert.sql @@ -5,7 +5,8 @@ SELECT opd.planning_acceptance_code, pr."uuid", opd.resolution_number, - opd.planning_decision_id + opd.planning_decision_id, + pr.file_number FROM oats.oats_planning_decisions opd JOIN alcs.planning_review pr ON opd.planning_review_id::TEXT = pr.file_number \ No newline at end of file diff --git a/services/apps/alcs/src/alcs/planning-review/planning-review-decision/planning-review-decision.controller.ts b/services/apps/alcs/src/alcs/planning-review/planning-review-decision/planning-review-decision.controller.ts index 7df9104110..0a24379981 100644 --- a/services/apps/alcs/src/alcs/planning-review/planning-review-decision/planning-review-decision.controller.ts +++ b/services/apps/alcs/src/alcs/planning-review/planning-review-decision/planning-review-decision.controller.ts @@ -32,7 +32,7 @@ import { PlanningReviewDecisionService } from './planning-review-decision.servic @UseGuards(RolesGuard) export class PlanningReviewDecisionController { constructor( - private plannigReviewDecisionService: PlanningReviewDecisionService, + private planningReviewDecisionService: PlanningReviewDecisionService, @InjectMapper() private mapper: Mapper, ) {} @@ -42,7 +42,7 @@ export class PlanningReviewDecisionController { @Param('fileNumber') fileNumber, ): Promise { const decisions = - await this.plannigReviewDecisionService.getByFileNumber(fileNumber); + await this.planningReviewDecisionService.getByFileNumber(fileNumber); return await this.mapper.mapArrayAsync( decisions, @@ -54,7 +54,7 @@ export class PlanningReviewDecisionController { @Get('/codes') @UserRoles(...ANY_AUTH_ROLE) async getCodes() { - const codes = await this.plannigReviewDecisionService.fetchCodes(); + const codes = await this.planningReviewDecisionService.fetchCodes(); return await this.mapper.mapArrayAsync( codes.outcomes, PlanningReviewDecisionOutcomeCode, @@ -65,7 +65,7 @@ export class PlanningReviewDecisionController { @Get('/:uuid') @UserRoles(...ANY_AUTH_ROLE) async get(@Param('uuid') uuid: string): Promise { - const decision = await this.plannigReviewDecisionService.get(uuid); + const decision = await this.planningReviewDecisionService.get(uuid); return this.mapper.mapAsync( decision, @@ -80,7 +80,7 @@ export class PlanningReviewDecisionController { @Body() createDto: CreatePlanningReviewDecisionDto, ): Promise { const newDecision = - await this.plannigReviewDecisionService.create(createDto); + await this.planningReviewDecisionService.create(createDto); return this.mapper.mapAsync( newDecision, @@ -95,7 +95,7 @@ export class PlanningReviewDecisionController { @Param('uuid') uuid: string, @Body() updateDto: UpdatePlanningReviewDecisionDto, ): Promise { - const updatedDecision = await this.plannigReviewDecisionService.update( + const updatedDecision = await this.planningReviewDecisionService.update( uuid, updateDto, ); @@ -110,7 +110,7 @@ export class PlanningReviewDecisionController { @Delete('/:uuid') @UserRoles(...ANY_AUTH_ROLE) async delete(@Param('uuid') uuid: string) { - return await this.plannigReviewDecisionService.delete(uuid); + return await this.planningReviewDecisionService.delete(uuid); } @Post('/:uuid/file') @@ -121,7 +121,7 @@ export class PlanningReviewDecisionController { } const file = req.body.file; - await this.plannigReviewDecisionService.attachDocument( + await this.planningReviewDecisionService.attachDocument( decisionUuid, file, req.user.entity, @@ -138,7 +138,7 @@ export class PlanningReviewDecisionController { @Param('documentUuid') documentUuid: string, @Body() body: { fileName: string }, ) { - await this.plannigReviewDecisionService.updateDocument( + await this.planningReviewDecisionService.updateDocument( documentUuid, body.fileName, ); @@ -154,7 +154,7 @@ export class PlanningReviewDecisionController { @Param('fileUuid') documentUuid: string, ) { const downloadUrl = - await this.plannigReviewDecisionService.getDownloadUrl(documentUuid); + await this.planningReviewDecisionService.getDownloadUrl(documentUuid); return { url: downloadUrl, }; @@ -166,7 +166,7 @@ export class PlanningReviewDecisionController { @Param('uuid') decisionUuid: string, @Param('fileUuid') documentUuid: string, ) { - const downloadUrl = await this.plannigReviewDecisionService.getDownloadUrl( + const downloadUrl = await this.planningReviewDecisionService.getDownloadUrl( documentUuid, true, ); @@ -181,7 +181,7 @@ export class PlanningReviewDecisionController { @Param('uuid') decisionUuid: string, @Param('fileUuid') documentUuid: string, ) { - await this.plannigReviewDecisionService.deleteDocument(documentUuid); + await this.planningReviewDecisionService.deleteDocument(documentUuid); return {}; } @@ -190,7 +190,7 @@ export class PlanningReviewDecisionController { async getNextAvailableResolutionNumber( @Param('resolutionYear') resolutionYear: number, ) { - return this.plannigReviewDecisionService.generateResolutionNumber( + return this.planningReviewDecisionService.generateResolutionNumber( resolutionYear, ); } diff --git a/services/apps/alcs/src/alcs/planning-review/planning-review-decision/planning-review-decision.service.ts b/services/apps/alcs/src/alcs/planning-review/planning-review-decision/planning-review-decision.service.ts index c9c12901f3..0e0dac8c1e 100644 --- a/services/apps/alcs/src/alcs/planning-review/planning-review-decision/planning-review-decision.service.ts +++ b/services/apps/alcs/src/alcs/planning-review/planning-review-decision/planning-review-decision.service.ts @@ -5,7 +5,7 @@ import { import { MultipartFile } from '@fastify/multipart'; import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { In, IsNull, Repository } from 'typeorm'; +import { IsNull, Repository } from 'typeorm'; import { DOCUMENT_SOURCE, DOCUMENT_SYSTEM,