-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1324 from bcgov/develop
Deployment PR - 864
- Loading branch information
Showing
56 changed files
with
1,331 additions
and
191 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
105 changes: 105 additions & 0 deletions
105
bin/migrate-oats-data/applications/application_decision_date.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,105 @@ | ||
| from common import ( | ||
| BATCH_UPLOAD_SIZE, | ||
| setup_and_get_logger, | ||
| add_timezone_and_keep_date_part, | ||
| ) | ||
| from db import inject_conn_pool | ||
| from psycopg2.extras import RealDictCursor, execute_batch | ||
|
|
||
| etl_name = "process_alcs_application_decision_date" | ||
| logger = setup_and_get_logger(etl_name) | ||
|
|
||
|
|
||
| @inject_conn_pool | ||
| def process_alcs_application_decision_date(conn=None, batch_size=BATCH_UPLOAD_SIZE): | ||
| """ | ||
| Imports decision_date from oats to alcs.application.decision_date. alcs.application.decision_date is the date of the first decision | ||
| In OATS the first decision date is stored in oats.oats_alr_appl_decisions. All subsequent decisions in OATS are the linked to reconsiderations and not application directly. | ||
| """ | ||
|
|
||
| logger.info(f"Start {etl_name}") | ||
| with conn.cursor(cursor_factory=RealDictCursor) as cursor: | ||
| with open( | ||
| "applications/sql/application_decision_date/application_decision_date_count.sql", | ||
| "r", | ||
| encoding="utf-8", | ||
| ) as sql_file: | ||
| count_query = sql_file.read() | ||
| cursor.execute(count_query) | ||
| count_total = dict(cursor.fetchone())["count"] | ||
|
|
||
| logger.info(f" Total Application data to update: {count_total}") | ||
|
|
||
| failed_inserts = 0 | ||
| successful_updates_count = 0 | ||
| last_application_id = 0 | ||
|
|
||
| with open( | ||
| "applications/sql/application_decision_date/application_decision_date.sql", | ||
| "r", | ||
| encoding="utf-8", | ||
| ) as sql_file: | ||
| application_sql = sql_file.read() | ||
| while True: | ||
| cursor.execute( | ||
| f"{application_sql} WHERE decision_date_for_applications.alr_application_id > {last_application_id} ORDER BY decision_date_for_applications.alr_application_id;" | ||
| ) | ||
|
|
||
| rows = cursor.fetchmany(batch_size) | ||
|
|
||
| if not rows: | ||
| break | ||
| try: | ||
| records_to_be_updated_count = len(rows) | ||
|
|
||
| _update_fee_fields_records(conn, batch_size, cursor, rows) | ||
|
|
||
| successful_updates_count = ( | ||
| successful_updates_count + records_to_be_updated_count | ||
| ) | ||
| last_application_id = dict(rows[-1])["alr_application_id"] | ||
|
|
||
| logger.debug( | ||
| f"retrieved/updated items count: {records_to_be_updated_count}; total successfully updated applications so far {successful_updates_count}; last updated application_id: {last_application_id}" | ||
| ) | ||
| except Exception as err: | ||
| # this is NOT going to be caused by actual data update failure. This code is only executed when the code error appears or connection to DB is lost | ||
| logger.exception(err) | ||
| conn.rollback() | ||
| failed_inserts = count_total - successful_updates_count | ||
| last_application_id = last_application_id + 1 | ||
|
|
||
| logger.info( | ||
| f"Finished {etl_name}: total amount of successful updates {successful_updates_count}, total failed updates {failed_inserts}" | ||
| ) | ||
|
|
||
|
|
||
| def _update_fee_fields_records(conn, batch_size, cursor, rows): | ||
| query = _get_update_query_from_oats_alr_applications_fields() | ||
| parsed_fee_data_list = _prepare_oats_alr_applications_data(rows) | ||
|
|
||
| if len(parsed_fee_data_list) > 0: | ||
| execute_batch(cursor, query, parsed_fee_data_list, page_size=batch_size) | ||
|
|
||
| conn.commit() | ||
|
|
||
|
|
||
| def _get_update_query_from_oats_alr_applications_fields(): | ||
| query = """ | ||
| UPDATE alcs.application | ||
| SET decision_date = COALESCE(%(decision_date)s, decision_date) | ||
| WHERE | ||
| file_number = %(alr_application_id)s::TEXT; | ||
| """ | ||
| return query | ||
|
|
||
|
|
||
| def _prepare_oats_alr_applications_data(row_data_list): | ||
| data_list = [] | ||
| for row in row_data_list: | ||
| data = dict(row) | ||
| data["decision_date"] = add_timezone_and_keep_date_part( | ||
| row.get("decision_date") | ||
| ) | ||
| data_list.append(data) | ||
| return data_list |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
158 changes: 158 additions & 0 deletions
158
bin/migrate-oats-data/applications/set_application_visibility.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,158 @@ | ||
| from common import setup_and_get_logger | ||
| from db import inject_conn_pool | ||
| from psycopg2.extras import RealDictCursor | ||
|
|
||
|
|
||
| etl_name = "set_application_visibility" | ||
| logger = setup_and_get_logger(etl_name) | ||
|
|
||
|
|
||
| @inject_conn_pool | ||
| def set_application_visibility(conn=None): | ||
| """This function is responsible for setting source column on application and mapping it to public portal visibility visibility.""" | ||
| _set_source_on_applications(conn) | ||
| _set_hide_from_portal_on_application(conn) | ||
|
|
||
|
|
||
| def _set_hide_from_portal_on_application(conn): | ||
| """ | ||
| This function is responsible for setting hide_from_portal colum on application table based on the source. | ||
| Args: | ||
| conn (psycopg2.extensions.connection): PostgreSQL database connection. Provided by the decorator. | ||
| """ | ||
| logger.info(f"Start {etl_name}") | ||
|
|
||
| try: | ||
| with conn.cursor(cursor_factory=RealDictCursor) as cursor: | ||
| count_visible_to_portal_query = f""" | ||
| SELECT count(*) FROM alcs.application a | ||
| WHERE a."source" = 'APPLICANT'; | ||
| """ | ||
| cursor.execute(count_visible_to_portal_query) | ||
| visible_to_portal_count = dict(cursor.fetchone())["count"] | ||
|
|
||
| count_hidden_from_portal_query = f""" | ||
| SELECT count(*) FROM alcs.application a | ||
| WHERE a."source" <> 'APPLICANT'; | ||
| """ | ||
| cursor.execute(count_hidden_from_portal_query) | ||
| hidden_from_portal_count = dict(cursor.fetchone())["count"] | ||
|
|
||
| count_total = visible_to_portal_count + hidden_from_portal_count | ||
|
|
||
| logger.info(f"Total application to set hide_from_portal: {count_total}") | ||
|
|
||
| set_hide_from_portal_false_query = f""" | ||
| UPDATE alcs.application | ||
| SET hide_from_portal = FALSE | ||
| WHERE alcs.application."source" = 'APPLICANT'; | ||
| """ | ||
| cursor.execute(set_hide_from_portal_false_query) | ||
| updated_visible_to_portal_count = cursor.rowcount | ||
|
|
||
| set_hide_from_portal_true_query = f""" | ||
| UPDATE alcs.application | ||
| SET hide_from_portal = TRUE | ||
| WHERE alcs.application."source" <> 'APPLICANT'; | ||
| """ | ||
| cursor.execute(set_hide_from_portal_true_query) | ||
| updated_hidden_from_portal_count = cursor.rowcount | ||
|
|
||
| updated_total_count = ( | ||
| updated_visible_to_portal_count + updated_hidden_from_portal_count | ||
| ) | ||
|
|
||
| logger.info( | ||
| f"Total hide_from_portal flag on applications updated: {updated_total_count}" | ||
| ) | ||
| conn.commit() | ||
| except Exception as err: | ||
| logger.exception(err) | ||
| cursor.close() | ||
| conn.close() | ||
|
|
||
| logger.info(f"Finished {etl_name}") | ||
|
|
||
|
|
||
| def _set_source_on_applications(conn): | ||
| """""" | ||
| logger.info(f"Start {etl_name} => set_source_on_applications") | ||
|
|
||
| try: | ||
| with conn.cursor(cursor_factory=RealDictCursor) as cursor: | ||
| count_created_by_applicant_query = f""" | ||
| SELECT count(*) FROM oats.oats_alr_applications oaa | ||
| JOIN alcs.application app ON app.file_number = oaa.alr_application_id::TEXT | ||
| WHERE oaa.who_created = 'PROXY_OATS_APPLICANT'; | ||
| """ | ||
| cursor.execute(count_created_by_applicant_query) | ||
| created_by_applicant_count = dict(cursor.fetchone())["count"] | ||
|
|
||
| count_created_by_oats_query = f""" | ||
| SELECT count(*) FROM oats.oats_alr_applications oaa | ||
| JOIN alcs.application app ON app.file_number = oaa.alr_application_id::TEXT | ||
| WHERE oaa.who_created <> 'PROXY_OATS_APPLICANT'; | ||
| """ | ||
| cursor.execute(count_created_by_oats_query) | ||
| created_by_oats_count = dict(cursor.fetchone())["count"] | ||
|
|
||
| count_created_in_alcs_query = f""" | ||
| SELECT count(*) FROM alcs.application a | ||
| WHERE a."source" <> 'APPLICANT' | ||
| AND a.audit_created_by <> 'oats_etl'; | ||
| """ | ||
| cursor.execute(count_created_in_alcs_query) | ||
| created_by_alcs_count = dict(cursor.fetchone())["count"] | ||
|
|
||
| count_total = ( | ||
| created_by_applicant_count | ||
| + created_by_oats_count | ||
| + created_by_alcs_count | ||
| ) | ||
|
|
||
| logger.info(f"Total application to set hide_from_portal: {count_total}") | ||
|
|
||
| update_created_by_applicant_query = f""" | ||
| UPDATE alcs.application | ||
| SET "source" = 'APPLICANT' | ||
| FROM oats.oats_alr_applications oaa | ||
| WHERE alcs.application.file_number = oaa.alr_application_id::TEXT | ||
| AND oaa.who_created = 'PROXY_OATS_APPLICANT'; | ||
| """ | ||
| cursor.execute(update_created_by_applicant_query) | ||
| updated_created_by_applicant_count = cursor.rowcount | ||
|
|
||
| update_created_by_oats_query = f""" | ||
| UPDATE alcs.application | ||
| SET "source" = 'OATS' | ||
| FROM oats.oats_alr_applications oaa | ||
| WHERE alcs.application.file_number = oaa.alr_application_id::TEXT | ||
| AND oaa.who_created <> 'PROXY_OATS_APPLICANT'; | ||
| """ | ||
| cursor.execute(update_created_by_oats_query) | ||
| updated_created_by_oats_count = cursor.rowcount | ||
|
|
||
| update_created_by_alcs_query = f""" | ||
| UPDATE alcs.application | ||
| SET "source" = 'APPLICANT' | ||
| WHERE alcs.application."source" <> 'APPLICANT' | ||
| AND alcs.application.audit_created_by <> 'oats_etl'; | ||
| """ | ||
| cursor.execute(update_created_by_alcs_query) | ||
| updated_created_by_alcs_count = cursor.rowcount | ||
|
|
||
| total_processed = ( | ||
| updated_created_by_applicant_count | ||
| + updated_created_by_oats_count | ||
| + updated_created_by_alcs_count | ||
| ) | ||
|
|
||
| logger.info(f"Total source on 'application' updated: {total_processed}") | ||
| conn.commit() | ||
| except Exception as err: | ||
| logger.exception(err) | ||
| cursor.close() | ||
| conn.close() | ||
|
|
||
| logger.info(f"Finished {etl_name}") |
Oops, something went wrong.