From 5b1ead370461b44a4911a69350ac8ed63785203e Mon Sep 17 00:00:00 2001 From: abimichel Date: Thu, 17 Oct 2024 13:21:58 -0700 Subject: [PATCH] upload ddls from ods_data_management schema --- shared/audit_batch_status.sql | 21 ++++++++++ ...nagement.sql => cdc_master_table_list.sql} | 42 +++++++++++-------- 2 files changed, 45 insertions(+), 18 deletions(-) create mode 100644 shared/audit_batch_status.sql rename shared/{ods_data_management.sql => cdc_master_table_list.sql} (60%) diff --git a/shared/audit_batch_status.sql b/shared/audit_batch_status.sql new file mode 100644 index 0000000..e2568b7 --- /dev/null +++ b/shared/audit_batch_status.sql @@ -0,0 +1,21 @@ +CREATE TABLE IF NOT EXISTS ods_data_management.audit_batch_status +( + object_name character varying(100) COLLATE pg_catalog."default", + application_name character varying(100) COLLATE pg_catalog."default", + etl_layer character varying(100) COLLATE pg_catalog."default", + object_execution_status character varying(100) COLLATE pg_catalog."default", + batch_run_date character varying(100) COLLATE pg_catalog."default" +) + +TABLESPACE pg_default; + +ALTER TABLE IF EXISTS ods_data_management.audit_batch_status + OWNER to ods_admin_user; + +COMMENT ON TABLE ods_data_management.audit_batch_status IS 'Table to track the execution status of ELT process by batch.'; + +COMMENT ON COLUMN ods_data_management.audit_batch_status.object_name IS 'Name of the object, usually a table or view, being processed in batch.'; +COMMENT ON COLUMN ods_data_management.audit_batch_status.application_name IS 'Name of the application associated with the object. This value should match the IRS/CMDB acronym.'; +COMMENT ON COLUMN ods_data_management.audit_batch_status.etl_layer IS 'Layer of the ELT process (e.g., replication, transformation).'; +COMMENT ON COLUMN ods_data_management.audit_batch_status.object_execution_status IS 'Status of the ELT execution (e.g., success, failure).'; +COMMENT ON COLUMN ods_data_management.audit_batch_status.batch_run_date IS 'Most recent date that the batch ELT process took place.'; \ No newline at end of file diff --git a/shared/ods_data_management.sql b/shared/cdc_master_table_list.sql similarity index 60% rename from shared/ods_data_management.sql rename to shared/cdc_master_table_list.sql index 098c439..34c2173 100644 --- a/shared/ods_data_management.sql +++ b/shared/cdc_master_table_list.sql @@ -1,6 +1,28 @@ -COMMENT ON SCHEMA ods_data_management IS 'Schema containing metadata, audit tables, and data management information for the Operational Data Store (ODS).'; +CREATE TABLE IF NOT EXISTS ods_data_management.cdc_master_table_list +( + business character varying(100) COLLATE pg_catalog."default", + application_name character varying(100) COLLATE pg_catalog."default", + custodian character varying(100) COLLATE pg_catalog."default", + source_schema_name character varying(100) COLLATE pg_catalog."default", + source_table_name character varying(100) COLLATE pg_catalog."default", + target_schema_name character varying(100) COLLATE pg_catalog."default", + target_table_name character varying(100) COLLATE pg_catalog."default", + truncate_flag character varying(1) COLLATE pg_catalog."default", + cdc_flag character varying(1) COLLATE pg_catalog."default", + full_inc_flag character varying(1) COLLATE pg_catalog."default", + cdc_column character varying(50) COLLATE pg_catalog."default", + active_ind character varying(1) COLLATE pg_catalog."default", + replication_order integer, + where_clause character varying(1000) COLLATE pg_catalog."default", + customsql_ind character varying(1) COLLATE pg_catalog."default", + customsql_query character varying(64000) COLLATE pg_catalog."default", + replication_source character varying(100) COLLATE pg_catalog."default" +) -COMMENT ON VIEW ods_data_management.size_of_schemas IS 'View summarizing the total size of each ODS schema in human-readable format.'; +TABLESPACE pg_default; + +ALTER TABLE IF EXISTS ods_data_management.cdc_master_table_list + OWNER to ods_admin_user; COMMENT ON TABLE ods_data_management.cdc_master_table_list IS 'Table providing an overview/summary of all replication processes, change data capture (CDC), and custom SQL.'; @@ -21,19 +43,3 @@ COMMENT ON COLUMN ods_data_management.cdc_master_table_list.where_clause IS 'Opt COMMENT ON COLUMN ods_data_management.cdc_master_table_list.customsql_ind IS 'Flag indicating if a custom SQL query is used (Y/N). If Y, then the customsql will be applied.'; COMMENT ON COLUMN ods_data_management.cdc_master_table_list.customsql_query IS 'Custom SQL query to use for replication, if applicable. This is required for geospatial conversion and keeping history tables.'; COMMENT ON COLUMN ods_data_management.cdc_master_table_list.replication_source IS 'Source system for replication data (e.g. Oracle, PostgreSQL)'; -- Note: Values should be standardized. - -COMMENT ON TABLE ods_data_management.audit_batch_status IS 'Table to track the execution status of ELT process by batch.'; - -COMMENT ON COLUMN ods_data_management.audit_batch_status.object_name IS 'Name of the object, usually a table or view, being processed in batch.'; -COMMENT ON COLUMN ods_data_management.audit_batch_status.application_name IS 'Name of the application associated with the object. This value should match the IRS/CMDB acronym.'; -COMMENT ON COLUMN ods_data_management.audit_batch_status.etl_layer IS 'Layer of the ELT process (e.g., replication, transformation).'; -COMMENT ON COLUMN ods_data_management.audit_batch_status.object_execution_status IS 'Status of the ELT execution (e.g., success, failure).'; -COMMENT ON COLUMN ods_data_management.audit_batch_status.batch_run_date IS 'Most recent date that the batch ELT process took place.'; - -COMMENT ON TABLE ods_data_management.audit_batch_id IS 'Table to store additional metadata related to directed acyclic graph (DAG) execution.'; -- Note: In current architecture, this is only populated when the 'permitting_pipeline_etl_batch_id_creation/update' DAGs run. - -COMMENT ON COLUMN ods_data_management.audit_batch_id.etl_batch_id IS 'Date of the DAG execution.'; -- Note: This column should be renamed. _id is misleading. -COMMENT ON COLUMN ods_data_management.audit_batch_id.etl_batch_name IS 'Name of the DAG being executed.'; -COMMENT ON COLUMN ods_data_management.audit_batch_id.etl_batch_status IS 'Current status of the ETL batch execution (e.g., started, success, failed).'; -COMMENT ON COLUMN ods_data_management.audit_batch_id.etl_batch_start_time IS 'Start time of the DAG execution.'; -COMMENT ON COLUMN ods_data_management.audit_batch_id.etl_batch_end_time IS 'End time of the DAG execution.';