Skip to content

Commit

Permalink
upload ddls from ods_data_management schema
Browse files Browse the repository at this point in the history
  • Loading branch information
abimichel committed Oct 17, 2024
1 parent f6e7b87 commit 5b1ead3
Show file tree
Hide file tree
Showing 2 changed files with 45 additions and 18 deletions.
21 changes: 21 additions & 0 deletions shared/audit_batch_status.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
CREATE TABLE IF NOT EXISTS ods_data_management.audit_batch_status
(
object_name character varying(100) COLLATE pg_catalog."default",
application_name character varying(100) COLLATE pg_catalog."default",
etl_layer character varying(100) COLLATE pg_catalog."default",
object_execution_status character varying(100) COLLATE pg_catalog."default",
batch_run_date character varying(100) COLLATE pg_catalog."default"
)

TABLESPACE pg_default;

ALTER TABLE IF EXISTS ods_data_management.audit_batch_status
OWNER to ods_admin_user;

COMMENT ON TABLE ods_data_management.audit_batch_status IS 'Table to track the execution status of ELT process by batch.';

COMMENT ON COLUMN ods_data_management.audit_batch_status.object_name IS 'Name of the object, usually a table or view, being processed in batch.';
COMMENT ON COLUMN ods_data_management.audit_batch_status.application_name IS 'Name of the application associated with the object. This value should match the IRS/CMDB acronym.';
COMMENT ON COLUMN ods_data_management.audit_batch_status.etl_layer IS 'Layer of the ELT process (e.g., replication, transformation).';
COMMENT ON COLUMN ods_data_management.audit_batch_status.object_execution_status IS 'Status of the ELT execution (e.g., success, failure).';
COMMENT ON COLUMN ods_data_management.audit_batch_status.batch_run_date IS 'Most recent date that the batch ELT process took place.';
42 changes: 24 additions & 18 deletions shared/ods_data_management.sql → shared/cdc_master_table_list.sql
Original file line number Diff line number Diff line change
@@ -1,6 +1,28 @@
COMMENT ON SCHEMA ods_data_management IS 'Schema containing metadata, audit tables, and data management information for the Operational Data Store (ODS).';
CREATE TABLE IF NOT EXISTS ods_data_management.cdc_master_table_list
(
business character varying(100) COLLATE pg_catalog."default",
application_name character varying(100) COLLATE pg_catalog."default",
custodian character varying(100) COLLATE pg_catalog."default",
source_schema_name character varying(100) COLLATE pg_catalog."default",
source_table_name character varying(100) COLLATE pg_catalog."default",
target_schema_name character varying(100) COLLATE pg_catalog."default",
target_table_name character varying(100) COLLATE pg_catalog."default",
truncate_flag character varying(1) COLLATE pg_catalog."default",
cdc_flag character varying(1) COLLATE pg_catalog."default",
full_inc_flag character varying(1) COLLATE pg_catalog."default",
cdc_column character varying(50) COLLATE pg_catalog."default",
active_ind character varying(1) COLLATE pg_catalog."default",
replication_order integer,
where_clause character varying(1000) COLLATE pg_catalog."default",
customsql_ind character varying(1) COLLATE pg_catalog."default",
customsql_query character varying(64000) COLLATE pg_catalog."default",
replication_source character varying(100) COLLATE pg_catalog."default"
)

COMMENT ON VIEW ods_data_management.size_of_schemas IS 'View summarizing the total size of each ODS schema in human-readable format.';
TABLESPACE pg_default;

ALTER TABLE IF EXISTS ods_data_management.cdc_master_table_list
OWNER to ods_admin_user;

COMMENT ON TABLE ods_data_management.cdc_master_table_list IS 'Table providing an overview/summary of all replication processes, change data capture (CDC), and custom SQL.';

Expand All @@ -21,19 +43,3 @@ COMMENT ON COLUMN ods_data_management.cdc_master_table_list.where_clause IS 'Opt
COMMENT ON COLUMN ods_data_management.cdc_master_table_list.customsql_ind IS 'Flag indicating if a custom SQL query is used (Y/N). If Y, then the customsql will be applied.';
COMMENT ON COLUMN ods_data_management.cdc_master_table_list.customsql_query IS 'Custom SQL query to use for replication, if applicable. This is required for geospatial conversion and keeping history tables.';
COMMENT ON COLUMN ods_data_management.cdc_master_table_list.replication_source IS 'Source system for replication data (e.g. Oracle, PostgreSQL)'; -- Note: Values should be standardized.

COMMENT ON TABLE ods_data_management.audit_batch_status IS 'Table to track the execution status of ELT process by batch.';

COMMENT ON COLUMN ods_data_management.audit_batch_status.object_name IS 'Name of the object, usually a table or view, being processed in batch.';
COMMENT ON COLUMN ods_data_management.audit_batch_status.application_name IS 'Name of the application associated with the object. This value should match the IRS/CMDB acronym.';
COMMENT ON COLUMN ods_data_management.audit_batch_status.etl_layer IS 'Layer of the ELT process (e.g., replication, transformation).';
COMMENT ON COLUMN ods_data_management.audit_batch_status.object_execution_status IS 'Status of the ELT execution (e.g., success, failure).';
COMMENT ON COLUMN ods_data_management.audit_batch_status.batch_run_date IS 'Most recent date that the batch ELT process took place.';

COMMENT ON TABLE ods_data_management.audit_batch_id IS 'Table to store additional metadata related to directed acyclic graph (DAG) execution.'; -- Note: In current architecture, this is only populated when the 'permitting_pipeline_etl_batch_id_creation/update' DAGs run.

COMMENT ON COLUMN ods_data_management.audit_batch_id.etl_batch_id IS 'Date of the DAG execution.'; -- Note: This column should be renamed. _id is misleading.
COMMENT ON COLUMN ods_data_management.audit_batch_id.etl_batch_name IS 'Name of the DAG being executed.';
COMMENT ON COLUMN ods_data_management.audit_batch_id.etl_batch_status IS 'Current status of the ETL batch execution (e.g., started, success, failed).';
COMMENT ON COLUMN ods_data_management.audit_batch_id.etl_batch_start_time IS 'Start time of the DAG execution.';
COMMENT ON COLUMN ods_data_management.audit_batch_id.etl_batch_end_time IS 'End time of the DAG execution.';

0 comments on commit 5b1ead3

Please sign in to comment.