From b4c64a1ccc814e526cdec85514cbf229b24071ee Mon Sep 17 00:00:00 2001 From: Derrick Brittain Date: Wed, 10 Aug 2022 13:50:14 -0700 Subject: [PATCH] feat: add merge tables optional param --- materializationengine/celery_worker.py | 9 ++++++--- materializationengine/config.py | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/materializationengine/celery_worker.py b/materializationengine/celery_worker.py index 187b0ce3..20286b7a 100644 --- a/materializationengine/celery_worker.py +++ b/materializationengine/celery_worker.py @@ -112,15 +112,18 @@ def setup_periodic_tasks(sender, **kwargs): run_periodic_materialization, ) + merge_tables = get_config_param("MERGE_TABLES") periodic_tasks = { "run_daily_periodic_materialization": run_periodic_materialization.s( - days_to_expire=2 + days_to_expire=2, merge_tables=merge_tables ), "run_weekly_periodic_materialization": run_periodic_materialization.s( - days_to_expire=7 + days_to_expire=7, merge_tables=merge_tables ), "run_lts_periodic_materialization": run_periodic_materialization.s( - days_to_expire=days_till_next_month(datetime.datetime.utcnow()) + days_to_expire=days_till_next_month( + datetime.datetime.utcnow(), merge_tables=merge_tables + ) ), "run_periodic_database_update": run_periodic_database_update.s(), "remove_expired_databases": remove_expired_databases.s( diff --git a/materializationengine/config.py b/materializationengine/config.py index b94c3c43..c691a7a5 100644 --- a/materializationengine/config.py +++ b/materializationengine/config.py @@ -39,6 +39,7 @@ class BaseConfig: INFO_API_VERSION = 2 MIN_DATABASES = 2 MAX_DATABASES = 2 + MERGE_TABLES = True AUTH_SERVICE_NAMESPACE = "datastack" if os.environ.get("DAF_CREDENTIALS", None) is not None: with open(os.environ.get("DAF_CREDENTIALS"), "r") as f: