feat: add merge tables optional param

CAVEconnectome · Aug 10, 2022 · b4c64a1 · b4c64a1
1 parent e790e66
commit b4c64a1
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 3 deletions.
diff --git a/materializationengine/celery_worker.py b/materializationengine/celery_worker.py
@@ -112,15 +112,18 @@ def setup_periodic_tasks(sender, **kwargs):
         run_periodic_materialization,
     )
 
+    merge_tables = get_config_param("MERGE_TABLES")
     periodic_tasks = {
         "run_daily_periodic_materialization": run_periodic_materialization.s(
-            days_to_expire=2
+            days_to_expire=2, merge_tables=merge_tables
         ),
         "run_weekly_periodic_materialization": run_periodic_materialization.s(
-            days_to_expire=7
+            days_to_expire=7, merge_tables=merge_tables
         ),
         "run_lts_periodic_materialization": run_periodic_materialization.s(
-            days_to_expire=days_till_next_month(datetime.datetime.utcnow())
+            days_to_expire=days_till_next_month(
+                datetime.datetime.utcnow(), merge_tables=merge_tables
+            )
         ),
         "run_periodic_database_update": run_periodic_database_update.s(),
         "remove_expired_databases": remove_expired_databases.s(

diff --git a/materializationengine/config.py b/materializationengine/config.py
@@ -39,6 +39,7 @@ class BaseConfig:
     INFO_API_VERSION = 2
     MIN_DATABASES = 2
     MAX_DATABASES = 2
+    MERGE_TABLES = True
     AUTH_SERVICE_NAMESPACE = "datastack"
     if os.environ.get("DAF_CREDENTIALS", None) is not None:
         with open(os.environ.get("DAF_CREDENTIALS"), "r") as f: