remove corelated query from scd2 implementation

dlt-hub · Apr 22, 2024 · 544a0a4 · 544a0a4
1 parent 15855d7
commit 544a0a4
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/dlt/destinations/sql_jobs.py b/dlt/destinations/sql_jobs.py
@@ -579,9 +579,11 @@ def gen_scd2_sql(
                     staging_table_name = sql_client.make_qualified_table_name(table["name"])
                 sql.append(f"""
                     INSERT INTO {table_name}
-                    SELECT *
+                    SELECT s.*
                     FROM {staging_table_name} AS s
-                    WHERE NOT EXISTS (SELECT 1 FROM {table_name} AS f WHERE f.{unique_column} = s.{unique_column});
+                    LEFT JOIN {table_name} as t
+                        on s.{unique_column} = t.{unique_column}
+                    WHERE t.{unique_column} IS NULL or t.{unique_column} = '';
                 """)
         return sql