Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Respect transient config for dbt Python models #802

Merged
merged 7 commits into from
Oct 11, 2023
Merged

Respect transient config for dbt Python models #802

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a00a4c9
add transient configs
jeremyyeo Sep 17, 2023
2011c4f
Merge branch 'main' into fix-776/transient-python-models
colin-rogers-dbt Sep 19, 2023
7074d92
Backwards-compatible handling of `temporary` and `transient` configs …
dbeatty10 Oct 11, 2023
f0a0ed2
Merge branch 'main' into dbeatty/transient-python-models
dbeatty10 Oct 11, 2023
93a850d
Fix Jinja syntax errors
dbeatty10 Oct 11, 2023
43db7d2
add test for table_type on python models
mikealfare Oct 11, 2023
7db3a11
Merge branch 'main' into dbeatty/transient-python-models
mikealfare Oct 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changes/unreleased/Fixes-20230918-105721.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Fixes
body: Make python models use transient config
time: 2023-09-18T10:57:21.113134+12:00
custom:
Author: jeremyyeo
Issue: "776"
15 changes: 13 additions & 2 deletions dbt/include/snowflake/macros/materializations/table.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,18 @@

{% endmaterialization %}

{% macro py_write_table(compiled_code, target_relation, temporary=False) %}
{% macro py_write_table(compiled_code, target_relation, temporary=False, table_type=none) %}
{#- The following logic is only for backwards-compatiblity with deprecated `temporary` parameter -#}
{% if table_type is not none %}
{#- Just use the table_type as-is -#}
{% elif temporary -%}
{#- Case 1 when the deprecated `temporary` parameter is used without the replacement `table_type` parameter -#}
{%- set table_type = "temporary" -%}
{% else %}
{#- Case 2 when the deprecated `temporary` parameter is used without the replacement `table_type` parameter -#}
{#- Snowflake treats "" as meaning "permanent" -#}
{%- set table_type = "" -%}
{%- endif %}
{{ compiled_code }}
def materialize(session, df, target_relation):
# make sure pandas exists
Expand All @@ -52,7 +63,7 @@ def materialize(session, df, target_relation):
# session.write_pandas does not have overwrite function
df = session.createDataFrame(df)
{% set target_relation_name = resolve_model_name(target_relation) %}
df.write.mode("overwrite").save_as_table('{{ target_relation_name }}', create_temp_table={{temporary}})
df.write.mode("overwrite").save_as_table('{{ target_relation_name }}', table_type='{{table_type}}')

def main(session):
dbt = dbtObj(session.table)
Expand Down
19 changes: 12 additions & 7 deletions dbt/include/snowflake/macros/relations/table/create.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,15 @@
{% macro snowflake__create_table_as(temporary, relation, compiled_code, language='sql') -%}
{%- set transient = config.get('transient', default=true) -%}

{% if temporary -%}
{%- set table_type = "temporary"
mikealfare marked this conversation as resolved.
Show resolved Hide resolved
{%- elif transient -%}
{%- set table_type = "transient"
{%- else -%}
{%- set table_type = ""
{%- endif %}

{%- if language == 'sql' -%}
{%- set transient = config.get('transient', default=true) -%}
{%- set cluster_by_keys = config.get('cluster_by', default=none) -%}
{%- set enable_automatic_clustering = config.get('automatic_clustering', default=false) -%}
{%- set copy_grants = config.get('copy_grants', default=false) -%}
Expand All @@ -17,11 +26,7 @@

{{ sql_header if sql_header is not none }}

create or replace {% if temporary -%}
temporary
{%- elif transient -%}
transient
{%- endif %} table {{ relation }}
create or replace {{ table_type }} table {{ relation }}
{%- set contract_config = config.get('contract') -%}
{%- if contract_config.enforced -%}
{{ get_assert_columns_equivalent(sql) }}
Expand All @@ -46,7 +51,7 @@
{%- endif -%}

{%- elif language == 'python' -%}
{{ py_write_table(compiled_code=compiled_code, target_relation=relation, temporary=temporary) }}
{{ py_write_table(compiled_code=compiled_code, target_relation=relation, table_type=table_type) }}
{%- else -%}
{% do exceptions.raise_compiler_error("snowflake__create_table_as macro didn't get supported language, it got %s" % language) %}
{%- endif -%}
Expand Down
Loading