Skip to content

Commit

Permalink
Add file_format and table_format configurations
Browse files Browse the repository at this point in the history
  • Loading branch information
damian3031 committed Oct 15, 2024
1 parent ae2edc4 commit c098ede
Show file tree
Hide file tree
Showing 2 changed files with 149 additions and 0 deletions.
33 changes: 33 additions & 0 deletions dbt/include/trino/macros/adapters.sql
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,39 @@

{% macro properties() %}
{%- set _properties = config.get('properties') -%}
{%- set table_format = config.get('table_format') -%}
{%- set file_format = config.get('file_format') -%}

{%- if file_format -%}
{%- if _properties -%}
{%- if _properties.format -%}
{% set msg %}
You can specify either 'file_format' or 'properties.format' configurations, but not both.
{% endset %}
{% do exceptions.raise_compiler_error(msg) %}
{%- else -%}
{%- do _properties.update({'format': file_format}) -%}
{%- endif -%}
{%- else -%}
{%- set _properties = {'format': file_format} -%}
{%- endif -%}
{%- endif -%}

{%- if table_format -%}
{%- if _properties -%}
{%- if _properties.type -%}
{% set msg %}
You can specify either 'table_format' or 'properties.type' configurations, but not both.
{% endset %}
{% do exceptions.raise_compiler_error(msg) %}
{%- else -%}
{%- do _properties.update({'type': table_format}) -%}
{%- endif -%}
{%- else -%}
{%- set _properties = {'type': table_format} -%}
{%- endif -%}
{%- endif -%}

{%- if _properties is not none -%}
WITH (
{%- for key, value in _properties.items() -%}
Expand Down
116 changes: 116 additions & 0 deletions tests/functional/adapter/test_table_properties.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,3 +47,119 @@ def test_table_properties(self, project):
assert "WITH (" in logs
assert "format = 'PARQUET'" in logs
assert "format_version = 2" in logs


@pytest.mark.iceberg
class TestFileFormatConfig(BaseTableProperties):
# Configuration in dbt_project.yml
@pytest.fixture(scope="class")
def project_config_update(self):
return {
"name": "properties_test",
"models": {
"+materialized": "table",
"file_format": "'PARQUET'",
},
}

def test_table_properties(self, project):
# Seed seed
results = run_dbt(["seed"], expect_pass=True)
assert len(results) == 1

# Create model with properties
results, logs = run_dbt_and_capture(["--debug", "run"], expect_pass=True)
assert len(results) == 1
assert "WITH (" in logs
assert "format = 'PARQUET'" in logs


@pytest.mark.iceberg
class TestFileFormatConfigAndFormatTablePropertyFail(BaseTableProperties):
# Configuration in dbt_project.yml
@pytest.fixture(scope="class")
def project_config_update(self):
return {
"name": "properties_test",
"models": {
"+materialized": "table",
"+properties": {
"format": "'PARQUET'",
},
"file_format": "'ORC'",
},
}

def test_table_properties(self, project):
# Seed seed
results = run_dbt(["seed"], expect_pass=True)
assert len(results) == 1

# Create model with properties
results, logs = run_dbt_and_capture(["--debug", "run"], expect_pass=False)
assert len(results) == 1
assert (
"You can specify either 'file_format' or 'properties.format' configurations, but not both."
in logs
)


@pytest.mark.hive
# Setting `type` property is available only in Starburst Galaxy
# https://docs.starburst.io/starburst-galaxy/data-engineering/working-with-data-lakes/table-formats/gl-iceberg.html
@pytest.mark.skip_profile("trino_starburst")
class TestTableFormatConfig(BaseTableProperties):
# Configuration in dbt_project.yml
@pytest.fixture(scope="class")
def project_config_update(self):
return {
"name": "properties_test",
"models": {
"+materialized": "table",
"table_format": "'iceberg'",
},
}

def test_table_properties(self, project):
# Seed seed
results = run_dbt(["seed"], expect_pass=True)
assert len(results) == 1

# Create model with properties
results, logs = run_dbt_and_capture(["--debug", "run"], expect_pass=True)
assert len(results) == 1
assert "WITH (" in logs
assert "type = 'iceberg'" in logs


@pytest.mark.hive
# Setting `type` property is available only in Starburst Galaxy
# https://docs.starburst.io/starburst-galaxy/data-engineering/working-with-data-lakes/table-formats/gl-iceberg.html
@pytest.mark.skip_profile("trino_starburst")
class TestTableFormatConfigAndTypeTablePropertyFail(BaseTableProperties):
# Configuration in dbt_project.yml
@pytest.fixture(scope="class")
def project_config_update(self):
return {
"name": "properties_test",
"models": {
"+materialized": "table",
"+properties": {
"type": "'iceberg'",
},
"table_format": "'iceberg'",
},
}

def test_table_properties(self, project):
# Seed seed
results = run_dbt(["seed"], expect_pass=True)
assert len(results) == 1

# Create model with properties
results, logs = run_dbt_and_capture(["--debug", "run"], expect_pass=False)
assert len(results) == 1
assert (
"You can specify either 'table_format' or 'properties.type' configurations, but not both."
in logs
)

0 comments on commit c098ede

Please sign in to comment.