From 23ddc9e1526416b6a8ceeb9a317ca75ba0e92acf Mon Sep 17 00:00:00 2001 From: Brendan Cook Date: Tue, 28 Nov 2023 10:07:34 +1100 Subject: [PATCH] get redshift external tables (#753) * get redshift external tables redshift external tables are not listed in information_schema.tables but we can extract the same info from svv_external tables and union the results. also quote database name as redshift allows dashes which require quotes * add changelog * snowflake doesn't accept quotes --------- Co-authored-by: Anders --- CHANGELOG.md | 6 ++++++ macros/sql/get_tables_by_pattern_sql.sql | 26 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b7799278..a72abb7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,10 +12,16 @@ ## Fixes - deduplicate macro for Databricks now uses the QUALIFY clause, which fixes NULL columns issues from the default natural join logic - deduplicate macro for Redshift now uses the QUALIFY clause, which fixes NULL columns issues from the default natural join logic +- get_tables_by_pattern_sql will now: + - return redshift external tables ([#752](https://github.com/dbt-labs/dbt-utils/issues/752) + - work with valid redshift database names that contain dashes +## Under the hood +- created a new dispatch redshift__get_tables_by_pattern which unions the result of the default macro and querying svv_external_tables for the same conditions (schema name, pattern, exclude pattern). ## Contributors: [@graciegoheen](https://github.com/graciegoheen) [@yauhen-sobaleu](https://github.com/yauhen-sobaleu) +[@brendan-cook-87](https://github.com/brendan-cook-87) # dbt utils v1.1.1 ## New features diff --git a/macros/sql/get_tables_by_pattern_sql.sql b/macros/sql/get_tables_by_pattern_sql.sql index 75d67860..03f96624 100644 --- a/macros/sql/get_tables_by_pattern_sql.sql +++ b/macros/sql/get_tables_by_pattern_sql.sql @@ -16,6 +16,32 @@ {% endmacro %} +{% macro redshift__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} + + {% set sql %} + select distinct + table_schema as {{ adapter.quote('table_schema') }}, + table_name as {{ adapter.quote('table_name') }}, + {{ dbt_utils.get_table_types_sql() }} + from "{{ database }}"."information_schema"."tables" + where table_schema ilike '{{ schema_pattern }}' + and table_name ilike '{{ table_pattern }}' + and table_name not ilike '{{ exclude }}' + union all + select distinct + schemaname as {{ adapter.quote('table_schema') }}, + tablename as {{ adapter.quote('table_name') }}, + 'external' as {{ adapter.quote('table_type') }} + from svv_external_tables + where redshift_database_name = '{{ database }}' + and schemaname ilike '{{ schema_pattern }}' + and table_name ilike '{{ table_pattern }}' + and table_name not ilike '{{ exclude }}' + {% endset %} + + {{ return(sql) }} +{% endmacro %} + {% macro bigquery__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %}