fix: Uppercase column names issue in aws asset inventory (#990)

* fixed uppercase column names issue * installing last working dbt-postgres version * removed specific version, updated to include shards from makefile --------- Co-authored-by: Kevin Rheinheimer <[email protected]>
cloudquery · Jul 29, 2024 · 651c9e6 · 651c9e6
1 parent 73f4edf
commit 651c9e6
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 16 deletions.
diff --git a/.github/workflows/transformations_docs_deploy.yml b/.github/workflows/transformations_docs_deploy.yml
@@ -21,6 +21,7 @@ jobs:
         shard: [
           1,  2,  3,  4,
           5,  6,  7,  8,
+          9,  10
         ]
     timeout-minutes: 30
     services:

diff --git a/transformations/aws/asset-inventory-free/macros/aws_tables_dyn.sql b/transformations/aws/asset-inventory-free/macros/aws_tables_dyn.sql
@@ -5,11 +5,11 @@
 {% macro postgres__aws_tables_dyn() %}
 SELECT 
     t.table_name,
-    MAX(CASE WHEN UPPER(c.column_name) = 'ARN' THEN 1 ELSE 0 END) AS ARN_EXIST,
-    MAX(CASE WHEN UPPER(c.column_name) = 'ACCOUNT_ID' THEN 1 ELSE 0 END) AS ACCOUNT_ID_EXIST,
-    MAX(CASE WHEN UPPER(c.column_name) = 'REQUEST_ACCOUNT_ID' THEN 1 ELSE 0 END) AS REQUEST_ACCOUNT_ID_EXIST,
-    MAX(CASE WHEN UPPER(c.column_name) = 'REGION' THEN 1 ELSE 0 END) AS REGION_EXIST,
-    MAX(CASE WHEN UPPER(c.column_name) = 'TAGS' THEN 1 ELSE 0 END) AS TAGS_EXIST  
+    MAX(CASE WHEN UPPER(c.column_name) = 'ARN' THEN 1 ELSE 0 END) AS arn_exist,
+    MAX(CASE WHEN UPPER(c.column_name) = 'ACCOUNT_ID' THEN 1 ELSE 0 END) AS account_id_exist,
+    MAX(CASE WHEN UPPER(c.column_name) = 'REQUEST_ACCOUNT_ID' THEN 1 ELSE 0 END) AS request_account_id_exist,
+    MAX(CASE WHEN UPPER(c.column_name) = 'REGION' THEN 1 ELSE 0 END) AS region_exist,
+    MAX(CASE WHEN UPPER(c.column_name) = 'TAGS' THEN 1 ELSE 0 END) AS tags_exist  
 FROM 
     INFORMATION_SCHEMA.TABLES t
 LEFT JOIN 
@@ -27,11 +27,11 @@ HAVING
 {% macro snowflake__aws_tables_dyn() %}
 SELECT 
     t.table_name as "table_name",
-    MAX(CASE WHEN UPPER(c.column_name) = 'ARN' THEN 1 ELSE 0 END) AS ARN_EXIST,
-    MAX(CASE WHEN UPPER(c.column_name) = 'ACCOUNT_ID' THEN 1 ELSE 0 END) AS ACCOUNT_ID_EXIST,
-    MAX(CASE WHEN UPPER(c.column_name) = 'REQUEST_ACCOUNT_ID' THEN 1 ELSE 0 END) AS REQUEST_ACCOUNT_ID_EXIST,
-    MAX(CASE WHEN UPPER(c.column_name) = 'REGION' THEN 1 ELSE 0 END) AS REGION_EXIST,
-    MAX(CASE WHEN UPPER(c.column_name) = 'TAGS' THEN 1 ELSE 0 END) AS TAGS_EXIST
+    MAX(CASE WHEN UPPER(c.column_name) = 'ARN' THEN 1 ELSE 0 END) AS "arn_exist",
+    MAX(CASE WHEN UPPER(c.column_name) = 'ACCOUNT_ID' THEN 1 ELSE 0 END) AS "account_id_exist",
+    MAX(CASE WHEN UPPER(c.column_name) = 'REQUEST_ACCOUNT_ID' THEN 1 ELSE 0 END) AS "request_account_id_exist",
+    MAX(CASE WHEN UPPER(c.column_name) = 'REGION' THEN 1 ELSE 0 END) AS "region_exist",
+    MAX(CASE WHEN UPPER(c.column_name) = 'TAGS' THEN 1 ELSE 0 END) AS "tags_exist"
 FROM 
     INFORMATION_SCHEMA.TABLES t
 LEFT JOIN 
@@ -50,11 +50,11 @@ HAVING
 
 SELECT 
     t.table_name,
-    MAX(CASE WHEN UPPER(c.column_name) = 'ARN' THEN 1 ELSE 0 END) AS ARN_EXIST,
-    MAX(CASE WHEN UPPER(c.column_name) = 'ACCOUNT_ID' THEN 1 ELSE 0 END) AS ACCOUNT_ID_EXIST,
-    MAX(CASE WHEN UPPER(c.column_name) = 'REQUEST_ACCOUNT_ID' THEN 1 ELSE 0 END) AS REQUEST_ACCOUNT_ID_EXIST,
-    MAX(CASE WHEN UPPER(c.column_name) = 'REGION' THEN 1 ELSE 0 END) AS REGION_EXIST,
-    MAX(CASE WHEN UPPER(c.column_name) = 'TAGS' THEN 1 ELSE 0 END) AS TAGS_EXIST
+    MAX(CASE WHEN UPPER(c.column_name) = 'ARN' THEN 1 ELSE 0 END) AS arn_exist,
+    MAX(CASE WHEN UPPER(c.column_name) = 'ACCOUNT_ID' THEN 1 ELSE 0 END) AS account_id_exist,
+    MAX(CASE WHEN UPPER(c.column_name) = 'REQUEST_ACCOUNT_ID' THEN 1 ELSE 0 END) AS request_account_id_exist,
+    MAX(CASE WHEN UPPER(c.column_name) = 'REGION' THEN 1 ELSE 0 END) AS region_exist,
+    MAX(CASE WHEN UPPER(c.column_name) = 'TAGS' THEN 1 ELSE 0 END) AS tags_exist
 FROM 
     {{ full_table_name("INFORMATION_SCHEMA.TABLES") }} t
 INNER JOIN 

diff --git a/transformations/aws/asset-inventory-free/models/aws_resources.sql b/transformations/aws/asset-inventory-free/models/aws_resources.sql
@@ -5,7 +5,7 @@
 -- Generate dynamic SQL statements
 {% for row in run_query(aws_tables) -%}
     {% if row.table_name is not none and row.table_name != '' -%}
-        {{ aws_asset_resources(row.table_name, row.ARN_EXIST, row.ACCOUNT_ID_EXIST, row.REQUEST_ACCOUNT_ID_EXIST, row.REGION_EXIST, row.TAGS_EXIST) }}
+        {{ aws_asset_resources(row.table_name, row.arn_exist, row.account_id_exist, row.request_account_id_exist, row.region_exist, row.tags_exist) }}
         {% if not loop.last -%} UNION ALL {% endif -%}
     {%- endif %}
 {%- endfor %}