From c2754f7a4e6dae95653870a333b245bee267959c Mon Sep 17 00:00:00 2001 From: k-rheinheimer <114611405+k-rheinheimer@users.noreply.github.com> Date: Thu, 29 Aug 2024 11:49:52 -0400 Subject: [PATCH] fix: Removing line breaks (#1052) * removing line breaks * updating versions * adding skip dependent tables to test to shorten length * adding more whitespace trimming * more whitespace trimming --- .../macros/aws_asset_resources.sql | 8 ++++---- .../aws/asset-inventory-free/models/aws_resources.sql | 11 +++++------ .../aws/asset-inventory-free/tests/bigquery.yml | 5 +++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/transformations/aws/asset-inventory-free/macros/aws_asset_resources.sql b/transformations/aws/asset-inventory-free/macros/aws_asset_resources.sql index a25da2693..0861f4cea 100644 --- a/transformations/aws/asset-inventory-free/macros/aws_asset_resources.sql +++ b/transformations/aws/asset-inventory-free/macros/aws_asset_resources.sql @@ -73,8 +73,8 @@ SPLIT_PART(arn, ':', 3) AS service, FROM {{ table_name | string }} {% endmacro %} -{% macro bigquery__aws_asset_resources(table_name, ARN_EXIST, ACCOUNT_ID_EXIST, REQUEST_ACCOUNT_ID_EXIST, REGION_EXIST, TAGS_EXIST) -%} -SELECT -_cq_id, _cq_source_name, _cq_sync_time,{% if ACCOUNT_ID_EXIST %} account_id {% else %} SPLIT(arn, ':')[5] {% endif %} AS account_id, {% if REQUEST_ACCOUNT_ID_EXIST %} request_account_id {% else %} SPLIT(arn, ':')[5] {% endif %} AS request_account_id, CASE WHEN SPLIT(SPLIT(arn, ':')[6], '/')[2] = '' AND SPLIT(arn, ':')[7] = '' THEN NULL ELSE SPLIT(SPLIT(arn, ':')[6], '/')[1] END AS TYPE, arn, {% if REGION_EXIST %} region {% else %} 'unavailable' {% endif %} AS region, {% if TAGS_EXIST %} TO_JSON_STRING(tags) {% else %} TO_JSON_STRING(STRUCT()) {% endif %} AS tags, SPLIT(arn, ':')[2] AS PARTITIONS, SPLIT(arn, ':')[3] AS service, '{{ table_name | string }}' AS _cq_table +{%- macro bigquery__aws_asset_resources(table_name, ARN_EXIST, ACCOUNT_ID_EXIST, REQUEST_ACCOUNT_ID_EXIST, REGION_EXIST, TAGS_EXIST) -%} +SELECT +_cq_id, _cq_source_name, _cq_sync_time,{% if ACCOUNT_ID_EXIST %} account_id {% else %} SPLIT(arn, ':')[5] {% endif %} AS account_id, {% if REQUEST_ACCOUNT_ID_EXIST %} request_account_id {% else %} SPLIT(arn, ':')[5] {% endif %} AS request_account_id, CASE WHEN SPLIT(SPLIT(arn, ':')[6], '/')[2] = '' AND SPLIT(arn, ':')[7] = '' THEN NULL ELSE SPLIT(SPLIT(arn, ':')[6], '/')[1] END AS TYPE, arn, {% if REGION_EXIST %} region {% else %} 'unavailable' {% endif %} AS region, {% if TAGS_EXIST %} TO_JSON_STRING(tags) {% else %} TO_JSON_STRING(STRUCT()) {% endif %} AS tags, SPLIT(arn, ':')[2] AS PARTITIONS, SPLIT(arn, ':')[3] AS service, '{{ table_name | string }}' AS _cq_table FROM {{ full_table_name(table_name | string) }} -{% endmacro %} \ No newline at end of file +{%- endmacro -%} \ No newline at end of file diff --git a/transformations/aws/asset-inventory-free/models/aws_resources.sql b/transformations/aws/asset-inventory-free/models/aws_resources.sql index 8f5e76a0b..927cf4727 100644 --- a/transformations/aws/asset-inventory-free/models/aws_resources.sql +++ b/transformations/aws/asset-inventory-free/models/aws_resources.sql @@ -2,10 +2,9 @@ {{ aws_tables_dyn() }} {% endset %} --- Generate dynamic SQL statements -{% for row in run_query(aws_tables) -%} - {% if row.table_name is not none and row.table_name != '' -%} +{%- for row in run_query(aws_tables) -%} + {%- if row.table_name is not none and row.table_name != '' -%} {{ aws_asset_resources(row.table_name, row.arn_exist, row.account_id_exist, row.request_account_id_exist, row.region_exist, row.tags_exist) }} - {% if not loop.last -%} UNION ALL {% endif -%} - {%- endif %} -{%- endfor %} \ No newline at end of file + {%- if not loop.last -%} UNION ALL {% endif -%} + {%- endif -%} +{%- endfor -%} \ No newline at end of file diff --git a/transformations/aws/asset-inventory-free/tests/bigquery.yml b/transformations/aws/asset-inventory-free/tests/bigquery.yml index 467e9dcb2..61edaac1b 100644 --- a/transformations/aws/asset-inventory-free/tests/bigquery.yml +++ b/transformations/aws/asset-inventory-free/tests/bigquery.yml @@ -3,16 +3,17 @@ spec: name: aws path: cloudquery/aws registry: cloudquery - version: "v27.15.0" # latest version of source aws plugin + version: "v27.17.0" # latest version of source aws plugin destinations: ["bigquery"] tables: ["*"] + skip_dependent_tables: true --- kind: destination spec: name: bigquery path: cloudquery/bigquery registry: cloudquery - version: "v3.3.9" + version: "v4.1.7" write_mode: "append" spec: project_id: cq-integration-tests