chore: Reorganize gcp (#275)

.github/workflows/transformations_gcp_compliance_postgres.yml → .github/workflows/transformations_gcp_compliance_free_postgres.yml

@@ -1,24 +1,25 @@
-name: "Test GCP Compliance Policies: Postgres"
+name: "Test GCP Compliance Free Policies: Postgres"
 
 on:
   pull_request:
     paths:
-      - "transformations/gcp_compliance/**"
-      - ".github/workflows/transformations_gcp_compliance_postgres.yml"
+      - "transformations/gcp/compliance-free/**"
+      - ".github/workflows/transformations_gcp_compliance_free_postgres.yml"
   push:
     branches:
       - main
     paths:
-      - "transformations/gcp_compliance/**"
-      - ".github/workflows/transformations_gcp_compliance_postgres.yml"
+      - "transformations/gcp/compliance-free/**"
+      - ".github/workflows/transformations_gcp_compliance_free_postgres.yml"
 
 jobs:
-  test-policies:
+  transformations-gcp-compliance-free:
+    name: transformations/gcp/compliance-free
     timeout-minutes: 30
     runs-on: ubuntu-latest
     defaults:
       run:
-        working-directory: ./transformations/gcp_compliance
+        working-directory: ./transformations/gcp/compliance-free
     services:
       postgres:
         image: postgres:11
@@ -41,17 +42,17 @@ jobs:
         with:
           python-version: "3.9"
           cache: "pip"
-          cache-dependency-path: "./transformations/gcp_compliance/requirements.txt"
+          cache-dependency-path: "./transformations/gcp/compliance-free/requirements.txt"
       - name: Install dependencies
         run: pip install -r requirements.txt
       - name: Setup CloudQuery
         uses: cloudquery/setup-cloudquery@v3
         with:
           version: v4.0.0
       - name: Migrate DB
-        run: cloudquery migrate tests/gcp.yml tests/pg.yml 
+        run: cloudquery migrate tests/spec.yml
         env:
-          CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres  
+          CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
       - name: DBT dependencies
         run: |
           dbt deps --target dev-pg --profiles-dir ./tests

.github/workflows/transformations_gcp_compliance_premium_postgres.yml

@@ -0,0 +1,61 @@
+name: "Test GCP Compliance Premium Policies: Postgres"
+
+on:
+  pull_request:
+    paths:
+      - "transformations/gcp/compliance-premium/**"
+      - ".github/workflows/transformations_gcp_compliance_premium_postgres.yml"
+  push:
+    branches:
+      - main
+    paths:
+      - "transformations/gcp/compliance-premium/**"
+      - ".github/workflows/transformations_gcp_compliance_premium_postgres.yml"
+
+jobs:
+  transformations-gcp-compliance-premium:
+    name: transformations/gcp/compliance-premium
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./transformations/gcp/compliance-premium
+    services:
+      postgres:
+        image: postgres:11
+        env:
+          POSTGRES_PASSWORD: pass
+          POSTGRES_USER: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+          cache: "pip"
+          cache-dependency-path: "./transformations/gcp/compliance-premium/requirements.txt"
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+      - name: Setup CloudQuery
+        uses: cloudquery/setup-cloudquery@v3
+        with:
+          version: v4.0.0
+      - name: Migrate DB
+        run: cloudquery migrate tests/spec.yml
+        env:
+          CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
+      - name: DBT dependencies
+        run: |
+          dbt deps --target dev-pg --profiles-dir ./tests
+      - name: Run Policies
+        run: |
+          dbt run --target dev-pg --profiles-dir ./tests

release-please-config-free.json

@@ -4,10 +4,6 @@
       "component": "transformation-aws-compliance-free",
       "changelog-path": "CHANGELOG-free.md"
     },
-    "transformations/gcp_compliance": {
-      "component": "transformation-gcp-compliance-free",
-      "changelog-path": "CHANGELOG-free.md"
-    },
     "transformations/aws_data_resilience": {
       "component": "transformation-aws-data-resilience",
       "changelog-path": "CHANGELOG.md"

release-please-config-premium.json

@@ -3,10 +3,6 @@
     "transformations/aws_compliance": {
       "component": "transformation-aws-compliance-premium",
       "changelog-path": "CHANGELOG-premium.md"
-    },
-    "transformations/gcp_compliance": {
-      "component": "transformation-gcp-compliance-premium",
-      "changelog-path": "CHANGELOG-premium.md"
     }
   },
   "pull-request-title-pattern": "chore${scope}: Release${component} v${version}",

scripts/dbt-pack/README.md

@@ -20,4 +20,7 @@ node index.js dbt-pack --project-dir=../../transformations/azure/compliance-prem
 
 node index.js dbt-pack --project-dir=../../transformations/k8s/compliance-free
 node index.js dbt-pack --project-dir=../../transformations/k8s/compliance-premium
+
+node index.js dbt-pack --project-dir=../../transformations/gcp/compliance-free
+node index.js dbt-pack --project-dir=../../transformations/gcp/compliance-premium
 ```

transformations/gcp_compliance/dbt_project.yml → transformations/gcp/compliance-free/dbt_project.yml

@@ -12,11 +12,11 @@ profile: 'gcp_compliance'
 # These configurations specify where dbt should look for different types of files.
 # The `model-paths` config, for example, states that models in this project can be
 # found in the "models/" directory. You probably won't need to change these!
-model-paths: ["models"]
+model-paths: ["models", "../models"]
 analysis-paths: ["analyses"]
 test-paths: ["tests"]
 seed-paths: ["seeds"]
-macro-paths: ["macros"]
+macro-paths: ["macros", "../macros"]
 snapshot-paths: ["snapshots"]
 
 clean-targets:         # directories to be removed by `dbt clean`

transformations/gcp_compliance/manifest-free.json → transformations/gcp/compliance-free/manifest.json

@@ -6,7 +6,7 @@
   "addon_type": "transformation",
   "addon_format": "zip",
   "message": "@./changelog.md",
-  "doc": "./README-free.md",
+  "doc": "./README.md",
   "path": "./build/gcp_compliance_free.zip",
   "plugin_deps": ["cloudquery/source/[email protected]"],
   "addon_deps": []

transformations/gcp/compliance-free/tests/spec.yml

@@ -0,0 +1,17 @@
+kind: source
+spec:
+  name: gcp
+  path: cloudquery/gcp
+  version: "v10.0.0" # latest version of source gcp plugin
+  destinations: ["postgresql"]
+  tables: ["*"]
+---
+kind: destination
+spec:
+  name: "postgresql"
+  path: "cloudquery/postgresql"
+  registry: cloudquery
+  version: "v7.1.0" # latest version of postgresql plugin
+  spec:
+    batch_size: 10000
+    connection_string: ${CQ_DSN}

transformations/gcp/compliance-premium/.gitignore

@@ -0,0 +1,4 @@
+
+target/
+dbt_packages/
+logs/

transformations/gcp/compliance-premium/dbt_project.yml

@@ -0,0 +1,40 @@
+
+# Name your project! Project names should contain only lowercase characters
+# and underscores. A good package name should reflect your organization's
+# name or the intended use of these models
+name: 'gcp_compliance'
+version: '1.0.0'
+config-version: 2
+
+# This setting configures which "profile" dbt uses for this project.
+profile: 'gcp_compliance'
+
+# These configurations specify where dbt should look for different types of files.
+# The `model-paths` config, for example, states that models in this project can be
+# found in the "models/" directory. You probably won't need to change these!
+model-paths: ["models", "../models"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros", "../macros"]
+snapshot-paths: ["snapshots"]
+
+clean-targets:         # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_packages"
+
+# Configuring models
+# Full documentation: https://docs.getdbt.com/docs/configuring-models
+
+# In this example config, we tell dbt to build all models in the example/
+# directory as views. These settings can be overridden in the individual model
+# files using the `{{ config(...) }}` macro.
+models:
+  gcp_compliance:
+    # Config indicated by + and applies to all files under models/example/
+    # example:
+    #   +materialized: view
+
+
+
+

transformations/gcp_compliance/manifest-premium.json → transformations/gcp/compliance-premium/manifest.json

@@ -6,7 +6,7 @@
   "addon_type": "transformation",
   "addon_format": "zip",
   "message": "@./changelog.md",
-  "doc": "./README-premium.md",
+  "doc": "./README.md",
   "path": "./build/gcp_compliance_premium.zip",
   "plugin_deps": ["cloudquery/source/[email protected]"],
   "addon_deps": []

transformations/gcp/compliance-premium/requirements.txt

@@ -0,0 +1 @@
+dbt-postgres==1.6.6

transformations/gcp/compliance-premium/tests/profiles.yml

@@ -0,0 +1,23 @@
+gcp_compliance: # This should match the name in your dbt_project.yml
+  target: dev
+  outputs:
+    dev-snowflake:
+      type: snowflake
+      account: "{{ env_var('SNOW_ACCOUNT') }}"
+
+      # User/password auth
+      user:  "{{ env_var('SNOW_USER') }}"
+      password: "{{ env_var('SNOW_PASSWORD') }}"
+
+      database: "{{ env_var('SNOW_DATABASE') }}"
+      warehouse: "{{ env_var('SNOW_WAREHOUSE') }}"
+      schema: "{{ env_var('SNOW_SCHEMA') }}"
+    dev-pg:
+      type: postgres
+      host: 127.0.0.1
+      user: postgres
+      pass: pass
+      port: 5432
+      dbname: postgres
+      schema: public # default schema where dbt will build the models
+      threads: 1 # number of threads to use when running in parallel

transformations/gcp/compliance-premium/tests/spec.yml

@@ -0,0 +1,17 @@
+kind: source
+spec:
+  name: gcp
+  path: cloudquery/gcp
+  version: "v10.0.0" # latest version of source gcp plugin
+  destinations: ["postgresql"]
+  tables: ["*"]
+---
+kind: destination
+spec:
+  name: "postgresql"
+  path: "cloudquery/postgresql"
+  registry: cloudquery
+  version: "v7.1.0" # latest version of postgresql plugin
+  spec:
+    batch_size: 10000
+    connection_string: ${CQ_DSN}