chore: Better publish process (#271)

cloudquery · Nov 24, 2023 · 687842f · 687842f
1 parent f19a76c
commit 687842f
Show file tree

Hide file tree

Showing 208 changed files with 2,596 additions and 74 deletions.
diff --git a/...sformations_azure_compliance_postgres.yml → ...ations_azure_compliance_free_postgres.yml b/...sformations_azure_compliance_postgres.yml → ...ations_azure_compliance_free_postgres.yml
@@ -1,24 +1,25 @@
-name: "Test Azure Compliance Policies: Postgres"
+name: "Test Azure Compliance Free Policies: Postgres"
 
 on:
   pull_request:
     paths:
-      - "transformations/azure_compliance/**"
-      - ".github/workflows/transformations_azure_compliance_postgres.yml"
+      - "transformations/azure/compliance-free/**"
+      - ".github/workflows/transformations_azure_compliance_free_postgres.yml"
   push:
     branches:
       - main
     paths:
-      - "transformations/azure_compliance/**"
-      - ".github/workflows/transformations_azure_compliance_postgres.yml"
+      - "transformations/azure/compliance-free/**"
+      - ".github/workflows/transformations_azure_compliance_free_postgres.yml"
 
 jobs:
-  test-policies:
+  transformations-azure-compliance-free:
+    name: transformations/azure/compliance-free
     timeout-minutes: 30
     runs-on: ubuntu-latest
     defaults:
       run:
-        working-directory: ./transformations/azure_compliance
+        working-directory: ./transformations/azure/compliance-free
     services:
       postgres:
         image: postgres:11
@@ -41,17 +42,17 @@ jobs:
         with:
           python-version: "3.9"
           cache: "pip"
-          cache-dependency-path: "./transformations/azure_compliance/requirements.txt"
+          cache-dependency-path: "./transformations/azure/compliance-free/requirements.txt"
       - name: Install dependencies
         run: pip install -r requirements.txt
       - name: Setup CloudQuery
         uses: cloudquery/setup-cloudquery@v3
         with:
           version: v4.0.0
       - name: Migrate DB
-        run: cloudquery migrate tests/azure.yml tests/pg.yml 
+        run: cloudquery migrate tests/azure.yml tests/pg.yml
         env:
-          CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres  
+          CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
       - name: DBT dependencies
         run: |
           dbt deps --target dev-pg --profiles-dir ./tests

diff --git a/.github/workflows/transformations_azure_compliance_premium_postgres.yml b/.github/workflows/transformations_azure_compliance_premium_postgres.yml
@@ -0,0 +1,61 @@
+name: "Test Azure Compliance Premium Policies: Postgres"
+
+on:
+  pull_request:
+    paths:
+      - "transformations/azure/compliance-premium/**"
+      - ".github/workflows/transformations_azure_compliance_premium_postgres.yml"
+  push:
+    branches:
+      - main
+    paths:
+      - "transformations/azure/compliance-premium/**"
+      - ".github/workflows/transformations_azure_compliance_premium_postgres.yml"
+
+jobs:
+  transformations-azure-compliance-premium:
+    name: transformations/azure/compliance-premium
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./transformations/azure/compliance-premium
+    services:
+      postgres:
+        image: postgres:11
+        env:
+          POSTGRES_PASSWORD: pass
+          POSTGRES_USER: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+          cache: "pip"
+          cache-dependency-path: "./transformations/azure/compliance-premium/requirements.txt"
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+      - name: Setup CloudQuery
+        uses: cloudquery/setup-cloudquery@v3
+        with:
+          version: v3.29.1
+      - name: Migrate DB
+        run: cloudquery migrate tests/azure.yml tests/pg.yml
+        env:
+          CQ_DSN: postgresql://postgres:pass@localhost:5432/postgres
+      - name: DBT dependencies
+        run: |
+          dbt deps --target dev-pg --profiles-dir ./tests
+      - name: Run Policies
+        run: |
+          dbt run --target dev-pg --profiles-dir ./tests
diff --git a/release-please-config-free.json b/release-please-config-free.json
@@ -4,10 +4,6 @@
       "component": "transformation-aws-compliance-free",
       "changelog-path": "CHANGELOG-free.md"
     },
-    "transformations/azure_compliance": {
-      "component": "transformation-azure-compliance-free",
-      "changelog-path": "CHANGELOG-free.md"
-    },
     "transformations/gcp_compliance": {
       "component": "transformation-gcp-compliance-free",
       "changelog-path": "CHANGELOG-free.md"

diff --git a/release-please-config-premium.json b/release-please-config-premium.json
@@ -4,10 +4,6 @@
       "component": "transformation-aws-compliance-premium",
       "changelog-path": "CHANGELOG-premium.md"
     },
-    "transformations/azure_compliance": {
-      "component": "transformation-azure-compliance-premium",
-      "changelog-path": "CHANGELOG-premium.md"
-    },
     "transformations/gcp_compliance": {
       "component": "transformation-gcp-compliance-premium",
       "changelog-path": "CHANGELOG-premium.md"

diff --git a/scripts/dbt-pack/.eslintrc.cjs b/scripts/dbt-pack/.eslintrc.cjs
@@ -0,0 +1,13 @@
+module.exports = {
+  root: true,
+  parserOptions: {
+    ecmaVersion: 12,
+    sourceType: "module",
+  },
+  plugins: ["unicorn"],
+  extends: ["eslint:recommended", "prettier", "plugin:unicorn/recommended"],
+  env: {
+    es2021: true,
+    node: true,
+  },
+};
diff --git a/scripts/dbt-pack/.gitignore b/scripts/dbt-pack/.gitignore
@@ -0,0 +1 @@
+node_modules
diff --git a/scripts/dbt-pack/.prettierrc b/scripts/dbt-pack/.prettierrc
@@ -0,0 +1 @@
+{}
diff --git a/scripts/dbt-pack/README.md b/scripts/dbt-pack/README.md
@@ -0,0 +1,20 @@
+# dbt-pack
+
+A utility to pack a dbt project into a single zip file.
+
+## Prerequisites
+
+- [Node.js LTS](https://nodejs.org/en/)
+
+## Setup
+
+```bash
+npm ci
+```
+
+## Usage
+
+```bash
+node index.js dbt-pack --project-dir=../../transformations/azure/compliance-free
+node index.js dbt-pack --project-dir=../../transformations/azure/compliance-premium
+```
diff --git a/scripts/dbt-pack/index.js b/scripts/dbt-pack/index.js
@@ -0,0 +1,26 @@
+#!/usr/bin/env node
+import path from "node:path";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import { pack } from "./src/index.js";
+
+yargs(hideBin(process.argv))
+  .command(
+    "dbt-pack",
+    "Package dbt project into a single zip",
+    () => {},
+    async ({ projectDir }) => {
+      await pack({
+        projectDir: path.resolve(projectDir),
+      });
+    },
+  )
+  .option("project-dir", {
+    alias: "p",
+    type: "string",
+    description: "Path to dbt project directory",
+    demandOption: true,
+  })
+  .demandCommand(1)
+  .strict()
+  .parse();