From 458d02bcc1a10edf7791cd12b349d4aa8a9e1a0f Mon Sep 17 00:00:00 2001 From: erezrokah Date: Mon, 4 Dec 2023 19:03:34 +0100 Subject: [PATCH] chore: Auto add checks to readme --- ...ormations_aws_compliance_free_postgres.yml | 9 +- ...ations_aws_compliance_premium_postgres.yml | 9 +- scripts/dbt-pack/index.js | 12 ++- scripts/dbt-pack/src/index.js | 84 +++++++++++++++---- transformations/aws/compliance-free/Makefile | 3 + transformations/aws/compliance-free/README.md | 5 +- .../aws/compliance-premium/Makefile | 3 + .../aws/compliance-premium/README.md | 5 +- 8 files changed, 111 insertions(+), 19 deletions(-) create mode 100644 transformations/aws/compliance-free/Makefile create mode 100644 transformations/aws/compliance-premium/Makefile diff --git a/.github/workflows/transformations_aws_compliance_free_postgres.yml b/.github/workflows/transformations_aws_compliance_free_postgres.yml index 26e8c97ef..62bc94979 100644 --- a/.github/workflows/transformations_aws_compliance_free_postgres.yml +++ b/.github/workflows/transformations_aws_compliance_free_postgres.yml @@ -81,4 +81,11 @@ jobs: - name: Run Unpacked DBT working-directory: ./transformations/aws/compliance-free/build/aws_compliance_free run: | - dbt run --target dev-pg --profiles-dir ../../tests \ No newline at end of file + dbt run --target dev-pg --profiles-dir ../../tests + - name: Gen docs + run: | + make gen-docs + - uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: "chore: Update readme" + file_pattern: 'transformations/aws/compliance-free/README.md' \ No newline at end of file diff --git a/.github/workflows/transformations_aws_compliance_premium_postgres.yml b/.github/workflows/transformations_aws_compliance_premium_postgres.yml index b7a3f4e14..45902c4d8 100644 --- a/.github/workflows/transformations_aws_compliance_premium_postgres.yml +++ b/.github/workflows/transformations_aws_compliance_premium_postgres.yml @@ -81,4 +81,11 @@ jobs: - name: Run Unpacked DBT working-directory: ./transformations/aws/compliance-premium/build/aws_compliance_premium run: | - dbt run --target dev-pg --profiles-dir ../../tests --exclude aws_compliance__foundational_security \ No newline at end of file + dbt run --target dev-pg --profiles-dir ../../tests --exclude aws_compliance__foundational_security + - name: Gen docs + run: | + make gen-docs + - uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: "chore: Update readme" + file_pattern: 'transformations/aws/compliance-premium/README.md' \ No newline at end of file diff --git a/scripts/dbt-pack/index.js b/scripts/dbt-pack/index.js index 353362b23..eb7809c6d 100644 --- a/scripts/dbt-pack/index.js +++ b/scripts/dbt-pack/index.js @@ -2,7 +2,7 @@ import path from "node:path"; import yargs from "yargs"; import { hideBin } from "yargs/helpers"; -import { pack } from "./src/index.js"; +import { pack, appendQueries } from "./src/index.js"; yargs(hideBin(process.argv)) .command( @@ -15,6 +15,16 @@ yargs(hideBin(process.argv)) }); }, ) + .command( + "dbt-docs", + "Append checks to README.md", + () => {}, + async ({ projectDir }) => { + await appendQueries({ + projectDir: path.resolve(projectDir), + }); + }, + ) .option("project-dir", { alias: "p", type: "string", diff --git a/scripts/dbt-pack/src/index.js b/scripts/dbt-pack/src/index.js index 4c91e1172..975e06f94 100644 --- a/scripts/dbt-pack/src/index.js +++ b/scripts/dbt-pack/src/index.js @@ -11,20 +11,18 @@ const validateProjectDirectory = async (dbtProjectDirectory) => { `dbt project directory '${dbtProjectDirectory}' does not exist`, ); } - if (!(await pathExists(`${dbtProjectDirectory}/dbt_project.yml`))) { - throw new Error( - `dbt project directory '${dbtProjectDirectory}' does not contain a dbt_project.yml file`, - ); - } - if (!(await pathExists(`${dbtProjectDirectory}/requirements.txt`))) { - throw new Error( - `dbt project directory '${dbtProjectDirectory}' does not contain a requirements.txt file`, - ); - } - if (!(await pathExists(`${dbtProjectDirectory}/manifest.json`))) { - throw new Error( - `dbt project directory '${dbtProjectDirectory}' does not contain a manifest.json file`, - ); + const requiredFiles = [ + "dbt_project.yml", + "requirements.txt", + "manifest.json", + "README.md", + ]; + for (const requiredFile of requiredFiles) { + if (!(await pathExists(`${dbtProjectDirectory}/${requiredFile}`))) { + throw new Error( + `dbt project directory '${dbtProjectDirectory}' does not contain a ${requiredFile} file`, + ); + } } }; @@ -129,3 +127,61 @@ export const pack = async ({ projectDir }) => { const filesToPack = await analyzeManifestFile(projectDir); await zipProject(projectDir, filesToPack); }; + +const getModels = async (dbtProjectDirectory) => { + const dirents = await fs.readdir(`${dbtProjectDirectory}/models`, { + withFileTypes: true, + }); + const models = dirents + .filter((dirent) => dirent.isFile() && dirent.name.endsWith(".sql")) + .map((dirent) => dirent.name); + const withContent = await Promise.all( + models.map(async (model) => + fs.readFile(`${dbtProjectDirectory}/models/${model}`, "utf8"), + ), + ); + return withContent; +}; + +const getChecksByFramework = async (models) => { + const pattern = /\({{ .*?\('(.*?)','(.*?)'\) }}\)/g; + const queries = models.flatMap((model) => { + const matches = [...model.matchAll(pattern)]; + if (matches.length > 0) { + return matches.map((match) => { + const [, framework, checkId] = match; + return { framework, checkId }; + }).sort((a, b) => a.checkId.localeCompare(b.checkId, undefined, { numeric: true })); + } + return []; + }); + // eslint-disable-next-line unicorn/no-array-reduce + return queries.reduce((accumulator, { framework, checkId }) => { + if (!accumulator[framework]) { + accumulator[framework] = []; + } + accumulator[framework].push(checkId); + return accumulator; + }, {}); +} + +const updateReadme = async (dbtProjectDirectory, checksByFramework) => { + const readmeFile = `${dbtProjectDirectory}/README.md`; + const readme = await fs.readFile(readmeFile, "utf8"); + const pattern = /[\S\s]*/; + const includedChecks = Object.entries(checksByFramework).map(([framework, checkIds]) => { + const listItems = checkIds.map((checkId) => `- [x] \`${checkId}\``).join("\n"); + return `\n\n##### \`${framework}\`\n\n${listItems}`; + }).join(""); + const updatedReadme = readme.replace(pattern, ` +#### Included Checks${includedChecks} +`); + await fs.writeFile(readmeFile, updatedReadme); +} + +export const appendQueries = async ({ projectDir }) => { + await validateProjectDirectory(projectDir); + const models = await getModels(projectDir); + const checksByFramework = await getChecksByFramework(models); + await updateReadme(projectDir, checksByFramework); +}; diff --git a/transformations/aws/compliance-free/Makefile b/transformations/aws/compliance-free/Makefile new file mode 100644 index 000000000..d1bb66d3e --- /dev/null +++ b/transformations/aws/compliance-free/Makefile @@ -0,0 +1,3 @@ +.PHONY: gen-docs +gen-docs: + node ../../../scripts/dbt-pack/index.js dbt-docs --project-dir . diff --git a/transformations/aws/compliance-free/README.md b/transformations/aws/compliance-free/README.md index 9d9ea1774..50ed927c8 100644 --- a/transformations/aws/compliance-free/README.md +++ b/transformations/aws/compliance-free/README.md @@ -22,4 +22,7 @@ The pack contains the free version. - **aws_compliance_pci_dss_v3.2.1**: AWS PCI DSS V3.2.1 benchmark. - **aws_compliance\_\_foundational_security**: AWS Foundational Security benchmark, available only for Snowflake -The free version contains 10% of the full pack's queries. \ No newline at end of file +The free version contains 10% of the full pack's checks. + + + \ No newline at end of file diff --git a/transformations/aws/compliance-premium/Makefile b/transformations/aws/compliance-premium/Makefile new file mode 100644 index 000000000..d1bb66d3e --- /dev/null +++ b/transformations/aws/compliance-premium/Makefile @@ -0,0 +1,3 @@ +.PHONY: gen-docs +gen-docs: + node ../../../scripts/dbt-pack/index.js dbt-docs --project-dir . diff --git a/transformations/aws/compliance-premium/README.md b/transformations/aws/compliance-premium/README.md index 4faffd20b..674d16306 100644 --- a/transformations/aws/compliance-premium/README.md +++ b/transformations/aws/compliance-premium/README.md @@ -22,4 +22,7 @@ The pack contains the premium version. - **aws_compliance_pci_dss_v3.2.1**: AWS PCI DSS V3.2.1 benchmark. - **aws_compliance\_\_foundational_security**: AWS Foundational Security benchmark, available only for Snowflake -The premium version contains all queries. \ No newline at end of file +The premium version contains all checks. + + + \ No newline at end of file