chore: Auto add checks to readme

.github/workflows/transformations_aws_compliance_free_postgres.yml

@@ -81,4 +81,11 @@ jobs:
       - name: Run Unpacked DBT
         working-directory: ./transformations/aws/compliance-free/build/aws_compliance_free
         run: |
-          dbt run --target dev-pg --profiles-dir ../../tests
+          dbt run --target dev-pg --profiles-dir ../../tests
+      - name: Gen docs
+        run: |
+          make gen-docs
+      - uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: "chore: Update readme"
+          file_pattern: 'transformations/aws/compliance-free/README.md'

.github/workflows/transformations_aws_compliance_premium_postgres.yml

@@ -81,4 +81,11 @@ jobs:
       - name: Run Unpacked DBT
         working-directory: ./transformations/aws/compliance-premium/build/aws_compliance_premium
         run: |
-          dbt run --target dev-pg --profiles-dir ../../tests --exclude aws_compliance__foundational_security
+          dbt run --target dev-pg --profiles-dir ../../tests --exclude aws_compliance__foundational_security
+      - name: Gen docs
+        run: |
+          make gen-docs
+      - uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          commit_message: "chore: Update readme"
+          file_pattern: 'transformations/aws/compliance-premium/README.md'

scripts/dbt-pack/index.js

@@ -2,7 +2,7 @@
 import path from "node:path";
 import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
-import { pack } from "./src/index.js";
+import { pack, appendQueries } from "./src/index.js";
 
 yargs(hideBin(process.argv))
   .command(
@@ -15,6 +15,16 @@ yargs(hideBin(process.argv))
       });
     },
   )
+  .command(
+    "dbt-docs",
+    "Append checks to README.md",
+    () => {},
+    async ({ projectDir }) => {
+      await appendQueries({
+        projectDir: path.resolve(projectDir),
+      });
+    },
+  )
   .option("project-dir", {
     alias: "p",
     type: "string",

scripts/dbt-pack/src/index.js

@@ -11,20 +11,18 @@ const validateProjectDirectory = async (dbtProjectDirectory) => {
       `dbt project directory '${dbtProjectDirectory}' does not exist`,
     );
   }
-  if (!(await pathExists(`${dbtProjectDirectory}/dbt_project.yml`))) {
-    throw new Error(
-      `dbt project directory '${dbtProjectDirectory}' does not contain a dbt_project.yml file`,
-    );
-  }
-  if (!(await pathExists(`${dbtProjectDirectory}/requirements.txt`))) {
-    throw new Error(
-      `dbt project directory '${dbtProjectDirectory}' does not contain a requirements.txt file`,
-    );
-  }
-  if (!(await pathExists(`${dbtProjectDirectory}/manifest.json`))) {
-    throw new Error(
-      `dbt project directory '${dbtProjectDirectory}' does not contain a manifest.json file`,
-    );
+  const requiredFiles = [
+    "dbt_project.yml",
+    "requirements.txt",
+    "manifest.json",
+    "README.md",
+  ];
+  for (const requiredFile of requiredFiles) {
+    if (!(await pathExists(`${dbtProjectDirectory}/${requiredFile}`))) {
+      throw new Error(
+        `dbt project directory '${dbtProjectDirectory}' does not contain a ${requiredFile} file`,
+      );
+    }
   }
 };
 
@@ -129,3 +127,61 @@ export const pack = async ({ projectDir }) => {
   const filesToPack = await analyzeManifestFile(projectDir);
   await zipProject(projectDir, filesToPack);
 };
+
+const getModels = async (dbtProjectDirectory) => {
+  const dirents = await fs.readdir(`${dbtProjectDirectory}/models`, {
+    withFileTypes: true,
+  });
+  const models = dirents
+    .filter((dirent) => dirent.isFile() && dirent.name.endsWith(".sql"))
+    .map((dirent) => dirent.name);
+  const withContent = await Promise.all(
+    models.map(async (model) =>
+      fs.readFile(`${dbtProjectDirectory}/models/${model}`, "utf8"),
+    ),
+  );
+  return withContent;
+};
+
+const getChecksByFramework = async (models) => {
+  const pattern = /\({{ .*?\('(.*?)','(.*?)'\) }}\)/g;
+  const queries = models.flatMap((model) => {
+    const matches = [...model.matchAll(pattern)];
+    if (matches.length > 0) {
+      return matches.map((match) => {
+        const [, framework, checkId] = match;
+        return { framework, checkId };
+      }).sort((a, b) => a.checkId.localeCompare(b.checkId, undefined, { numeric: true }));
+    }
+    return [];
+  });
+  // eslint-disable-next-line unicorn/no-array-reduce
+  return queries.reduce((accumulator, { framework, checkId }) => {
+    if (!accumulator[framework]) {
+      accumulator[framework] = [];
+    }
+    accumulator[framework].push(checkId);
+    return accumulator;
+  }, {});
+}
+
+const updateReadme = async (dbtProjectDirectory, checksByFramework) => {
+  const readmeFile = `${dbtProjectDirectory}/README.md`;
+  const readme = await fs.readFile(readmeFile, "utf8");
+  const pattern = /<!-- AUTO-GENERATED-INCLUDED-CHECKS-START -->[\S\s]*<!-- AUTO-GENERATED-INCLUDED-CHECKS-END -->/;
+  const includedChecks = Object.entries(checksByFramework).map(([framework, checkIds]) => {
+    const listItems = checkIds.map((checkId) => `- [x] \`${checkId}\``).join("\n");
+    return `\n\n##### \`${framework}\`\n\n${listItems}`;
+  }).join("");
+  const updatedReadme = readme.replace(pattern, `<!-- AUTO-GENERATED-INCLUDED-CHECKS-START -->
+#### Included Checks${includedChecks}
+<!-- AUTO-GENERATED-INCLUDED-CHECKS-END -->`);
+  await fs.writeFile(readmeFile, updatedReadme);
+}
+
+export const appendQueries = async ({ projectDir }) => {
+  await validateProjectDirectory(projectDir);
+  const models = await getModels(projectDir);
+  const checksByFramework = await getChecksByFramework(models);
+  await updateReadme(projectDir, checksByFramework);
+};

transformations/aws/compliance-free/Makefile

@@ -0,0 +1,3 @@
+.PHONY: gen-docs
+gen-docs:
+	node ../../../scripts/dbt-pack/index.js dbt-docs --project-dir .

transformations/aws/compliance-free/README.md

@@ -22,4 +22,7 @@ The pack contains the free version.
 - **aws_compliance_pci_dss_v3.2.1**: AWS PCI DSS V3.2.1 benchmark.
 - **aws_compliance\_\_foundational_security**: AWS Foundational Security benchmark, available only for Snowflake
 
-The free version contains 10% of the full pack's queries.
+The free version contains 10% of the full pack's checks.
+
+<!-- AUTO-GENERATED-INCLUDED-CHECKS-START -->
+<!-- AUTO-GENERATED-INCLUDED-CHECKS-END -->

transformations/aws/compliance-premium/Makefile

@@ -0,0 +1,3 @@
+.PHONY: gen-docs
+gen-docs:
+	node ../../../scripts/dbt-pack/index.js dbt-docs --project-dir .

transformations/aws/compliance-premium/README.md

@@ -22,4 +22,7 @@ The pack contains the premium version.
 - **aws_compliance_pci_dss_v3.2.1**: AWS PCI DSS V3.2.1 benchmark.
 - **aws_compliance\_\_foundational_security**: AWS Foundational Security benchmark, available only for Snowflake
 
-The premium version contains all queries.
+The premium version contains all checks.
+
+<!-- AUTO-GENERATED-INCLUDED-CHECKS-START -->
+<!-- AUTO-GENERATED-INCLUDED-CHECKS-END -->