Generate docs sidebar sections automatically (#480)

Add a `config.json` field within each `navigation` entry called
`generateFrom`. This designates a relative path from `docs/pages` from
which to generate entries within the sidebar.

When generating pages, a function called `generateNavPaths` looks for
pages to use as second-level section introductions. As with the
Docusaurus convention, second-level section introductions have the same
name as their parent directory.

Adding a `generateFrom` field is consistent with the Docusaurus
approach to sidebar generation, in which a configuration field indicates
which directory to generate a section from. This gives us control over
the title and icons we use for navigation sections, which aren't
available to fetch from a directory tree alone. It also lets us use the
current, hardcoded `entries` approach for some sections if we need to.

Also un-skips some accidentally skipped tests.

layouts/DocsPage/types.ts

@@ -43,6 +43,7 @@ export interface NavigationCategory {
   icon: IconName;
   title: string;
   entries: NavigationItem[];
+  generateFrom?: string;
 }
 
 interface LinkWithRedirect {

server/config-docs.ts

@@ -8,11 +8,12 @@ import type { Redirect } from "next/dist/lib/load-custom-routes";
 
 import Ajv from "ajv";
 import { validateConfig } from "./config-common";
-import { resolve, join } from "path";
-import { existsSync, readFileSync } from "fs";
+import { dirname, resolve, join } from "path";
+import fs from "fs";
 import { isExternalLink, isHash, splitPath } from "../utils/url";
 import { NavigationCategory, NavigationItem } from "../layouts/DocsPage/types";
 import { loadConfig as loadSiteConfig } from "./config-site";
+import { generateNavPaths } from "./pages-helpers";
 
 const { latest } = loadSiteConfig();
 export interface Config {
@@ -31,8 +32,8 @@ const getConfigPath = (version: string) =>
 export const load = (version: string) => {
   const path = getConfigPath(version);
 
-  if (existsSync(path)) {
-    const content = readFileSync(path, "utf-8");
+  if (fs.existsSync(path)) {
+    const content = fs.readFileSync(path, "utf-8");
 
     return JSON.parse(content) as Config;
   } else {
@@ -61,6 +62,8 @@ const validator = ajv.compile({
         properties: {
           icon: { type: "string" },
           title: { type: "string" },
+          generateFrom: { type: "string" },
+          // Entries must be empty if generateFrom is present.
           entries: {
             type: "array",
             items: {
@@ -228,7 +231,7 @@ const correspondingFileExistsForURL = (
 
   if (
     [docsPagePath, indexPath, introPath].find((p) => {
-      return existsSync(p);
+      return fs.existsSync(p);
     }) == undefined
   ) {
     return false;
@@ -304,8 +307,6 @@ export const normalize = (config: Config, version: string): Config => {
   return config;
 };
 
-/* Load, validate and normalize config. */
-
 export const loadConfig = (version: string) => {
   const config = load(version);
 
@@ -327,5 +328,18 @@ export const loadConfig = (version: string) => {
 
   validateConfig<Config>(validator, config);
 
+  config.navigation.forEach((item, i) => {
+    if (!!item.generateFrom && item.entries.length > 0) {
+      throw "a navigation item cannot contain both generateFrom and entries";
+    }
+
+    if (!!item.generateFrom) {
+      config.navigation[i].entries = generateNavPaths(
+        fs,
+        join("content", version, "docs", "pages", item.generateFrom)
+      );
+    }
+  });
+
   return normalize(config, version);
 };

server/fixtures/result/code-snippet-heredoc.mdx

@@ -67,6 +67,8 @@
 
   </CodeLine>
 
+  <br />
+
   <CommandComment data-type="descr">
     Create role
   </CommandComment>

server/pages-helpers.ts

@@ -4,9 +4,9 @@
 
 import type { MDXPage, MDXPageData, MDXPageFrontmatter } from "./types-unist";
 
-import { resolve } from "path";
 import { readSync } from "to-vfile";
 import matter from "gray-matter";
+import { sep, parse, dirname, resolve, join } from "path";
 
 export const extensions = ["md", "mdx", "ts", "tsx", "js", "jsx"];
 
@@ -61,3 +61,78 @@ export const getPageInfo = <T = MDXPageFrontmatter>(
 
   return result;
 };
+
+const getEntryForPath = (fs, filePath) => {
+  const txt = fs.readFileSync(filePath, "utf8");
+  const { data } = matter(txt);
+  const slug = filePath.split("docs/pages")[1].replace(".mdx", "/");
+  return {
+    title: data.title,
+    slug: slug,
+  };
+};
+
+export const generateNavPaths = (fs, dirPath) => {
+  const firstLvl = fs.readdirSync(dirPath, "utf8");
+  let result = [];
+  let firstLvlFiles = new Set();
+  let firstLvlDirs = new Set();
+  firstLvl.forEach((p) => {
+    const fullPath = join(dirPath, p);
+    const info = fs.statSync(fullPath);
+    if (info.isDirectory()) {
+      firstLvlDirs.add(fullPath);
+      return;
+    }
+    firstLvlFiles.add(fullPath);
+  });
+  let sectionIntros = new Set();
+  firstLvlDirs.forEach((d: string) => {
+    const { name } = parse(d);
+    const asFile = join(d, name + ".mdx");
+
+    if (!fs.existsSync(asFile)) {
+      throw `subdirectory in generated sidebar section ${d} has no category page ${asFile}`;
+    }
+    sectionIntros.add(asFile);
+    return;
+  });
+
+  // Add files with no corresponding directory to the navigation first. Section
+  // introductions, by convention, have a filename that corresponds to the
+  // subdirectory containing pages in the section, or have the name
+  // "introduction.mdx".
+  firstLvlFiles.forEach((f) => {
+    result.push(getEntryForPath(fs, f));
+  });
+
+  sectionIntros.forEach((si: string) => {
+    const { slug, title } = getEntryForPath(fs, si);
+    const section = {
+      title: title,
+      slug: slug,
+      entries: [],
+    };
+    const sectionDir = dirname(si);
+    const secondLvl = fs.readdirSync(sectionDir, "utf8");
+    secondLvl.forEach((f2) => {
+      const { name } = parse(f2);
+
+      // The directory name is the same as the filename, meaning that we have
+      // already used this as a category page.
+      if (sectionDir.endsWith(name)) {
+        return;
+      }
+
+      const fullPath2 = join(sectionDir, f2);
+      const stat = fs.statSync(fullPath2);
+      if (stat.isDirectory()) {
+        return;
+      }
+
+      section.entries.push(getEntryForPath(fs, fullPath2));
+    });
+    result.push(section);
+  });
+  return result;
+};

uvu-tests/config-docs.test.ts

@@ -2,9 +2,10 @@ import { Redirect } from "next/dist/lib/load-custom-routes";
 import { suite } from "uvu";
 import * as assert from "uvu/assert";
 import { Config, checkURLsForCorrespondingFiles } from "../server/config-docs";
+import { generateNavPaths } from "../server/pages-helpers";
 import { randomUUID } from "crypto";
 import { join } from "path";
-import { opendirSync } from "fs";
+import { Volume, createFsFromVolume } from "memfs";
 
 const Suite = suite("server/config-docs");
 
@@ -112,4 +113,90 @@ Suite("Ensures that URLs correspond to docs pages", () => {
   assert.equal(actual, expected);
 });
 
+Suite("generateNavPaths generates a sidebar from a file tree", () => {
+  const files = {
+    "/docs/pages/database-access/introduction.mdx": `---
+title: Protect Databases with Teleport
+---`,
+    "/docs/pages/database-access/guides/guides.mdx": `---
+title: Database Access Guides
+---`,
+    "/docs/pages/database-access/guides/postgres.mdx": `---
+title: Postgres Guide
+---`,
+    "/docs/pages/database-access/guides/mysql.mdx": `---
+title: MySQL Guide
+---`,
+    "/docs/pages/database-access/rbac/rbac.mdx": `---
+title: Database Access RBAC
+---`,
+    "/docs/pages/database-access/rbac/get-started.mdx": `---
+title: Get Started with DB RBAC
+---`,
+    "/docs/pages/database-access/rbac/reference.mdx": `---
+title: Database RBAC Reference
+---`,
+  };
+
+  const expected = [
+    {
+      title: "Protect Databases with Teleport",
+      slug: "/database-access/introduction/",
+    },
+    {
+      title: "Database Access Guides",
+      slug: "/database-access/guides/guides/",
+      entries: [
+        {
+          title: "MySQL Guide",
+          slug: "/database-access/guides/mysql/",
+        },
+        {
+          title: "Postgres Guide",
+          slug: "/database-access/guides/postgres/",
+        },
+      ],
+    },
+    {
+      title: "Database Access RBAC",
+      slug: "/database-access/rbac/rbac/",
+      entries: [
+        {
+          title: "Get Started with DB RBAC",
+          slug: "/database-access/rbac/get-started/",
+        },
+        {
+          title: "Database RBAC Reference",
+          slug: "/database-access/rbac/reference/",
+        },
+      ],
+    },
+  ];
+
+  const vol = Volume.fromJSON(files);
+  const fs = createFsFromVolume(vol);
+  const actual = generateNavPaths(fs, "/docs/pages/database-access");
+  assert.equal(actual, expected);
+});
+
+Suite(
+  "generateNavPaths throws if there is no category page in a subdirectory",
+  () => {
+    const files = {
+      "/docs/pages/database-access/guides/postgres.mdx": `---
+title: Postgres Guide
+---`,
+      "/docs/pages/database-access/guides/mysql.mdx": `---
+title: MySQL Guide
+---`,
+    };
+
+    const vol = Volume.fromJSON(files);
+    const fs = createFsFromVolume(vol);
+    assert.throws(() => {
+      generateNavPaths(fs, "/docs/pages/database-access");
+    }, "database-access/guides/guides.mdx");
+  }
+);
+
 Suite.run();

uvu-tests/remark-code-snippet.test.ts

@@ -243,7 +243,7 @@ Suite("Variables in multiline command support", () => {
   assert.equal(result, expected);
 });
 
-Suite.only("Includes empty lines in example command output", () => {
+Suite("Includes empty lines in example command output", () => {
   const value = readFileSync(
     resolve("server/fixtures/code-snippet-empty-line.mdx"),
     "utf-8"

uvu-tests/remark-includes.test.ts

@@ -590,30 +590,27 @@ boundary" section.
   }
 );
 
-Suite.only(
-  "Interprets anchor-only links correctly when loading partials",
-  () => {
-    const actual = transformer({
-      value: `Here is the outer page.
+Suite("Interprets anchor-only links correctly when loading partials", () => {
+  const actual = transformer({
+    value: `Here is the outer page.
 
 (!anchor-links.mdx!)
 
 `,
-      path: "server/fixtures/mypage.mdx",
-    }).toString();
+    path: "server/fixtures/mypage.mdx",
+  }).toString();
 
-    assert.equal(
-      actual,
-      `Here is the outer page.
+  assert.equal(
+    actual,
+    `Here is the outer page.
 
 This is a [link to an anchor](#this-is-a-section).
 
 ## This is a section.
 
 This is content within the section.
 `
-    );
-  }
-);
+  );
+});
 
 Suite.run();