4th commit

Taskfile

@@ -3,11 +3,7 @@ PATH="$PWD/node_modules/.bin":$PATH
 set -e
 
 remark_run() {
-    node -r ts-node/register/transpile-only node_modules/remark-cli/cli.js readme.md "$@"
-}
-
-remark_output() {
-    remark_run --output
+    node -r ts-node/register/transpile-only node_modules/remark-cli/cli.js README.md "$@"
 }
 
 "$@"

package.json

@@ -1,12 +1,14 @@
 {
   "name": "tailwind-components",
   "scripts": {
-    "test:r": "mocha -r ts-node/register/transpile-only src/**/*.spec.ts",
+    "test:r": "mocha -r ts-node/register/transpile-only --no-timeouts src/**/*.spec.ts",
     "test:w": "mocha -r ts-node/register/transpile-only --no-timeouts --watch-files src/**/*.ts --watch src/**/*.spec.ts",
     "test:d": "node --inspect -r ts-node/register/transpile-only node_modules/mocha/bin/_mocha --no-timeouts --watch-files src/**/*.ts --watch src/**/*.spec.ts",
     "test:brk": "node --inspect-brk -r ts-node/register/transpile-only node_modules/mocha/bin/_mocha --no-timeouts src/**/*.spec.ts",
     "remark": "sh Taskfile remark_run",
-    "remark:update": "sh Taskfile remark_run --output"
+    "remark:update": "sh Taskfile remark_run --output",
+    "program:run": "ts-node src/main.ts",
+    "generate": "npm run program:run && npm run remark:update"
   },
   "config": {
     "commitizen": {
@@ -19,14 +21,22 @@
     }
   },
   "devDependencies": {
+    "@types/lodash": "^4.14.162",
     "@types/mocha": "^8.0.3",
     "@types/node": "^14.11.10",
-    "graphql-scraper": "^1.2.1",
+    "@types/pluralize": "^0.0.29",
+    "@types/puppeteer": "^3.0.2",
+    "@types/string-similarity": "^3.0.0",
+    "earljs": "^0.0.13",
+    "lodash": "^4.17.20",
     "mocha": "^8.1.3",
+    "pluralize": "^8.0.0",
     "prettier": "^2.1.2",
+    "puppeteer": "^5.3.1",
     "remark": "^13.0.0",
     "remark-cli": "^9.0.0",
     "remark-toc": "^7.0.0",
+    "string-similarity": "^4.0.2",
     "ts-node": "^9.0.0",
     "typescript": "^4.0.3"
   }

src/generate.spec.ts

@@ -0,0 +1,56 @@
+import { generate } from './generate';
+import { expect } from 'earljs';
+
+it('get category exact match', async () => {
+    const items = [
+        {
+            name: 'Card',
+            link: 'https://tailwindcomponents.com/component/card-1',
+        },
+    ];
+    const result = await generate({ items });
+    expect(result).toEqual(
+        expect.stringMatching('\\* Card - https://tailwindcomponents.com/component/card-1'),
+    );
+});
+
+it('get category from word', async () => {
+    const items = [
+        {
+            name: 'curvy card',
+            link: 'https://example.com',
+        },
+    ];
+    const result = await generate({ items });
+    expect(result).toEqual(expect.stringMatching('\\* curvy card - https://example.com'));
+});
+
+it('get category exact pluralize', async () => {
+    const items = [
+        {
+            name: 'Cards',
+            link: 'https://example.com',
+        },
+    ];
+    const result = await generate({ items });
+    expect(result).toEqual(expect.stringMatching('## Card\n\\* Cards - https://example.com'));
+});
+
+it('sorted alphabetically', async () => {
+    const items = [
+        {
+            name: 'Cards',
+            link: 'https://example.com',
+        },
+        {
+            name: 'Button',
+            link: 'https://example.com',
+        },
+    ];
+    const result = await generate({ items });
+    expect(result).toEqual(
+        expect.stringMatching(
+            '## Button\n\\* Button - https://example.com\n## Card\n\\* Cards - https://example.com',
+        ),
+    );
+});

src/generate.ts

@@ -0,0 +1,70 @@
+import { CompomentLink } from './types';
+import stringSimilarity from 'string-similarity';
+import _ from 'lodash';
+import { plural } from 'pluralize';
+
+const categoryList = new Map([
+    ['Alert', ['toast']],
+    ['Avatar', []],
+    ['Breadcrumb', []],
+    ['Button', []],
+    ['Badge', []],
+    ['Card', []],
+    ['Dropdown', []],
+    ['Form', []],
+    ['Hero', []],
+    ['Layout', []],
+    ['Modal', []],
+    ['Navigation', ['navbar']],
+    ['Pagination', []],
+    ['Sidebar', ['side panel']],
+    ['Step', []],
+    ['Section', []],
+    ['Switch', ['toggle']],
+    ['Table', []],
+    ['Tab', []],
+    ['Other', []],
+]);
+
+type GenerateArgs = {
+    items: CompomentLink[];
+};
+
+export async function generate({ items }: GenerateArgs) {
+    const byCategory = _(items)
+        .groupBy((item) => getCategory(item.category || item.name))
+        .toPairs()
+        .sortBy(0)
+        .fromPairs()
+        .value();
+    const content: string[] = [];
+    for (const [category, items] of Object.entries(byCategory)) {
+        content.push(`## ${category}`);
+        content.push(items.map(createLink).join('\n'));
+    }
+    return `# Tailwind Components\n## Table of Contents\n${content.join('\n')}`;
+}
+
+function getCategory(name: string) {
+    for (const [category, keywords] of categoryList.entries()) {
+        const categoryLower = category.toLowerCase();
+        const categoryPlural = plural(categoryLower);
+        if (categoryLower === name.toLowerCase()) {
+            return category;
+        }
+        const words = name.split(/\s+/);
+        if (
+            words.find((word) => {
+                const wordLower = word.toLowerCase();
+                return wordLower === categoryLower || categoryPlural === plural(wordLower);
+            })
+        ) {
+            return category;
+        }
+    }
+    return 'Other';
+}
+
+function createLink(item: CompomentLink) {
+    return `* ${item.name} - ${item.link}`;
+}

src/main.ts

@@ -0,0 +1,20 @@
+import puppeteer from 'puppeteer';
+import { getScrapers } from './scrapers';
+import { CompomentLink } from './types';
+import { generate } from './generate';
+import { promises as fs } from 'fs';
+
+async function main() {
+    const browser = await puppeteer.launch({ headless: false, slowMo: 0 });
+    const [page] = await browser.pages();
+    const scrapers = await getScrapers();
+    const items: CompomentLink[] = [];
+    for await (const scraper of scrapers) {
+        items.push(...(await scraper({ page })));
+    }
+    await browser.close();
+    const content = await generate({ items });
+    await fs.writeFile('README.md', content);
+}
+
+main();

src/scrapers/index.ts

@@ -0,0 +1,4 @@
+export async function getScrapers() {
+    const result = Promise.all([import('./tailwindcomponents').then((m) => m.default)]);
+    return result;
+}

src/scrapers/tailwindcomponents.ts

@@ -0,0 +1,31 @@
+import { CompomentLink, ScraperArgs } from '../types';
+
+export default async function tailwindcomponents({ page }: ScraperArgs) {
+    await page.goto('https://tailwindcomponents.com/components');
+    const itemSelector = '#app > .min-h-screen > main > div a.flex.flex-col';
+    const nextPageSelector = 'a[title="Next »"]';
+    const items: CompomentLink[] = await page.$$eval(itemSelector, (elements) => {
+        return elements.map((a) => ({
+            name: a.querySelector('h4')!.textContent as string,
+            link: `https://tailwindcomponents.com${a.getAttribute('href')}`,
+        }));
+    });
+    let nextPage: string | null = '';
+
+    do {
+        if (!(await page.$(nextPageSelector))) {
+            break;
+        }
+        nextPage = await page.$eval(nextPageSelector, (a) => a.getAttribute('href'));
+        await page.goto(nextPage!);
+        const nextItems = await page.$$eval(itemSelector, (elements) => {
+            return elements.map((a) => ({
+                name: a.querySelector('h4')!.textContent as string,
+                link: `https://tailwindcomponents.com${a.getAttribute('href')}`,
+            }));
+        });
+        items.push(...nextItems);
+    } while (nextPage);
+
+    return items;
+}

src/types.ts

@@ -0,0 +1,11 @@
+import { Page } from 'puppeteer';
+
+export type ScraperArgs = {
+    page: Page;
+};
+
+export type CompomentLink = {
+    category?: string;
+    name: string;
+    link: string;
+};

tsconfig.json

@@ -23,7 +23,7 @@
     "declaration": false,
     "declarationMap": false,
     "skipLibCheck": true,
-    "lib": ["esnext"]
+    "lib": ["dom", "esnext"]
   },
   "include": ["src/**/*.ts"]
 }