Merge pull request #430 from kitspace/readme-processing-2

More readme fixes

e2e/cypress/integration/readme.spec.js

@@ -157,4 +157,63 @@ describe('Readme style', () => {
         cy.request($a[0].href).its('status').should('equal', 200)
       })
   })
+
+  it('renders :emoji: in readme and project description', () => {
+    const username = getFakeUsername()
+    const email = faker.unique(faker.internet.email)
+    const password = '123456'
+
+    const repoName = 'ogx360'
+    const syncedRepoUrl = 'https://github.com/kitspace-test-repos/ogx360'
+
+    cy.createUser(username, email, password)
+    cy.visit('/')
+    cy.get('[data-cy=user-menu]')
+
+    cy.forceVisit('/projects/new')
+
+    // Migrate the repo
+    cy.get('[data-cy=sync-field]').type(syncedRepoUrl)
+    cy.get('button').contains('Sync').click()
+
+    // Wait for redirection for project page
+    cy.url({ timeout: 60_000 }).should('contain', `${username}/${repoName}`)
+    // Wait for the repo to finish processing, by checking the visibility of info-bar.
+    cy.get('[data-cy=info-bar]', { timeout: 60_000 }).should('be.visible')
+
+    // the project description isAdd modern xinput USB support to your Original 📺 🎮
+    cy.get('[data-cy=project-description]').should('contain.text', '📺 🎮')
+    cy.get('[data-cy=readme]').should('contain.text', '🤓')
+  })
+
+  it('preserves URLs for GitHub Actions badges', () => {
+    const username = getFakeUsername()
+    const email = faker.unique(faker.internet.email)
+    const password = '123456'
+
+    const repoName = 'ogx360'
+    const syncedRepoUrl = 'https://github.com/kitspace-test-repos/ogx360'
+
+    cy.createUser(username, email, password)
+    cy.visit('/')
+    cy.get('[data-cy=user-menu]')
+
+    cy.forceVisit('/projects/new')
+
+    // Migrate the repo
+    cy.get('[data-cy=sync-field]').type(syncedRepoUrl)
+    cy.get('button').contains('Sync').click()
+
+    // Wait for redirection for project page
+    cy.url({ timeout: 60_000 }).should('contain', `${username}/${repoName}`)
+    // Wait for the repo to finish processing, by checking the visibility of info-bar.
+    cy.get('[data-cy=info-bar]', { timeout: 60_000 }).should('be.visible')
+
+    // The first image in the readme is the GitHub Actions badge.
+    cy.get('[data-cy=readme] img')
+      .first()
+      .each($img => {
+        cy.request($img[0].src).its('status').should('equal', 200)
+      })
+  })
 })

frontend/src/components/Board/Readme.jsx

@@ -38,6 +38,31 @@ const Readme = ({ renderedReadme }) => (
       #readme input[type='checkbox'] {
         margin-right: 5px;
       }
+
+      #readme table {
+        border-collapse: collapse;
+        border-spacing: 0;
+        display: block;
+        max-width: 100%;
+        overflow: auto;
+        width: 100%;
+        width: max-content;
+      }
+
+      #readme table tr {
+        background-color: #ffffff;
+        border-top: 1px solid hsla(210, 18%, 91%, 1);
+      }
+
+      #readme table th,
+      #readme table td {
+        padding: 6px 13px;
+        border: 1px solid #d0d7de;
+      }
+
+      #readme table tr:nth-child(2n) {
+        background-color: #f6f8fa;
+      }
     `}</style>
   </div>
 )

processor/package.json

@@ -24,14 +24,12 @@
     "bullmq": "^1.91.1",
     "cheerio": "^1.0.0-rc.10",
     "chokidar": "^3.4.3",
-    "escape-html": "^1.0.3",
     "express": "^4.17.2",
     "express-fileupload": "^1.2.1",
     "globule": "^1.0.0",
     "hast-util-has-property": "^2.0.0",
     "js-yaml": "^3.3.1",
     "jszip": "^3.7.0",
-    "linkify-it": "^4.0.1",
     "lodash.debounce": "^4.0.8",
     "loglevel": "^1.7.1",
     "meilisearch": "^0.25.1",
@@ -46,6 +44,7 @@
     "rehype-shift-heading": "^1.0.2",
     "rehype-slug": "^5.0.1",
     "rehype-stringify": "^9.0.3",
+    "remark-emoji": "^3.0.2",
     "remark-gfm": "^3.0.1",
     "remark-parse": "^10.0.1",
     "remark-rehype": "^10.1.0",

processor/src/giteatDB.ts

@@ -76,7 +76,8 @@ export const giteaDB: GiteaDB = {
       row =>
         row.repo_id === repoId &&
         row.type === TaskType.Migration &&
-        row.status === MigrationStatus.Done,
+        row.status === MigrationStatus.Done &&
+        row.default_branch !== ''
     )
   },
 

processor/src/tasks/processReadme/index.ts

@@ -1,13 +1,15 @@
+import path from 'node:path'
+
 import { unified } from 'unified'
 import globule from 'globule'
-import path from 'node:path'
 import rehypeAutolinkHeadings from 'rehype-autolink-headings'
 import rehypeHighlight from 'rehype-highlight'
 import rehypeRaw from 'rehype-raw'
 import rehypeSanitize from 'rehype-sanitize'
 import rehypeShiftHeading from 'rehype-shift-heading'
 import rehypeSlug from 'rehype-slug'
 import rehypeStringify from 'rehype-stringify'
+import remarkEmoji from 'remark-emoji'
 import remarkGfm from 'remark-gfm'
 import remarkParse from 'remark-parse'
 import remarkRehype from 'remark-rehype'
@@ -101,6 +103,7 @@ async function renderMarkdown(
 ) {
   const Remarker = unified()
     .use(remarkParse)
+    .use(remarkEmoji)
     .use(remarkGfm)
     .use(remarkRehype, { allowDangerousHtml: true })
     .use(rehypeRaw)

processor/src/tasks/writeKitspaceYaml.ts

@@ -1,59 +1,57 @@
-import escape from 'escape-html'
-import LinkifyIt from 'linkify-it'
 import path from 'node:path'
 
+import { unified } from 'unified'
+import rehypeSanitize from 'rehype-sanitize'
+import rehypeStringify from 'rehype-stringify'
+import remarkEmoji from 'remark-emoji'
+import remarkGfm from 'remark-gfm'
+import remarkParse from 'remark-parse'
+import remarkRehype from 'remark-rehype'
+
 import { JobData } from '../jobData.js'
 import * as utils from '../utils.js'
 
-const linkify = new LinkifyIt()
-
-export default function writeKitspaceYaml(
+export default async function writeKitspaceYaml(
   job,
   { kitspaceYaml, outputDir }: Partial<JobData>,
 ) {
   const kitspaceYamlJson = path.join(outputDir, 'kitspace-yaml.json')
   job.updateProgress({ status: 'in_progress', file: kitspaceYamlJson })
-  const KitspaceYamlJsonLinkified = linkifyKitspaceYaml(kitspaceYaml)
+
+  const rendered = await renderKitspaceYamlSummaries(kitspaceYaml)
 
   return utils
-    .writeFile(kitspaceYamlJson, JSON.stringify(KitspaceYamlJsonLinkified, null, 2))
+    .writeFile(kitspaceYamlJson, JSON.stringify(rendered, null, 2))
     .then(() => job.updateProgress({ status: 'done', file: kitspaceYamlJson }))
     .catch(error =>
       job.updateProgress({ status: 'failed', file: kitspaceYamlJson, error }),
     )
 }
 
-function linkifyKitspaceYaml(kitspaceYaml) {
+async function renderKitspaceYamlSummaries(kitspaceYaml) {
   if (kitspaceYaml.multi) {
-    const linkifiedKitspaceYaml = kitspaceYaml
-    Object.keys(kitspaceYaml.multi).forEach(subProject => {
-      linkifiedKitspaceYaml.multi[subProject] = linkifyProjectSummary(kitspaceYaml.multi[subProject])
-    })
-    return linkifiedKitspaceYaml
-  }
-
-  return linkifyProjectSummary(kitspaceYaml)
-}
-
-function linkifyProjectSummary(kitspaceYaml) {
-  let escapedSummary = escape(kitspaceYaml.summary || '')
-  const matches = linkify.match(escapedSummary)
-
-  if (matches) {
-    for (const match of matches) {
-      // Use https by default
-      const url = new URL(match.url)
-      if (!match.schema) {
-        url.protocol = 'https:'
+    const rendered = { multi: {} }
+    for (const key of Object.keys(kitspaceYaml.multi)) {
+      const subProject = kitspaceYaml.multi[key]
+      rendered.multi[key] = {
+        ...subProject,
+        summary: await renderSummary(subProject.summary),
       }
-
-      escapedSummary = escapedSummary.replace(
-        match.raw,
-        `<a href="${url.toString()}" rel="noopener noreferrer" target="_blank">${match.text}</a>`
-      )
     }
+    return rendered
   }
+  return { ...kitspaceYaml, summary: await renderSummary(kitspaceYaml.summary) }
+}
 
-  kitspaceYaml.summary = escapedSummary
-  return kitspaceYaml
+const Remarker = unified()
+  .use(remarkParse)
+  .use(remarkEmoji)
+  .use(remarkGfm)
+  .use(remarkRehype)
+  .use(rehypeSanitize)
+  .use(rehypeStringify)
+
+async function renderSummary(summary = ''): Promise<string> {
+  const rendered = await Remarker.process(summary)
+  return String(rendered)
 }

processor/src/utils.ts

@@ -80,9 +80,14 @@ export function toGitHubRawUrl(url: string) {
   if (parsedUrl.hostname === 'github.com') {
     parsedUrl.hostname = 'raw.githubusercontent.com'
     const urlPath = parsedUrl.pathname.split('/')
-    // Remove `/blob/` or '/raw/' from the path.
-    parsedUrl.pathname = urlPath.slice(0, 3).concat(urlPath.slice(4)).join('/')
-    url = parsedUrl.toString()
+    // Avoid modifying github actions status badges.
+    const isWorkflowPath = ['workflows', 'actions'].includes(urlPath?.[3])
+
+    if (!isWorkflowPath) {
+      // Remove `/blob/` or '/raw/' from the path.
+      parsedUrl.pathname = urlPath.slice(0, 3).concat(urlPath.slice(4)).join('/')
+      url = parsedUrl.toString()
+    }
   }
   return url
 }

processor/src/watcher.ts

@@ -41,13 +41,8 @@ export function watch(repoDir, { giteaDB }: WatchOptions) {
             dirWatchers[gitDir].queuing = false
             return
           }
-          defaultBranch = giteaRepo.default_branch
-          originalUrl = giteaRepo.original_url
-          repoDescription = giteaRepo.description
+
           giteaId = giteaRepo.id
-          // use case-correct names from the DB
-          ownerName = giteaRepo.owner_name
-          repoName = giteaRepo.name
 
           if (giteaRepo.is_empty) {
             await giteaDB.waitForNonEmpty(giteaId)
@@ -56,6 +51,16 @@ export function watch(repoDir, { giteaDB }: WatchOptions) {
           if (giteaRepo.is_mirror) {
             await giteaDB.waitForRepoMigration(giteaId)
           }
+
+          // Get the repo info again after the migration is done.
+          // Some fields, (e.g., default_branch) only gets populated after migration
+          const finalGiteaRepoData = await giteaDB.getRepoInfo(ownerName, repoName)
+          originalUrl = finalGiteaRepoData.original_url
+          repoDescription = finalGiteaRepoData.description
+          defaultBranch = finalGiteaRepoData.default_branch
+          // use case-correct names from the DB
+          ownerName = finalGiteaRepoData.owner_name
+          repoName = finalGiteaRepoData.name
         }
 
         await addProjectToQueues({

processor/yarn.lock

@@ -1125,6 +1125,11 @@ emoji-regex@^8.0.0:
   resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37"
   integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==
 
+emoticon@^4.0.0:
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/emoticon/-/emoticon-4.0.1.tgz#2d2bbbf231ce3a5909e185bbb64a9da703a1e749"
+  integrity sha512-dqx7eA9YaqyvYtUhJwT4rC1HIp82j5ybS1/vQ42ur+jBe17dJMwZE4+gvL1XadSFfxaPFFGt3Xsw+Y8akThDlw==
+
 encodeurl@~1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59"
@@ -2314,13 +2319,6 @@ lie@~3.3.0:
   dependencies:
     immediate "~3.0.5"
 
-linkify-it@^4.0.1:
-  version "4.0.1"
-  resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-4.0.1.tgz#01f1d5e508190d06669982ba31a7d9f56a5751ec"
-  integrity sha512-C7bfi1UZmoj8+PQx22XyeXCuBlokoyWQL5pWSP+EI6nzRylyThouddufc2c1NDIcP9k5agmN9fLpA7VNJfIiqw==
-  dependencies:
-    uc.micro "^1.0.1"
-
 locate-path@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-2.0.0.tgz#2b568b265eec944c6d9c0de9c3dbbbca0354cd8e"
@@ -3036,6 +3034,13 @@ node-cleanup@^2.1.2:
   resolved "https://registry.yarnpkg.com/node-cleanup/-/node-cleanup-2.1.2.tgz#7ac19abd297e09a7f72a71545d951b517e4dde2c"
   integrity sha1-esGavSl+Caf3KnFUXZUbUX5N3iw=
 
+node-emoji@^1.11.0:
+  version "1.11.0"
+  resolved "https://registry.yarnpkg.com/node-emoji/-/node-emoji-1.11.0.tgz#69a0150e6946e2f115e9d7ea4df7971e2628301c"
+  integrity sha512-wo2DpQkQp7Sjm2A0cq+sN7EHKO6Sl0ctXeBdFZrL9T9+UywORbufTcTZxom8YqpLQt/FqNMUkOpkZrJVYSKD3A==
+  dependencies:
+    lodash "^4.17.21"
+
 [email protected]:
   version "2.6.7"
   resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
@@ -3521,6 +3526,15 @@ rehype-stringify@^9.0.3:
     hast-util-to-html "^8.0.0"
     unified "^10.0.0"
 
+remark-emoji@^3.0.2:
+  version "3.0.2"
+  resolved "https://registry.yarnpkg.com/remark-emoji/-/remark-emoji-3.0.2.tgz#786e88af1ecae682d74d7e1219989f34708205da"
+  integrity sha512-hEgxEv2sBtvhT3tNG/tQeeFY3EbslftaOoG14dDZndLo25fWJ6Fbg4ukFbIotOWWrfXyASjXjyHT+6n366k3mg==
+  dependencies:
+    emoticon "^4.0.0"
+    node-emoji "^1.11.0"
+    unist-util-visit "^4.1.0"
+
 remark-gfm@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/remark-gfm/-/remark-gfm-3.0.1.tgz#0b180f095e3036545e9dddac0e8df3fa5cfee54f"
@@ -4046,11 +4060,6 @@ typescript@^4.5.5:
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.5.5.tgz#d8c953832d28924a9e3d37c73d729c846c5896f3"
   integrity sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA==
 
-uc.micro@^1.0.1:
-  version "1.0.6"
-  resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.6.tgz#9c411a802a409a91fc6cf74081baba34b24499ac"
-  integrity sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA==
-
 unbox-primitive@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.1.tgz#085e215625ec3162574dc8859abee78a59b14471"
@@ -4138,7 +4147,7 @@ unist-util-visit@^3.0.0:
     unist-util-is "^5.0.0"
     unist-util-visit-parents "^4.0.0"
 
-unist-util-visit@^4.0.0, unist-util-visit@^4.1.1:
+unist-util-visit@^4.0.0, unist-util-visit@^4.1.0, unist-util-visit@^4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/unist-util-visit/-/unist-util-visit-4.1.1.tgz#1c4842d70bd3df6cc545276f5164f933390a9aad"
   integrity sha512-n9KN3WV9k4h1DxYR1LoajgN93wpEi/7ZplVe02IoB4gH5ctI1AaF2670BLHQYbwj+pY83gFtyeySFiyMHJklrg==