diff --git a/e2e/cypress/integration/readme.spec.js b/e2e/cypress/integration/readme.spec.js index b1e4e33b13..7e21a90248 100644 --- a/e2e/cypress/integration/readme.spec.js +++ b/e2e/cypress/integration/readme.spec.js @@ -157,4 +157,63 @@ describe('Readme style', () => { cy.request($a[0].href).its('status').should('equal', 200) }) }) + + it('renders :emoji: in readme and project description', () => { + const username = getFakeUsername() + const email = faker.unique(faker.internet.email) + const password = '123456' + + const repoName = 'ogx360' + const syncedRepoUrl = 'https://github.com/kitspace-test-repos/ogx360' + + cy.createUser(username, email, password) + cy.visit('/') + cy.get('[data-cy=user-menu]') + + cy.forceVisit('/projects/new') + + // Migrate the repo + cy.get('[data-cy=sync-field]').type(syncedRepoUrl) + cy.get('button').contains('Sync').click() + + // Wait for redirection for project page + cy.url({ timeout: 60_000 }).should('contain', `${username}/${repoName}`) + // Wait for the repo to finish processing, by checking the visibility of info-bar. + cy.get('[data-cy=info-bar]', { timeout: 60_000 }).should('be.visible') + + // the project description isAdd modern xinput USB support to your Original 📺 🎮 + cy.get('[data-cy=project-description]').should('contain.text', '📺 🎮') + cy.get('[data-cy=readme]').should('contain.text', '🤓') + }) + + it('preserves URLs for GitHub Actions badges', () => { + const username = getFakeUsername() + const email = faker.unique(faker.internet.email) + const password = '123456' + + const repoName = 'ogx360' + const syncedRepoUrl = 'https://github.com/kitspace-test-repos/ogx360' + + cy.createUser(username, email, password) + cy.visit('/') + cy.get('[data-cy=user-menu]') + + cy.forceVisit('/projects/new') + + // Migrate the repo + cy.get('[data-cy=sync-field]').type(syncedRepoUrl) + cy.get('button').contains('Sync').click() + + // Wait for redirection for project page + cy.url({ timeout: 60_000 }).should('contain', `${username}/${repoName}`) + // Wait for the repo to finish processing, by checking the visibility of info-bar. + cy.get('[data-cy=info-bar]', { timeout: 60_000 }).should('be.visible') + + // The first image in the readme is the GitHub Actions badge. + cy.get('[data-cy=readme] img') + .first() + .each($img => { + cy.request($img[0].src).its('status').should('equal', 200) + }) + }) }) diff --git a/frontend/src/components/Board/Readme.jsx b/frontend/src/components/Board/Readme.jsx index ee8f0f1b52..5ebebeb1cf 100644 --- a/frontend/src/components/Board/Readme.jsx +++ b/frontend/src/components/Board/Readme.jsx @@ -38,6 +38,31 @@ const Readme = ({ renderedReadme }) => ( #readme input[type='checkbox'] { margin-right: 5px; } + + #readme table { + border-collapse: collapse; + border-spacing: 0; + display: block; + max-width: 100%; + overflow: auto; + width: 100%; + width: max-content; + } + + #readme table tr { + background-color: #ffffff; + border-top: 1px solid hsla(210, 18%, 91%, 1); + } + + #readme table th, + #readme table td { + padding: 6px 13px; + border: 1px solid #d0d7de; + } + + #readme table tr:nth-child(2n) { + background-color: #f6f8fa; + } `} ) diff --git a/processor/package.json b/processor/package.json index 3ca7b3f211..071b00000e 100644 --- a/processor/package.json +++ b/processor/package.json @@ -24,14 +24,12 @@ "bullmq": "^1.91.1", "cheerio": "^1.0.0-rc.10", "chokidar": "^3.4.3", - "escape-html": "^1.0.3", "express": "^4.17.2", "express-fileupload": "^1.2.1", "globule": "^1.0.0", "hast-util-has-property": "^2.0.0", "js-yaml": "^3.3.1", "jszip": "^3.7.0", - "linkify-it": "^4.0.1", "lodash.debounce": "^4.0.8", "loglevel": "^1.7.1", "meilisearch": "^0.25.1", @@ -46,6 +44,7 @@ "rehype-shift-heading": "^1.0.2", "rehype-slug": "^5.0.1", "rehype-stringify": "^9.0.3", + "remark-emoji": "^3.0.2", "remark-gfm": "^3.0.1", "remark-parse": "^10.0.1", "remark-rehype": "^10.1.0", diff --git a/processor/src/giteatDB.ts b/processor/src/giteatDB.ts index 73a1b4eafa..1cecd806ba 100644 --- a/processor/src/giteatDB.ts +++ b/processor/src/giteatDB.ts @@ -76,7 +76,8 @@ export const giteaDB: GiteaDB = { row => row.repo_id === repoId && row.type === TaskType.Migration && - row.status === MigrationStatus.Done, + row.status === MigrationStatus.Done && + row.default_branch !== '' ) }, diff --git a/processor/src/tasks/processReadme/index.ts b/processor/src/tasks/processReadme/index.ts index bc8f04720b..0127422bd8 100644 --- a/processor/src/tasks/processReadme/index.ts +++ b/processor/src/tasks/processReadme/index.ts @@ -1,6 +1,7 @@ +import path from 'node:path' + import { unified } from 'unified' import globule from 'globule' -import path from 'node:path' import rehypeAutolinkHeadings from 'rehype-autolink-headings' import rehypeHighlight from 'rehype-highlight' import rehypeRaw from 'rehype-raw' @@ -8,6 +9,7 @@ import rehypeSanitize from 'rehype-sanitize' import rehypeShiftHeading from 'rehype-shift-heading' import rehypeSlug from 'rehype-slug' import rehypeStringify from 'rehype-stringify' +import remarkEmoji from 'remark-emoji' import remarkGfm from 'remark-gfm' import remarkParse from 'remark-parse' import remarkRehype from 'remark-rehype' @@ -101,6 +103,7 @@ async function renderMarkdown( ) { const Remarker = unified() .use(remarkParse) + .use(remarkEmoji) .use(remarkGfm) .use(remarkRehype, { allowDangerousHtml: true }) .use(rehypeRaw) diff --git a/processor/src/tasks/writeKitspaceYaml.ts b/processor/src/tasks/writeKitspaceYaml.ts index 3229bc4c54..f613b5c4d8 100644 --- a/processor/src/tasks/writeKitspaceYaml.ts +++ b/processor/src/tasks/writeKitspaceYaml.ts @@ -1,59 +1,57 @@ -import escape from 'escape-html' -import LinkifyIt from 'linkify-it' import path from 'node:path' +import { unified } from 'unified' +import rehypeSanitize from 'rehype-sanitize' +import rehypeStringify from 'rehype-stringify' +import remarkEmoji from 'remark-emoji' +import remarkGfm from 'remark-gfm' +import remarkParse from 'remark-parse' +import remarkRehype from 'remark-rehype' + import { JobData } from '../jobData.js' import * as utils from '../utils.js' -const linkify = new LinkifyIt() - -export default function writeKitspaceYaml( +export default async function writeKitspaceYaml( job, { kitspaceYaml, outputDir }: Partial, ) { const kitspaceYamlJson = path.join(outputDir, 'kitspace-yaml.json') job.updateProgress({ status: 'in_progress', file: kitspaceYamlJson }) - const KitspaceYamlJsonLinkified = linkifyKitspaceYaml(kitspaceYaml) + + const rendered = await renderKitspaceYamlSummaries(kitspaceYaml) return utils - .writeFile(kitspaceYamlJson, JSON.stringify(KitspaceYamlJsonLinkified, null, 2)) + .writeFile(kitspaceYamlJson, JSON.stringify(rendered, null, 2)) .then(() => job.updateProgress({ status: 'done', file: kitspaceYamlJson })) .catch(error => job.updateProgress({ status: 'failed', file: kitspaceYamlJson, error }), ) } -function linkifyKitspaceYaml(kitspaceYaml) { +async function renderKitspaceYamlSummaries(kitspaceYaml) { if (kitspaceYaml.multi) { - const linkifiedKitspaceYaml = kitspaceYaml - Object.keys(kitspaceYaml.multi).forEach(subProject => { - linkifiedKitspaceYaml.multi[subProject] = linkifyProjectSummary(kitspaceYaml.multi[subProject]) - }) - return linkifiedKitspaceYaml - } - - return linkifyProjectSummary(kitspaceYaml) -} - -function linkifyProjectSummary(kitspaceYaml) { - let escapedSummary = escape(kitspaceYaml.summary || '') - const matches = linkify.match(escapedSummary) - - if (matches) { - for (const match of matches) { - // Use https by default - const url = new URL(match.url) - if (!match.schema) { - url.protocol = 'https:' + const rendered = { multi: {} } + for (const key of Object.keys(kitspaceYaml.multi)) { + const subProject = kitspaceYaml.multi[key] + rendered.multi[key] = { + ...subProject, + summary: await renderSummary(subProject.summary), } - - escapedSummary = escapedSummary.replace( - match.raw, - `${match.text}` - ) } + return rendered } + return { ...kitspaceYaml, summary: await renderSummary(kitspaceYaml.summary) } +} - kitspaceYaml.summary = escapedSummary - return kitspaceYaml +const Remarker = unified() + .use(remarkParse) + .use(remarkEmoji) + .use(remarkGfm) + .use(remarkRehype) + .use(rehypeSanitize) + .use(rehypeStringify) + +async function renderSummary(summary = ''): Promise { + const rendered = await Remarker.process(summary) + return String(rendered) } diff --git a/processor/src/utils.ts b/processor/src/utils.ts index 1799a08025..4b25274756 100644 --- a/processor/src/utils.ts +++ b/processor/src/utils.ts @@ -80,9 +80,14 @@ export function toGitHubRawUrl(url: string) { if (parsedUrl.hostname === 'github.com') { parsedUrl.hostname = 'raw.githubusercontent.com' const urlPath = parsedUrl.pathname.split('/') - // Remove `/blob/` or '/raw/' from the path. - parsedUrl.pathname = urlPath.slice(0, 3).concat(urlPath.slice(4)).join('/') - url = parsedUrl.toString() + // Avoid modifying github actions status badges. + const isWorkflowPath = ['workflows', 'actions'].includes(urlPath?.[3]) + + if (!isWorkflowPath) { + // Remove `/blob/` or '/raw/' from the path. + parsedUrl.pathname = urlPath.slice(0, 3).concat(urlPath.slice(4)).join('/') + url = parsedUrl.toString() + } } return url } diff --git a/processor/src/watcher.ts b/processor/src/watcher.ts index e9202485ef..2eacdb8ed0 100644 --- a/processor/src/watcher.ts +++ b/processor/src/watcher.ts @@ -41,13 +41,8 @@ export function watch(repoDir, { giteaDB }: WatchOptions) { dirWatchers[gitDir].queuing = false return } - defaultBranch = giteaRepo.default_branch - originalUrl = giteaRepo.original_url - repoDescription = giteaRepo.description + giteaId = giteaRepo.id - // use case-correct names from the DB - ownerName = giteaRepo.owner_name - repoName = giteaRepo.name if (giteaRepo.is_empty) { await giteaDB.waitForNonEmpty(giteaId) @@ -56,6 +51,16 @@ export function watch(repoDir, { giteaDB }: WatchOptions) { if (giteaRepo.is_mirror) { await giteaDB.waitForRepoMigration(giteaId) } + + // Get the repo info again after the migration is done. + // Some fields, (e.g., default_branch) only gets populated after migration + const finalGiteaRepoData = await giteaDB.getRepoInfo(ownerName, repoName) + originalUrl = finalGiteaRepoData.original_url + repoDescription = finalGiteaRepoData.description + defaultBranch = finalGiteaRepoData.default_branch + // use case-correct names from the DB + ownerName = finalGiteaRepoData.owner_name + repoName = finalGiteaRepoData.name } await addProjectToQueues({ diff --git a/processor/yarn.lock b/processor/yarn.lock index 9163f808c3..3c7e814cf8 100644 --- a/processor/yarn.lock +++ b/processor/yarn.lock @@ -1125,6 +1125,11 @@ emoji-regex@^8.0.0: resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== +emoticon@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/emoticon/-/emoticon-4.0.1.tgz#2d2bbbf231ce3a5909e185bbb64a9da703a1e749" + integrity sha512-dqx7eA9YaqyvYtUhJwT4rC1HIp82j5ybS1/vQ42ur+jBe17dJMwZE4+gvL1XadSFfxaPFFGt3Xsw+Y8akThDlw== + encodeurl@~1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" @@ -2314,13 +2319,6 @@ lie@~3.3.0: dependencies: immediate "~3.0.5" -linkify-it@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-4.0.1.tgz#01f1d5e508190d06669982ba31a7d9f56a5751ec" - integrity sha512-C7bfi1UZmoj8+PQx22XyeXCuBlokoyWQL5pWSP+EI6nzRylyThouddufc2c1NDIcP9k5agmN9fLpA7VNJfIiqw== - dependencies: - uc.micro "^1.0.1" - locate-path@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/locate-path/-/locate-path-2.0.0.tgz#2b568b265eec944c6d9c0de9c3dbbbca0354cd8e" @@ -3036,6 +3034,13 @@ node-cleanup@^2.1.2: resolved "https://registry.yarnpkg.com/node-cleanup/-/node-cleanup-2.1.2.tgz#7ac19abd297e09a7f72a71545d951b517e4dde2c" integrity sha1-esGavSl+Caf3KnFUXZUbUX5N3iw= +node-emoji@^1.11.0: + version "1.11.0" + resolved "https://registry.yarnpkg.com/node-emoji/-/node-emoji-1.11.0.tgz#69a0150e6946e2f115e9d7ea4df7971e2628301c" + integrity sha512-wo2DpQkQp7Sjm2A0cq+sN7EHKO6Sl0ctXeBdFZrL9T9+UywORbufTcTZxom8YqpLQt/FqNMUkOpkZrJVYSKD3A== + dependencies: + lodash "^4.17.21" + node-fetch@2.6.7: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" @@ -3521,6 +3526,15 @@ rehype-stringify@^9.0.3: hast-util-to-html "^8.0.0" unified "^10.0.0" +remark-emoji@^3.0.2: + version "3.0.2" + resolved "https://registry.yarnpkg.com/remark-emoji/-/remark-emoji-3.0.2.tgz#786e88af1ecae682d74d7e1219989f34708205da" + integrity sha512-hEgxEv2sBtvhT3tNG/tQeeFY3EbslftaOoG14dDZndLo25fWJ6Fbg4ukFbIotOWWrfXyASjXjyHT+6n366k3mg== + dependencies: + emoticon "^4.0.0" + node-emoji "^1.11.0" + unist-util-visit "^4.1.0" + remark-gfm@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/remark-gfm/-/remark-gfm-3.0.1.tgz#0b180f095e3036545e9dddac0e8df3fa5cfee54f" @@ -4046,11 +4060,6 @@ typescript@^4.5.5: resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.5.5.tgz#d8c953832d28924a9e3d37c73d729c846c5896f3" integrity sha512-TCTIul70LyWe6IJWT8QSYeA54WQe8EjQFU4wY52Fasj5UKx88LNYKCgBEHcOMOrFF1rKGbD8v/xcNWVUq9SymA== -uc.micro@^1.0.1: - version "1.0.6" - resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.6.tgz#9c411a802a409a91fc6cf74081baba34b24499ac" - integrity sha512-8Y75pvTYkLJW2hWQHXxoqRgV7qb9B+9vFEtidML+7koHUFapnVJAZ6cKs+Qjz5Aw3aZWHMC6u0wJE3At+nSGwA== - unbox-primitive@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/unbox-primitive/-/unbox-primitive-1.0.1.tgz#085e215625ec3162574dc8859abee78a59b14471" @@ -4138,7 +4147,7 @@ unist-util-visit@^3.0.0: unist-util-is "^5.0.0" unist-util-visit-parents "^4.0.0" -unist-util-visit@^4.0.0, unist-util-visit@^4.1.1: +unist-util-visit@^4.0.0, unist-util-visit@^4.1.0, unist-util-visit@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/unist-util-visit/-/unist-util-visit-4.1.1.tgz#1c4842d70bd3df6cc545276f5164f933390a9aad" integrity sha512-n9KN3WV9k4h1DxYR1LoajgN93wpEi/7ZplVe02IoB4gH5ctI1AaF2670BLHQYbwj+pY83gFtyeySFiyMHJklrg==