From f0f55f4ac9c0b0b2a3cd46fecaa0af0387c1e7aa Mon Sep 17 00:00:00 2001 From: Ovilia Date: Fri, 10 Jan 2025 17:03:10 +0800 Subject: [PATCH 1/3] test: visual test support compare with a remote branch --- test/runTest/cli.js | 20 ++-- test/runTest/client/client.js | 165 +++++++++++++++++++++++++++++---- test/runTest/client/index.html | 36 +++++-- test/runTest/server.js | 49 +++++----- test/runTest/store.js | 56 ++++++++--- test/runTest/util.js | 46 ++++++--- 6 files changed, 287 insertions(+), 85 deletions(-) diff --git a/test/runTest/cli.js b/test/runTest/cli.js index 4ba445f78e..0448f6821b 100644 --- a/test/runTest/cli.js +++ b/test/runTest/cli.js @@ -39,7 +39,9 @@ program .option('--no-headless', 'Not headless') .option('-s, --speed ', 'Playback speed') .option('--expected ', 'Expected version') + .option('--expected-source ', 'Expected source') .option('--actual ', 'Actual version') + .option('--actual-source ', 'Actual source') .option('--renderer ', 'svg/canvas renderer') .option('--use-coarse-pointer ', '"auto" (by default) or "true" or "false"') .option('--threads ', 'How many threads to run concurrently') @@ -78,12 +80,12 @@ function getClientRelativePath(absPath) { return path.join('../', path.relative(__dirname, absPath)); } -function replaceEChartsVersion(interceptedRequest, version) { +function replaceEChartsVersion(interceptedRequest, source, version) { // TODO Extensions and maps if (interceptedRequest.url().endsWith('dist/echarts.js')) { - console.log('Use echarts version: ' + version); + console.log('Use echarts version: ' + source + ' ' + version); interceptedRequest.continue({ - url: `${origin}/test/runTest/${getVersionDir(version)}/${getEChartsTestFileName()}` + url: `${origin}/test/runTest/${getVersionDir(source, version)}/${getEChartsTestFileName()}` }); } else { @@ -167,7 +169,7 @@ async function waitForNetworkIdle(page) { /** * @param {puppeteer.Browser} browser */ -async function runTestPage(browser, testOpt, version, runtimeCode, isExpected) { +async function runTestPage(browser, testOpt, source, version, runtimeCode, isExpected) { const fileUrl = testOpt.fileUrl; const screenshots = []; const logs = []; @@ -175,7 +177,7 @@ async function runTestPage(browser, testOpt, version, runtimeCode, isExpected) { const page = await browser.newPage(); page.setRequestInterception(true); - page.on('request', request => replaceEChartsVersion(request, version)); + page.on('request', request => replaceEChartsVersion(request, source, version)); async function pageScreenshot() { if (!program.save) { @@ -338,12 +340,12 @@ function writePNG(diffPNG, diffPath) { /** * @param {puppeteer.Browser} browser */ -async function runTest(browser, testOpt, runtimeCode, expectedVersion, actualVersion) { +async function runTest(browser, testOpt, runtimeCode, expectedSource, expectedVersion, actualSource, actualVersion) { if (program.save) { testOpt.status === 'running'; - const expectedResult = await runTestPage(browser, testOpt, expectedVersion, runtimeCode, true); - const actualResult = await runTestPage(browser, testOpt, actualVersion, runtimeCode, false); + const expectedResult = await runTestPage(browser, testOpt, expectedSource, expectedVersion, runtimeCode, true); + const actualResult = await runTestPage(browser, testOpt, actualSource, actualVersion, runtimeCode, false); // sortScreenshots(expectedResult.screenshots); // sortScreenshots(actualResult.screenshots); @@ -432,7 +434,7 @@ async function runTests(pendingTests) { async function eachTask(testOpt) { console.log(`Running test: ${testOpt.name}, renderer: ${program.renderer}, useCoarsePointer: ${program.useCoarsePointer}`); try { - await runTest(browser, testOpt, runtimeCode, program.expected, program.actual); + await runTest(browser, testOpt, runtimeCode, program.expectedSource, program.expected, program.actualSource, program.actual); } catch (e) { // Restore status diff --git a/test/runTest/client/client.js b/test/runTest/client/client.js index 0c34e0ff0e..13f1af686a 100644 --- a/test/runTest/client/client.js +++ b/test/runTest/client/client.js @@ -21,6 +21,8 @@ const socket = io('/client'); // const LOCAL_SAVE_KEY = 'visual-regression-testing-config'; +let handlingSourceChange = false; + function getChangedObject(target, source) { let changedObject = {}; Object.keys(source).forEach(key => { @@ -83,9 +85,15 @@ function processTestsData(tests, oldTestsData) { // Keep select status not change. if (oldTestsData && oldTestsData[idx]) { test.selected = oldTestsData[idx].selected; + // Keep source information + test.expectedSource = oldTestsData[idx].expectedSource; + test.actualSource = oldTestsData[idx].actualSource; } else { test.selected = false; + // Initialize source information + test.expectedSource = app.runConfig.expectedSource; + test.actualSource = app.runConfig.actualSource; } }); return tests; @@ -101,6 +109,21 @@ try { } catch (e) {} +function getVersionFromSource(source, versions, nightlyVersions) { + if (source === 'branch') { + return 'master'; + } + else if (source === 'nightly') { + return nightlyVersions.length ? nightlyVersions[0] : null; + } + else if (source === 'local') { + return 'local'; + } + else { + return versions.length ? versions[0] : null; + } +} + const app = new Vue({ el: '#app', data: { @@ -112,9 +135,6 @@ const app = new Vue({ allSelected: false, lastSelectedIndex: -1, - expectedVersionsList: [], - actualVersionsList: [], - loadingVersion: false, showIframeDialog: false, @@ -128,21 +148,26 @@ const app = new Vue({ pageInvisible: false, + versions: [], + nightlyVersions: [], + branchVersions: [], + runConfig: Object.assign({ sortBy: 'name', - - isActualNightly: false, - isExpectedNightly: false, actualVersion: 'local', expectedVersion: null, - + expectedSource: 'release', + actualSource: 'release', renderer: 'canvas', useCoarsePointer: 'auto', threads: 4 }, urlRunConfig) }, - mounted() { + async mounted() { + // Add call to fetch branches + await this.fetchBranchVersions(); + // Sync config from server when first time open // or switching back socket.emit('syncRunConfig', { @@ -151,12 +176,37 @@ const app = new Vue({ forceSet: Object.keys(urlRunConfig).length > 0 }); socket.on('syncRunConfig_return', res => { - this.expectedVersionsList = res.expectedVersionsList; - this.actualVersionsList = res.actualVersionsList; - // Only assign on changed object to avoid unnecessary vue change. - Object.assign(this.runConfig, getChangedObject(this.runConfig, res.runConfig)); + this.versions = res.versions || []; + this.nightlyVersions = res.nightlyVersions || []; + + // Only set versions if they haven't been manually set + handlingSourceChange = true; + this.$nextTick(() => { + if (!this.runConfig.expectedVersion) { + this.runConfig.expectedVersion = getVersionFromSource( + this.runConfig.expectedSource, + this.versions, + this.nightlyVersions + ); + } - updateUrl(); + if (!this.runConfig.actualVersion) { + this.runConfig.actualVersion = getVersionFromSource( + this.runConfig.actualSource, + this.versions, + this.nightlyVersions + ); + } + + // Only apply other config changes from server + const configWithoutVersions = { ...res.runConfig }; + delete configWithoutVersions.expectedVersion; + delete configWithoutVersions.actualVersion; + Object.assign(this.runConfig, getChangedObject(this.runConfig, configWithoutVersions)); + + handlingSourceChange = false; + updateUrl(); + }); }); setTimeout(() => { @@ -254,6 +304,36 @@ const app = new Vue({ }); }, set() {} + }, + + expectedVersionsList() { + switch (this.runConfig.expectedSource) { + case 'release': + return this.versions; + case 'nightly': + return this.nightlyVersions; + case 'branch': + return this.branchVersions; + case 'local': + return ['local']; + default: + return []; + } + }, + + actualVersionsList() { + switch (this.runConfig.actualSource) { + case 'release': + return this.versions; + case 'nightly': + return this.nightlyVersions; + case 'branch': + return this.branchVersions; + case 'local': + return ['local']; + default: + return []; + } } }, @@ -266,6 +346,44 @@ const app = new Vue({ 'currentTestName'(newVal, oldVal) { updateUrl(); + }, + + 'runConfig.expectedSource': { + handler(newVal, oldVal) { + if (newVal === oldVal) { + return; + } + + handlingSourceChange = true; + this.$nextTick(() => { + this.runConfig.expectedVersion = getVersionFromSource( + newVal, + this.versions, + this.nightlyVersions + ); + handlingSourceChange = false; + }); + }, + deep: false + }, + + 'runConfig.actualSource': { + handler(newVal, oldVal) { + if (newVal === oldVal) { + return; + } + + handlingSourceChange = true; + this.$nextTick(() => { + this.runConfig.actualVersion = getVersionFromSource( + newVal, + this.versions, + this.nightlyVersions + ); + handlingSourceChange = false; + }); + }, + deep: false } }, @@ -339,8 +457,10 @@ const app = new Vue({ let searches = []; let ecVersion = test[version + 'Version']; + let ecSource = test[version + 'Source']; if (ecVersion !== 'local') { - searches.push('__ECDIST__=' + ecVersion); + let distPath = ecSource === 'branch' ? 'branch/' + ecVersion : ecVersion; + searches.push('__ECDIST__=' + distPath); } if (test.useSVG) { searches.push('__RENDERER__=svg'); @@ -367,8 +487,6 @@ const app = new Vue({ this.runConfig.expectedVersion = runResult.expectedVersion; this.runConfig.actualVersion = runResult.actualVersion; // TODO - this.runConfig.isExpectedNightly = runResult.expectedVersion.includes('-dev.'); - this.runConfig.isActualNightly = runResult.actualVersion.includes('-dev.'); this.runConfig.renderer = runResult.renderer; this.runConfig.useCoarsePointer = runResult.useCoarsePointer; @@ -397,6 +515,17 @@ const app = new Vue({ open(url, target) { window.open(url, target); + }, + + async fetchBranchVersions() { + try { + const response = await fetch('https://api.github.com/repos/apache/echarts/branches?per_page=100'); + const branches = await response.json(); + this.branchVersions = branches.map(branch => branch.name); + } catch (error) { + console.error('Failed to fetch branches:', error); + this.branchVersions = []; + } } } }); @@ -419,7 +548,9 @@ function runTests(tests, noHeadless) { app.running = true; socket.emit('run', { tests, + expectedSource: app.runConfig.expectedSource, expectedVersion: app.runConfig.expectedVersion, + actualSource: app.runConfig.actualSource, actualVersion: app.runConfig.actualVersion, threads: app.runConfig.threads, renderer: app.runConfig.renderer, @@ -498,7 +629,7 @@ function updateUrl() { // Only update url when version is changed. app.$watch('runConfig', (newVal, oldVal) => { - if (!app.pageInvisible) { + if (!app.pageInvisible && !handlingSourceChange) { socket.emit('syncRunConfig', { runConfig: app.runConfig, // Override server config from URL. diff --git a/test/runTest/client/index.html b/test/runTest/client/index.html index df8cdc9a78..6cdeb63cce 100644 --- a/test/runTest/client/index.html +++ b/test/runTest/client/index.html @@ -144,24 +144,40 @@

Visual Regression Testing Tool

Expected - - - - + + + + + + Actual - - - - + + + + + + diff --git a/test/runTest/server.js b/test/runTest/server.js index 5361a1b940..93fce69ff8 100644 --- a/test/runTest/server.js +++ b/test/runTest/server.js @@ -48,9 +48,10 @@ const { getAllTestsRuns, delTestsRun, RESULTS_ROOT_DIR, - checkStoreVersion + checkStoreVersion, + clearStaledResults } = require('./store'); -const {prepareEChartsLib, getActionsFullPath, fetchVersions} = require('./util'); +const {prepareEChartsLib, getActionsFullPath, fetchVersions, cleanBranchDirectory} = require('./util'); const fse = require('fs-extra'); const fs = require('fs'); const open = require('open'); @@ -62,6 +63,8 @@ console.info(chalk.green('useCNMirror:'), useCNMirror); const CLI_FIXED_THREADS_COUNT = 1; function serve() { + clearStaledResults(); + const server = http.createServer((request, response) => { return handler(request, response, { cleanUrls: false, @@ -144,7 +147,9 @@ function startTests(testsNameList, socket, { noHeadless, threadsCount, replaySpeed, + actualSource, actualVersion, + expectedSource, expectedVersion, renderer, useCoarsePointer, @@ -206,6 +211,8 @@ function startTests(testsNameList, socket, { '--speed', replaySpeed || 5, '--actual', actualVersion, '--expected', expectedVersion, + '--actual-source', actualSource, + '--expected-source', expectedSource, '--renderer', renderer || '', '--use-coarse-pointer', useCoarsePointer, '--threads', Math.min(threadsCount, CLI_FIXED_THREADS_COUNT), @@ -230,6 +237,9 @@ function checkPuppeteer() { async function start() { + // Clean branch directory before starting + cleanBranchDirectory(); + if (!checkPuppeteer()) { // TODO Check version. console.error(`Can't find puppeteer >= 9.0.0, run 'npm install' to update in the 'test/runTest' folder`); @@ -241,8 +251,6 @@ async function start() { try { stableVersions = await fetchVersions(false, useCNMirror); nightlyVersions = (await fetchVersions(true, useCNMirror)).slice(0, 100); - stableVersions.unshift('local'); - nightlyVersions.unshift('local'); } catch (e) { console.error('Failed to fetch version list:', e); console.log(`Try again later or try the CN mirror with: ${chalk.yellow('npm run test:visual -- -- --useCNMirror')}`) @@ -283,20 +291,10 @@ async function start() { return; } - const expectedVersionsList = _currentRunConfig.isExpectedNightly ? nightlyVersions : stableVersions; - const actualVersionsList = _currentRunConfig.isActualNightly ? nightlyVersions : stableVersions; - if (!expectedVersionsList.includes(_currentRunConfig.expectedVersion)) { - // Pick first version not local - _currentRunConfig.expectedVersion = expectedVersionsList[1]; - } - if (!actualVersionsList.includes(_currentRunConfig.actualVersion)) { - _currentRunConfig.actualVersion = 'local'; - } - socket.emit('syncRunConfig_return', { runConfig: _currentRunConfig, - expectedVersionsList, - actualVersionsList + versions: stableVersions, + nightlyVersions: nightlyVersions }); if (_currentTestHash !== getRunHash(_currentRunConfig)) { @@ -320,6 +318,7 @@ async function start() { }); socket.on('genTestsRunReport', async (params) => { + console.log('genTestsRunReport', params); const absPath = await genReport( path.join(RESULTS_ROOT_DIR, getRunHash(params)) ); @@ -340,16 +339,16 @@ async function start() { let startTime = Date.now(); - await prepareEChartsLib(data.expectedVersion, useCNMirror); // Expected version. - await prepareEChartsLib(data.actualVersion, useCNMirror); // Version to test + try { + await prepareEChartsLib(data.expectedSource, data.expectedVersion, useCNMirror); + await prepareEChartsLib(data.actualSource, data.actualVersion, useCNMirror); - // If aborted in the time downloading lib. - if (isAborted) { - return; - } + // If aborted in the time downloading lib. + if (isAborted) { + return; + } - // TODO Should broadcast to all sockets. - try { + // TODO Should broadcast to all sockets. if (!checkStoreVersion(data)) { throw new Error('Unmatched store version and run version.'); } @@ -361,7 +360,9 @@ async function start() { noHeadless: data.noHeadless, threadsCount: data.threads, replaySpeed: data.replaySpeed, + actualSource: data.actualSource, actualVersion: data.actualVersion, + expectedSource: data.expectedSource, expectedVersion: data.expectedVersion, renderer: data.renderer, useCoarsePointer: data.useCoarsePointer, diff --git a/test/runTest/store.js b/test/runTest/store.js index 3430bd47e8..f6bd6aeaa8 100644 --- a/test/runTest/store.js +++ b/test/runTest/store.js @@ -78,7 +78,9 @@ class Test { */ function getRunHash(params) { return [ + params.expectedSource, params.expectedVersion, + params.actualSource, params.actualVersion, params.renderer, params.useCoarsePointer @@ -91,10 +93,12 @@ function getRunHash(params) { function parseRunHash(str) { const parts = str.split(TEST_HASH_SPLITTER); return { - expectedVersion: parts[0], - actualVersion: parts[1], - renderer: parts[2], - useCoarsePointer: parts[3] + expectedSource: parts[0], + expectedVersion: parts[1], + actualSource: parts[2], + actualVersion: parts[3], + renderer: parts[4], + useCoarsePointer: parts[5] }; } @@ -102,6 +106,22 @@ function getResultBaseDir() { return path.join(RESULTS_ROOT_DIR, _runHash); } +module.exports.clearStaledResults = async function () { + // If split by __ and there is no 6 parts, it is staled. + try { + const dirs = await globby('*', { cwd: RESULTS_ROOT_DIR, onlyDirectories: true }); + for (let dir of dirs) { + const parts = dir.split(TEST_HASH_SPLITTER); + if (parts.length !== 6) { + await module.exports.delTestsRun(dir); + } + } + } + catch(e) { + console.error('Failed to clear staled results', e); + } +} + module.exports.getResultBaseDir = getResultBaseDir; module.exports.getRunHash = getRunHash; @@ -111,7 +131,9 @@ module.exports.getRunHash = getRunHash; module.exports.checkStoreVersion = function (runParams) { const storeParams = parseRunHash(_runHash); console.log('Store ', _runHash); - return storeParams.expectedVersion === runParams.expectedVersion + return storeParams.expectedSource === runParams.expectedSource + && storeParams.expectedVersion === runParams.expectedVersion + && storeParams.actualSource === runParams.actualSource && storeParams.actualVersion === runParams.actualVersion && storeParams.renderer === runParams.renderer && storeParams.useCoarsePointer === runParams.useCoarsePointer; @@ -301,14 +323,22 @@ module.exports.getAllTestsRuns = async function () { continue; } - params.lastRunTime = lastRunTime > 0 ? formatDate(lastRunTime) : 'N/A'; - params.total = total; - params.passed = passedCount; - params.finished = finishedCount; - params.id = dir; - params.diskSize = convertBytes(await getFolderSize(path.join(RESULTS_ROOT_DIR, dir))); - - results.push(params); + const runData = { + expectedSource: params.expectedSource, + expectedVersion: params.expectedVersion, + actualSource: params.actualSource, + actualVersion: params.actualVersion, + renderer: params.renderer, + useCoarsePointer: params.useCoarsePointer, + lastRunTime: lastRunTime > 0 ? formatDate(lastRunTime) : 'N/A', + total: total, + passed: passedCount, + finished: finishedCount, + id: dir, + diskSize: convertBytes(await getFolderSize(path.join(RESULTS_ROOT_DIR, dir))) + }; + + results.push(runData); }; return results; } diff --git a/test/runTest/util.js b/test/runTest/util.js index fd718cd3c4..832e60ad5c 100644 --- a/test/runTest/util.js +++ b/test/runTest/util.js @@ -42,9 +42,10 @@ module.exports.fileNameFromTest = function (testName) { return testName + '.html'; }; -function getVersionDir(version) { +function getVersionDir(source, version) { version = version || 'local'; - return `tmp/__version__/${version}`; + const dir = source === 'branch' ? 'branch/' : ''; + return `tmp/__version__/${dir}${version}`; }; module.exports.getVersionDir = getVersionDir; @@ -56,31 +57,52 @@ module.exports.getEChartsTestFileName = function () { return `echarts.test-${config.testVersion}.js`; }; -module.exports.prepareEChartsLib = function (version, useCNMirror) { +// Clean branch directory at the start of initing because code in branch may change +module.exports.cleanBranchDirectory = function () { + const branchDir = path.join(__dirname, 'tmp/__version__/branch'); + if (fs.existsSync(branchDir)) { + fse.removeSync(branchDir); + } +} + +module.exports.prepareEChartsLib = function (source, version, useCNMirror) { + console.log(`Preparing ECharts lib: ${source} ${version}`); - const versionFolder = path.join(__dirname, getVersionDir(version)); + const versionFolder = path.join(__dirname, getVersionDir(source, version)); const ecDownloadPath = `${versionFolder}/echarts.js`; fse.ensureDirSync(versionFolder); + if (!version || version === 'local') { // Developing version, make sure it's new build fse.copySync(path.join(__dirname, '../../dist/echarts.js'), `${versionFolder}/echarts.js`); let code = modifyEChartsCode(fs.readFileSync(ecDownloadPath, 'utf-8')); fs.writeFileSync(`${versionFolder}/${module.exports.getEChartsTestFileName()}`, code, 'utf-8'); - return Promise.resolve(); } + return new Promise((resolve, reject) => { const testLibPath = `${versionFolder}/${module.exports.getEChartsTestFileName()}`; if (!fs.existsSync(ecDownloadPath)) { const file = fs.createWriteStream(ecDownloadPath); - const isNightly = version.includes('-dev'); - const packageName = isNightly ? 'echarts-nightly' : 'echarts' + let url; - const url = useCNMirror - ? `https://registry.npmmirror.com/${packageName}/${version}/files/dist/echarts.js` - : `https://unpkg.com/${packageName}@${version}/dist/echarts.js`; - console.log(`Downloading ${packageName}@${version} from ${url}`); + if (source === 'branch') { + url = `https://raw.githubusercontent.com/apache/echarts/${version}/dist/echarts.js`; + } + else { + const isNightly = source === 'nightly'; + const packageName = isNightly ? 'echarts-nightly' : 'echarts'; + url = useCNMirror + ? `https://registry.npmmirror.com/${packageName}/${version}/files/dist/echarts.js` + : `https://unpkg.com/${packageName}@${version}/dist/echarts.js`; + } + + console.log(`Downloading ECharts from ${url}`); https.get(url, response => { + if (response.statusCode === 404) { + reject(`Failed to download: ${url} (404 Not Found)`); + return; + } response.pipe(file); file.on('finish', () => { @@ -89,7 +111,7 @@ module.exports.prepareEChartsLib = function (version, useCNMirror) { resolve(); }); }).on('error', (e) => { - reject(`Failed to download ${packageName}@${version} from ${url}: ${e}`); + reject(`Failed to download from ${url}: ${e}`); }); } else { From 7205ad224639ef7e7b6814c651a4e5fa8e1505f0 Mon Sep 17 00:00:00 2001 From: Ovilia Date: Mon, 13 Jan 2025 15:26:32 +0800 Subject: [PATCH 2/3] chore(test): use PR preview instead of branch --- test/runTest/client/client.js | 35 ++++++++--- test/runTest/client/index.html | 26 ++++++-- test/runTest/server.js | 67 +++++++++++---------- test/runTest/store.js | 24 ++++++-- test/runTest/util.js | 106 ++++++++++++++++++++------------- 5 files changed, 167 insertions(+), 91 deletions(-) diff --git a/test/runTest/client/client.js b/test/runTest/client/client.js index 13f1af686a..34891725fd 100644 --- a/test/runTest/client/client.js +++ b/test/runTest/client/client.js @@ -110,8 +110,9 @@ try { catch (e) {} function getVersionFromSource(source, versions, nightlyVersions) { - if (source === 'branch') { - return 'master'; + if (source === 'PR') { + // Default PR version can be empty since it needs to be manually selected + return '#'; } else if (source === 'nightly') { return nightlyVersions.length ? nightlyVersions[0] : null; @@ -150,6 +151,7 @@ const app = new Vue({ versions: [], nightlyVersions: [], + prVersions: [], branchVersions: [], runConfig: Object.assign({ @@ -178,6 +180,7 @@ const app = new Vue({ socket.on('syncRunConfig_return', res => { this.versions = res.versions || []; this.nightlyVersions = res.nightlyVersions || []; + this.prVersions = res.prVersions || []; // Only set versions if they haven't been manually set handlingSourceChange = true; @@ -222,6 +225,16 @@ const app = new Vue({ this.pageInvisible = true; } }); + + socket.on('run_error', err => { + app.$notify({ + title: 'Error', + message: err.message, + type: 'error', + duration: 5000 + }); + app.running = false; + }); }, computed: { @@ -312,8 +325,8 @@ const app = new Vue({ return this.versions; case 'nightly': return this.nightlyVersions; - case 'branch': - return this.branchVersions; + case 'PR': + return this.prVersions; case 'local': return ['local']; default: @@ -327,8 +340,8 @@ const app = new Vue({ return this.versions; case 'nightly': return this.nightlyVersions; - case 'branch': - return this.branchVersions; + case 'PR': + return this.prVersions; case 'local': return ['local']; default: @@ -632,8 +645,16 @@ app.$watch('runConfig', (newVal, oldVal) => { if (!app.pageInvisible && !handlingSourceChange) { socket.emit('syncRunConfig', { runConfig: app.runConfig, - // Override server config from URL. forceSet: true + }, err => { + if (err) { + app.$notify({ + title: 'Error', + message: err, + type: 'error', + duration: 5000 + }); + } }); } }, { deep: true }); \ No newline at end of file diff --git a/test/runTest/client/index.html b/test/runTest/client/index.html index 6cdeb63cce..f5270a90a5 100644 --- a/test/runTest/client/index.html +++ b/test/runTest/client/index.html @@ -1,4 +1,3 @@ - + Actual - + Visual Regression Testing Tool > +
Renderer diff --git a/test/runTest/server.js b/test/runTest/server.js index 93fce69ff8..a6f7e29337 100644 --- a/test/runTest/server.js +++ b/test/runTest/server.js @@ -51,7 +51,7 @@ const { checkStoreVersion, clearStaledResults } = require('./store'); -const {prepareEChartsLib, getActionsFullPath, fetchVersions, cleanBranchDirectory} = require('./util'); +const {prepareEChartsLib, getActionsFullPath, fetchVersions, cleanBranchDirectory, fetchRecentPRs} = require('./util'); const fse = require('fs-extra'); const fs = require('fs'); const open = require('open'); @@ -237,11 +237,11 @@ function checkPuppeteer() { async function start() { - // Clean branch directory before starting - cleanBranchDirectory(); + // Clean PR directories before starting + const {cleanPRDirectories} = require('./util'); + cleanPRDirectories(); if (!checkPuppeteer()) { - // TODO Check version. console.error(`Can't find puppeteer >= 9.0.0, run 'npm install' to update in the 'test/runTest' folder`); return; } @@ -342,38 +342,43 @@ async function start() { try { await prepareEChartsLib(data.expectedSource, data.expectedVersion, useCNMirror); await prepareEChartsLib(data.actualSource, data.actualVersion, useCNMirror); - - // If aborted in the time downloading lib. - if (isAborted) { - return; - } - - // TODO Should broadcast to all sockets. - if (!checkStoreVersion(data)) { - throw new Error('Unmatched store version and run version.'); - } - - await startTests( - data.tests, - io.of('/client'), - { - noHeadless: data.noHeadless, - threadsCount: data.threads, - replaySpeed: data.replaySpeed, - actualSource: data.actualSource, - actualVersion: data.actualVersion, - expectedSource: data.expectedSource, - expectedVersion: data.expectedVersion, - renderer: data.renderer, - useCoarsePointer: data.useCoarsePointer, - noSave: false - } - ); } catch (e) { console.error(e); + // Send error to client + socket.emit('run_error', { + message: e.toString() + }); + return; } + // If aborted in the time downloading lib. + if (isAborted) { + return; + } + + // TODO Should broadcast to all sockets. + if (!checkStoreVersion(data)) { + throw new Error('Unmatched store version and run version.'); + } + + await startTests( + data.tests, + io.of('/client'), + { + noHeadless: data.noHeadless, + threadsCount: data.threads, + replaySpeed: data.replaySpeed, + actualSource: data.actualSource, + actualVersion: data.actualVersion, + expectedSource: data.expectedSource, + expectedVersion: data.expectedVersion, + renderer: data.renderer, + useCoarsePointer: data.useCoarsePointer, + noSave: false + } + ); + if (!isAborted) { const deltaTime = Date.now() - startTime; console.log('Finished in ', Math.round(deltaTime / 1000) + ' second'); diff --git a/test/runTest/store.js b/test/runTest/store.js index f6bd6aeaa8..21d1d2d49e 100644 --- a/test/runTest/store.js +++ b/test/runTest/store.js @@ -77,11 +77,19 @@ class Test { * It depends on two versions and rendering mode. */ function getRunHash(params) { + // Replace # with PR- in the hash to avoid URL issues + const expectedVersion = params.expectedSource === 'PR' + ? params.expectedVersion.replace('#', 'PR-') + : params.expectedVersion; + const actualVersion = params.actualSource === 'PR' + ? params.actualVersion.replace('#', 'PR-') + : params.actualVersion; + return [ params.expectedSource, - params.expectedVersion, + expectedVersion, params.actualSource, - params.actualVersion, + actualVersion, params.renderer, params.useCoarsePointer ].join(TEST_HASH_SPLITTER); @@ -92,11 +100,19 @@ function getRunHash(params) { */ function parseRunHash(str) { const parts = str.split(TEST_HASH_SPLITTER); + // Convert back PR-123 to #123 for PR versions + const expectedVersion = parts[0] === 'PR' + ? parts[1].replace('PR-', '#') + : parts[1]; + const actualVersion = parts[2] === 'PR' + ? parts[3].replace('PR-', '#') + : parts[3]; + return { expectedSource: parts[0], - expectedVersion: parts[1], + expectedVersion: expectedVersion, actualSource: parts[2], - actualVersion: parts[3], + actualVersion: actualVersion, renderer: parts[4], useCoarsePointer: parts[5] }; diff --git a/test/runTest/util.js b/test/runTest/util.js index 832e60ad5c..f573c7935e 100644 --- a/test/runTest/util.js +++ b/test/runTest/util.js @@ -44,9 +44,13 @@ module.exports.fileNameFromTest = function (testName) { function getVersionDir(source, version) { version = version || 'local'; - const dir = source === 'branch' ? 'branch/' : ''; - return `tmp/__version__/${dir}${version}`; -}; + if (source === 'PR') { + // For PR preview artifacts + const prNumber = version.replace(/^#/, ''); + return `tmp/__version__/pr-${prNumber}`; + } + return `tmp/__version__/${version}`; +} module.exports.getVersionDir = getVersionDir; module.exports.getActionsFullPath = function (testName) { @@ -57,11 +61,16 @@ module.exports.getEChartsTestFileName = function () { return `echarts.test-${config.testVersion}.js`; }; -// Clean branch directory at the start of initing because code in branch may change -module.exports.cleanBranchDirectory = function () { - const branchDir = path.join(__dirname, 'tmp/__version__/branch'); - if (fs.existsSync(branchDir)) { - fse.removeSync(branchDir); +// Clean PR directories at the start of initing because PR code may change +module.exports.cleanPRDirectories = function () { + const baseDir = path.join(__dirname, 'tmp/__version__'); + if (fs.existsSync(baseDir)) { + const dirs = fs.readdirSync(baseDir); + dirs.forEach(dir => { + if (dir.startsWith('pr-')) { + fse.removeSync(path.join(baseDir, dir)); + } + }); } } @@ -70,57 +79,68 @@ module.exports.prepareEChartsLib = function (source, version, useCNMirror) { const versionFolder = path.join(__dirname, getVersionDir(source, version)); const ecDownloadPath = `${versionFolder}/echarts.js`; + const testLibPath = `${versionFolder}/${module.exports.getEChartsTestFileName()}`; + + // Check if both files exist and are not empty + if (fs.existsSync(ecDownloadPath) && fs.existsSync(testLibPath) && + fs.statSync(ecDownloadPath).size > 0 && fs.statSync(testLibPath).size > 0) { + return Promise.resolve(); + } + fse.ensureDirSync(versionFolder); if (!version || version === 'local') { // Developing version, make sure it's new build - fse.copySync(path.join(__dirname, '../../dist/echarts.js'), `${versionFolder}/echarts.js`); + fse.copySync(path.join(__dirname, '../../dist/echarts.js'), ecDownloadPath); let code = modifyEChartsCode(fs.readFileSync(ecDownloadPath, 'utf-8')); - fs.writeFileSync(`${versionFolder}/${module.exports.getEChartsTestFileName()}`, code, 'utf-8'); + fs.writeFileSync(testLibPath, code, 'utf-8'); return Promise.resolve(); } return new Promise((resolve, reject) => { - const testLibPath = `${versionFolder}/${module.exports.getEChartsTestFileName()}`; - if (!fs.existsSync(ecDownloadPath)) { - const file = fs.createWriteStream(ecDownloadPath); - let url; + let url; - if (source === 'branch') { - url = `https://raw.githubusercontent.com/apache/echarts/${version}/dist/echarts.js`; + if (source === 'PR') { + const prNumber = version.replace(/^#/, ''); + if (!/^\d+$/.test(prNumber)) { + reject('Invalid PR number format. Should be #123'); + return; } - else { - const isNightly = source === 'nightly'; - const packageName = isNightly ? 'echarts-nightly' : 'echarts'; - url = useCNMirror - ? `https://registry.npmmirror.com/${packageName}/${version}/files/dist/echarts.js` - : `https://unpkg.com/${packageName}@${version}/dist/echarts.js`; + url = `https://echarts-pr-${prNumber}.surge.sh/dist/echarts.js`; + } + else { + const isNightly = source === 'nightly'; + const packageName = isNightly ? 'echarts-nightly' : 'echarts'; + url = useCNMirror + ? `https://registry.npmmirror.com/${packageName}/${version}/files/dist/echarts.js` + : `https://unpkg.com/${packageName}@${version}/dist/echarts.js`; + } + + console.log(`Downloading ECharts from ${url}`); + https.get(url, response => { + if (response.statusCode === 404) { + reject(`PR artifact doesn't exist at ${url}. Make sure the PR build is complete.`); + return; } - console.log(`Downloading ECharts from ${url}`); - https.get(url, response => { - if (response.statusCode === 404) { - reject(`Failed to download: ${url} (404 Not Found)`); + let data = ''; + response.on('data', chunk => { + data += chunk; + }); + + response.on('end', () => { + if (!data) { + reject(`Downloaded file is empty from ${url}`); return; } - response.pipe(file); - - file.on('finish', () => { - const code = modifyEChartsCode(fs.readFileSync(ecDownloadPath, 'utf-8')); - fs.writeFileSync(testLibPath, code, 'utf-8'); - resolve(); - }); - }).on('error', (e) => { - reject(`Failed to download from ${url}: ${e}`); + fs.writeFileSync(ecDownloadPath, data, 'utf-8'); + const code = modifyEChartsCode(data); + fs.writeFileSync(testLibPath, code, 'utf-8'); + resolve(); }); - } - else { - // Always do code modification. - // In case we need to do replacement on old downloads. - let code = modifyEChartsCode(fs.readFileSync(ecDownloadPath, 'utf-8')); - fs.writeFileSync(testLibPath, code, 'utf-8'); - resolve(); - } + }).on('error', (e) => { + reject(`Failed to download from ${url}: ${e}`); + }); }); }; From 271f9db43e0394316ec3064bcfcc7a249d34a90d Mon Sep 17 00:00:00 2001 From: Ovilia Date: Tue, 14 Jan 2025 15:36:44 +0800 Subject: [PATCH 3/3] chore(test): fix local echarts file path --- test/runTest/client/client.js | 4 +++- test/runTest/util.js | 10 ++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/runTest/client/client.js b/test/runTest/client/client.js index 34891725fd..ba8d811177 100644 --- a/test/runTest/client/client.js +++ b/test/runTest/client/client.js @@ -472,7 +472,9 @@ const app = new Vue({ let ecVersion = test[version + 'Version']; let ecSource = test[version + 'Source']; if (ecVersion !== 'local') { - let distPath = ecSource === 'branch' ? 'branch/' + ecVersion : ecVersion; + let distPath = ecSource === 'PR' + ? 'pr-' + ecVersion.replace(/^#/, '') + : ecVersion; searches.push('__ECDIST__=' + distPath); } if (test.useSVG) { diff --git a/test/runTest/util.js b/test/runTest/util.js index f573c7935e..751b6a1bb3 100644 --- a/test/runTest/util.js +++ b/test/runTest/util.js @@ -81,12 +81,6 @@ module.exports.prepareEChartsLib = function (source, version, useCNMirror) { const ecDownloadPath = `${versionFolder}/echarts.js`; const testLibPath = `${versionFolder}/${module.exports.getEChartsTestFileName()}`; - // Check if both files exist and are not empty - if (fs.existsSync(ecDownloadPath) && fs.existsSync(testLibPath) && - fs.statSync(ecDownloadPath).size > 0 && fs.statSync(testLibPath).size > 0) { - return Promise.resolve(); - } - fse.ensureDirSync(versionFolder); if (!version || version === 'local') { @@ -96,6 +90,10 @@ module.exports.prepareEChartsLib = function (source, version, useCNMirror) { fs.writeFileSync(testLibPath, code, 'utf-8'); return Promise.resolve(); } + else if (fs.existsSync(ecDownloadPath) && fs.existsSync(testLibPath) && + fs.statSync(ecDownloadPath).size > 0 && fs.statSync(testLibPath).size > 0) { + return Promise.resolve(); + } return new Promise((resolve, reject) => { let url;