Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(smoke-tests): test updating from the latest release to the newly packaged app COMPASS-8532 COMPASS-8535 #6669

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

feat(smoke-tests): test updating from the latest release to the newly packaged app COMPASS-8532 COMPASS-8535 #6669

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
28bbb85
test updating from the latest release to the newly packaged one
lerouxb Jan 30, 2025
4074347
also run this in CI
lerouxb Jan 30, 2025
3c7162d
better type assertions
lerouxb Jan 31, 2025
1ec2465
auto-updating TO doesn't support windows yet because we don't have .e…
lerouxb Jan 31, 2025
a897d93
the installer to use for the latest release is not necessarily the sa…
lerouxb Jan 31, 2025
1c90eba
check the text in the toasts
lerouxb Jan 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .evergreen/functions.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -678,8 +678,8 @@ functions:

if [[ "$IS_WINDOWS" == "true" ]]; then
# TODO: windows_setup
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_zip --tests=auto-update-from
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_msi --tests=auto-update-from
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_zip --tests auto-update-from auto-update-to
npm run --unsafe-perm --workspace @mongodb-js/compass-smoke-tests start -- --package=windows_msi --tests auto-update-from auto-update-to
fi

if [[ "$IS_OSX" == "true" ]]; then
Expand Down
113 changes: 59 additions & 54 deletions packages/compass-e2e-tests/tests/auto-update.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,66 +13,71 @@ function wait(ms: number) {
}

describe('Auto-update', function () {
it('auto-update from', async function () {
if (process.env.TEST_NAME !== 'auto-update-from') {
// we don't want this test to execute along with all the others under
// normal circumstances because it is destructive - it overwrites Compass
// itself
this.skip();
}

// run the app and wait for it to auto-update
console.log('starting compass the first time');
const compass = await init('auto-update from', { firstRun: true });
const { browser } = compass;
try {
await browser.$(Selectors.AutoUpdateToast).waitForDisplayed();

if (process.env.AUTO_UPDATE_UPDATABLE === 'true') {
const restartButton = browser.$(Selectors.AutoUpdateRestartButton);
await restartButton.waitForDisplayed();

// We could click the restart button to apply the update and restart the
// app, but restarting the app confuses webdriverio or at least our test
// helpers. So we're going to just restart the app manually.
await browser.pause(1000);
} else {
// When auto-update is not supported the toast contains a link to
// download
const linkElement = browser.$(Selectors.AutoUpdateDownloadLink);
await linkElement.waitForDisplayed();
expect(await linkElement.getAttribute('href')).to.equal(
'https://www.mongodb.com/try/download/compass?utm_source=compass&utm_medium=product'
);
for (const testName of ['auto-update-from', 'auto-update-to']) {
it(testName, async function () {
if (process.env.TEST_NAME !== testName) {
// we don't want this test to execute along with all the others under
// normal circumstances because it is destructive - it overwrites Compass
// itself
this.skip();
}
} finally {
await browser.screenshot(screenshotPathName('auto-update-from'));
await cleanup(compass);
}

if (process.env.AUTO_UPDATE_UPDATABLE === 'true') {
console.log(
'pause to make sure the app properly exited before starting again'
);
await wait(10_000);

console.log('starting compass a second time');
// run the app again and check that the version changed
const compass = await init('auto-update from restart', {
firstRun: false,
});
// run the app and wait for it to auto-update
console.log('starting compass the first time');
const compass = await init(testName, { firstRun: true });
const { browser } = compass;
try {
await browser.$(Selectors.AutoUpdateToast).waitForDisplayed();
await browser
.$(Selectors.AutoUpdateReleaseNotesLink)
.waitForDisplayed();

if (process.env.AUTO_UPDATE_UPDATABLE === 'true') {
const restartButton = browser.$(Selectors.AutoUpdateRestartButton);
await restartButton.waitForDisplayed();

// We could click the restart button to apply the update and restart the
// app, but restarting the app confuses webdriverio or at least our test
// helpers. So we're going to just restart the app manually.
await browser.pause(1000);
} else {
// When auto-update is not supported the toast contains a link to
// download
const linkElement = browser.$(Selectors.AutoUpdateDownloadLink);
await linkElement.waitForDisplayed();
expect(await linkElement.getAttribute('href')).to.equal(
'https://www.mongodb.com/try/download/compass?utm_source=compass&utm_medium=product'
);

// TODO: when updating to a known version we know the version, so
lerouxb marked this conversation as resolved.
Show resolved Hide resolved
// check for the text
}
} finally {
await browser.screenshot(
screenshotPathName('auto-update-from-restart')
);
await browser.screenshot(screenshotPathName(testName));
await cleanup(compass);
}
}
});

if (process.env.AUTO_UPDATE_UPDATABLE === 'true') {
console.log(
'pause to make sure the app properly exited before starting again'
);
await wait(10_000);

console.log('starting compass a second time');
// run the app again and check that the version changed
const compass = await init(`${testName} restart`, {
firstRun: false,
});
const { browser } = compass;
try {
await browser.$(Selectors.AutoUpdateToast).waitForDisplayed();
await browser
.$(Selectors.AutoUpdateReleaseNotesLink)
.waitForDisplayed();
// TODO: when updating to a known version we know the version, so
lerouxb marked this conversation as resolved.
Show resolved Hide resolved
// check for the text
} finally {
await browser.screenshot(screenshotPathName(`${testName}-restart`));
await cleanup(compass);
}
}
});
}
});
16 changes: 14 additions & 2 deletions packages/compass-smoke-tests/src/build-info.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,10 @@ import { type PackageKind } from './packages';
import { type SmokeTestsContext } from './context';
import { pick } from 'lodash';

const SUPPORTED_CHANNELS = ['dev', 'beta', 'stable'] as const;

export type Channel = typeof SUPPORTED_CHANNELS[number];

function assertObjectHasKeys(
obj: unknown,
name: string,
Expand All @@ -25,13 +29,21 @@ function assertObjectHasKeys(

// subsets of the hadron-build info result

export const commonKeys = ['productName'] as const;
export type CommonBuildInfo = Record<typeof commonKeys[number], string>;
export const commonKeys = ['productName', 'version', 'channel'] as const;
export type CommonBuildInfo = Record<typeof commonKeys[number], string> & {
channel: Channel;
};

export function assertCommonBuildInfo(
buildInfo: unknown
): asserts buildInfo is CommonBuildInfo {
assertObjectHasKeys(buildInfo, 'buildInfo', commonKeys);
assert(
SUPPORTED_CHANNELS.includes((buildInfo as { channel: Channel }).channel),
lerouxb marked this conversation as resolved.
Show resolved Hide resolved
`Expected ${
(buildInfo as { channel: Channel }).channel
} to be in ${SUPPORTED_CHANNELS.join(',')}`
);
}

export const windowsFilenameKeys = [
Expand Down
156 changes: 41 additions & 115 deletions packages/compass-smoke-tests/src/cli.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,10 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import path from 'node:path';
import { once } from 'node:events';

import yargs from 'yargs';
import { hideBin } from 'yargs/helpers';
import { pick } from 'lodash';
import { execute, executeAsync } from './execute';
import {
type PackageDetails,
readPackageDetails,
Expand All @@ -16,18 +14,22 @@ import {
import { createSandbox } from './directories';
import { downloadFile } from './downloads';
import { type PackageKind, SUPPORTED_PACKAGES } from './packages';
import { getLatestRelease } from './releases';
import { SUPPORTED_TESTS } from './tests/types';
import { type SmokeTestsContext } from './context';

import { installMacDMG } from './installers/mac-dmg';
import { installMacZIP } from './installers/mac-zip';
import { installWindowsZIP } from './installers/windows-zip';
import { installWindowsMSI } from './installers/windows-msi';

import { testTimeToFirstQuery } from './tests/time-to-first-query';
import { testAutoUpdateFrom } from './tests/auto-update-from';
import { testAutoUpdateTo } from './tests/auto-update-to';

const SUPPORTED_PLATFORMS = ['win32', 'darwin', 'linux'] as const;
const SUPPORTED_ARCHS = ['x64', 'arm64'] as const;

const SUPPORTED_TESTS = ['time-to-first-query', 'auto-update-from'] as const;

function isSupportedPlatform(
value: unknown
): value is typeof SUPPORTED_PLATFORMS[number] {
Expand Down Expand Up @@ -198,9 +200,13 @@ async function run() {
])
);

const { kind, appName, filepath, autoUpdatable } = await getTestSubject(
context
);
const {
kind,
appName,
filepath,
buildInfo: { channel, version },
autoUpdatable,
} = await getTestSubject(context);
const install = getInstaller(kind);

try {
Expand All @@ -209,27 +215,51 @@ async function run() {
}

for (const testName of context.tests) {
const installerPath =
testName === 'auto-update-to'
? await getLatestRelease(
channel,
context.arch,
kind,
context.forceDownload
)
: filepath;

const { appPath, uninstall } = install({
appName,
filepath,
filepath: installerPath,
destinationPath: context.sandboxPath,
});

try {
if (testName === 'time-to-first-query') {
// Auto-update does not work on mac in CI at the moment. So in that case
// we just run the E2E tests to make sure the app at least starts up.
runTimeToFirstQuery({
testTimeToFirstQuery({
appName,
appPath,
});
}
if (testName === 'auto-update-from') {
await runUpdateTest({
await testAutoUpdateFrom({
appName,
appPath,
autoUpdatable,
});
}
if (testName === 'auto-update-to') {
assert(
context.bucketKeyPrefix !== undefined,
'Bucket key prefix is needed to download'
);

await testAutoUpdateTo({
appName,
appPath,
autoUpdatable,
testName,
channel,
bucketKeyPrefix: context.bucketKeyPrefix,
version,
});
}
} finally {
Expand All @@ -246,110 +276,6 @@ async function run() {
}
}

async function importUpdateServer() {
try {
return (await import('compass-mongodb-com')).default;
} catch (err: unknown) {
console.log('Remember to npm link compass-mongodb-com');
throw err;
}
}

async function startAutoUpdateServer() {
console.log('Starting auto-update server');
const { httpServer, updateChecker, start } = (await importUpdateServer())();
start();
await once(updateChecker, 'refreshed');

return httpServer;
}

type RunE2ETestOptions = {
appName: string;
appPath: string;
};

function runTimeToFirstQuery({ appName, appPath }: RunE2ETestOptions) {
execute(
'npm',
[
'run',
'--unsafe-perm',
'test-packaged',
'--workspace',
'compass-e2e-tests',
'--',
'--test-filter=time-to-first-query',
],
{
// We need to use a shell to get environment variables setup correctly
shell: true,
env: {
...process.env,
COMPASS_APP_NAME: appName,
COMPASS_APP_PATH: appPath,
},
}
);
}

type RunUpdateTestOptions = {
appName: string;
appPath: string;
autoUpdatable?: boolean;
testName: string;
};

async function runUpdateTest({
appName,
appPath,
autoUpdatable,
testName,
}: RunUpdateTestOptions) {
process.env.PORT = '0'; // dynamic port
process.env.UPDATE_CHECKER_ALLOW_DOWNGRADES = 'true';

const server = await startAutoUpdateServer();

const address = server.address();
assert(typeof address === 'object' && address !== null);
const port = address.port;
const HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE = `http://localhost:${port}`;
console.log({ HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE });

try {
// must be async because the update server is running in the same process
await executeAsync(
'npm',
[
'run',
'--unsafe-perm',
'test-packaged',
'--workspace',
'compass-e2e-tests',
'--',
'--test-filter=auto-update',
],
{
// We need to use a shell to get environment variables setup correctly
shell: true,
env: {
...process.env,
HADRON_AUTO_UPDATE_ENDPOINT_OVERRIDE,
AUTO_UPDATE_UPDATABLE: (!!autoUpdatable).toString(),
TEST_NAME: testName,
COMPASS_APP_NAME: appName,
COMPASS_APP_PATH: appPath,
},
}
);
} finally {
console.log('Stopping auto-update server');
server.close();
delete process.env.UPDATE_CHECKER_ALLOW_DOWNGRADES;
}
}

run()
.then(function () {
console.log('done');
Expand Down
Loading
Loading