From 72388df0aa02d323fc472d577cb4bbbfdfabc454 Mon Sep 17 00:00:00 2001 From: Seung Park Date: Thu, 11 Apr 2024 10:32:41 -0400 Subject: [PATCH 1/4] update Dockerfile --- Dockerfile | 4 ++-- Dockerfile.local | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4c6b50500..e36ca2d8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,8 @@ RUN cd ./modules/oas-page-builder \ # where repo work will happen FROM ubuntu:20.04 ARG WORK_DIRECTORY=/home/docsworker-xlarge -ARG SNOOTY_PARSER_VERSION=0.16.4 -ARG SNOOTY_FRONTEND_VERSION=0.16.8 +ARG SNOOTY_PARSER_VERSION=0.16.5 +ARG SNOOTY_FRONTEND_VERSION=0.16.9 ARG MUT_VERSION=0.11.1 ARG REDOC_CLI_VERSION=1.2.3 ARG NPM_BASE_64_AUTH diff --git a/Dockerfile.local b/Dockerfile.local index b918c4860..59abe87c3 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -1,8 +1,8 @@ FROM arm64v8/ubuntu:20.04 as initial ARG NPM_BASE_64_AUTH ARG NPM_EMAIL -ARG SNOOTY_PARSER_VERSION=0.16.4 -ARG SNOOTY_FRONTEND_VERSION=0.16.8 +ARG SNOOTY_PARSER_VERSION=0.16.5 +ARG SNOOTY_FRONTEND_VERSION=0.16.9 ARG MUT_VERSION=0.10.7 ARG REDOC_CLI_VERSION=1.2.3 ARG NPM_BASE_64_AUTH From 0cb298a52a74c9a9cd0439469ecf337243078c3c Mon Sep 17 00:00:00 2001 From: Seung Park Date: Thu, 11 Apr 2024 10:36:34 -0400 Subject: [PATCH 2/4] update mut version in Dockerfile --- Dockerfile | 2 +- Dockerfile.local | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e36ca2d8f..947f95652 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ FROM ubuntu:20.04 ARG WORK_DIRECTORY=/home/docsworker-xlarge ARG SNOOTY_PARSER_VERSION=0.16.5 ARG SNOOTY_FRONTEND_VERSION=0.16.9 -ARG MUT_VERSION=0.11.1 +ARG MUT_VERSION=0.11.2 ARG REDOC_CLI_VERSION=1.2.3 ARG NPM_BASE_64_AUTH ARG NPM_EMAIL diff --git a/Dockerfile.local b/Dockerfile.local index 59abe87c3..6f3fbaa5a 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -3,7 +3,7 @@ ARG NPM_BASE_64_AUTH ARG NPM_EMAIL ARG SNOOTY_PARSER_VERSION=0.16.5 ARG SNOOTY_FRONTEND_VERSION=0.16.9 -ARG MUT_VERSION=0.10.7 +ARG MUT_VERSION=0.11.2 ARG REDOC_CLI_VERSION=1.2.3 ARG NPM_BASE_64_AUTH ARG NPM_EMAIL From bb578460fbcad2802e1e66773acef14045d6f596 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Thu, 11 Apr 2024 13:39:51 -0400 Subject: [PATCH 3/4] DOP-4414: Lambda function to handle automated test deploys (#1011) * DOP-4414 pushing to preprd * DOP-4414 logging more * DOP-4414 logging * DOP-4414 logging * DOP-4414 logging * DOP-4414 logging * DOP-4414 initial changes * DOP-4414 majority of ticket * DOP-4414 cleaning up * DOP-4414 refactoring * DOP-4414 webhook api construct * DOP-4414 checks build on main * DOP-4414 update validation * DOP-4414 deploy to preprd * DOP-4414 testing webhook * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 testing lambda * DOP-4414 more changes * DOP-4414 cleaning up * DOP-4414 cleaning up * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 push to preprd * DOP-4414 return true early * DOP-4414 return true early * DOP-4414 return true early * DOP-4414 return true early * DOP-4414 return true early * DOP-4414 return true early * DOP-4414 return true early * DOP-4414 return true early * DOP-4414 remove validation * DOP-4414 fewer sites * DOP-4414 fewer sites * DOP-4414 fewer sites * DOP-4414 fewer sites * DOP-4414 fewer sites * DOP-4414 fewer sites * DOP-4414 fewer sites * DOP-4414 fewer sites * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing job prefix * DOP-4414 testing * DOP-4414 testing * DOP-4414 testing * DOP-4414 testing * DOP-4414 testing * DOP-4414 testing * DOP-4414 testing * DOP-4414 testing * DOP-4414 testing * DOP-4414 testing * DOP-4414 testing * DOP-4414 testing jobupdates queue * DOP-4414 testing refactor * DOP-4414 testing refactor * DOP-4414 testing jobupdatesqueue * DOP-4414 cleaning up * DOP-4414 webhook update and cleaning * DOP-4414 secret * DOP-4414 checking main * DOP-4414 checking main * DOP-4414 types and env vars * DOP-4414 comment out check for main branch * DOP-4414 entitlements repos * DOP-4414 testing * revert "DOP-4414 testing" This reverts commit c4ac8373134709786a22d3d7a859f9a819b1bcc7. reverting changes * Revert "DOP-4414 entitlements repos" This reverts commit c86c4665f8d052fec06d3c0e867ce4623e92c128. reverting commit * Revert "DOP-4414 comment out check for main branch" This reverts commit bce69c38757ce6d3e7ec3cef5b143a77c9842c0a. reverting commit * reverting commit Revert "DOP-4414 types and env vars" This reverts commit aee5cd49ab37e7404585cdc83b3f48df4be7c638. reverting commit * DOP-4414 comment out main check for testing * DOP-4414 testing * DOP-4414 add branch to deploy enhanced worker wq * DOP-4414 add branch to deploy enhanced webhook * DOP-4414 add branch to deploy enhanced webhook * DOP-4414 add branch to deploy enhanced webhook * DOP-4414 add branch to deploy enhanced webhook * DOP-4414 nits * DOP-4414 fixing small error * DOP-4414 refactoring promises * DOP-4414 testing new promise format * DOP-4414 testing new promise format * DOP-4414 testing new promise format * DOP-4414 refactoring and cleaning * DOP-4414 refactoring and cleaning * DOP-4414 adding projects collection as constant * DOP-4414 add docs_metadata as constant * DOP-4414 remove unecessary vars from the configs * DOP-4414 readded name * DOP-4414 testing * DOP-4414 testing env vars * DOP-4414 testing env vars * DOP-4414 testing env vars * DOP-4414 testing env vars * DOP-4414 testing env vars * DOP-4414 testing env vars removing from yamls * DOP-4414 testing env vars removing adding back to yamll * DOP-4414 testing env vars removing adding back to yamll * DOP-4414 testing env vars removing from get-env-vars * DOP-4414 testing env vars removing from get-env-vars * DOP-4414 testing env vars removing jsons * DOP-4414 ssm putparam * DOP-4414 removing env vars * DOP-4414 removing env vars * DOP-4414 removing more env vars * DOP-4414 removing more env vars * DOP-4414 uncomment main branch check * DOP-4414 cleaning up * DOP-4414 last cleaning * DOP-4414 removing from preprd * DOP-4414 playing with prefixes, all changes in this commit should not persist * DOP-4414 push to preprd * DOP-4414 trying to get jobhandler lambda to update and reflect changes * Fix workflow cache * Fix workflow paths * DOP-4414 nit changing how returnVal is declared * Fix workflow paths * Fix workflow paths * DOP-4414 changing prefix again * DOP-4414 adding code for newHead and changing mut prefix path * DOP-4414 removing newHead and passing in hash through title * DOP-4414 fixing conditional for amending mut prefix so that it executes * DOP-4414 placing commitHash at end of mutprefix instead of beginngin * DOP-4414 adding extra part to mutprefix path * DOP-4414 reversing changes made for testing purposes * DOP-4414 reverting more changes that were made for testing * DOP-4414 conditional nit --------- Co-authored-by: branberry --- .../deploy-stg-enhanced-webhooks.yml | 7 +- .../workflows/deploy-stg-enhanced-worker.yml | 1 - .github/workflows/update-feature-branch.yml | 12 + api/controllers/v2/github.ts | 264 +++++++++++++++--- api/controllers/v2/slack.ts | 4 +- .../constructs/api/webhook-api-construct.ts | 16 ++ .../constructs/api/webhook-env-construct.ts | 2 + .../lib/constructs/worker/worker-construct.ts | 2 +- .../constructs/worker/worker-env-construct.ts | 2 + src/commands/src/shared/next-gen-deploy.ts | 2 +- src/entities/job.ts | 4 +- src/job/jobHandler.ts | 2 +- src/job/productionJobHandler.ts | 2 +- src/repositories/baseRepository.ts | 6 +- src/repositories/jobRepository.ts | 2 +- src/repositories/projectsRepository.ts | 22 ++ .../repoEntitlementsRepository.ts | 2 +- 17 files changed, 299 insertions(+), 53 deletions(-) create mode 100644 src/repositories/projectsRepository.ts diff --git a/.github/workflows/deploy-stg-enhanced-webhooks.yml b/.github/workflows/deploy-stg-enhanced-webhooks.yml index f4d4dd938..97decd119 100644 --- a/.github/workflows/deploy-stg-enhanced-webhooks.yml +++ b/.github/workflows/deploy-stg-enhanced-webhooks.yml @@ -1,9 +1,9 @@ on: push: - paths: ["api/v2/**", "cdk-infra/lib/constructs/api/**", "cdk-infra/utils/**"] + paths: ['api/**', 'cdk-infra/lib/constructs/api/**', 'cdk-infra/utils/**'] branches: - - "main" - - "integration" + - 'main' + - 'integration' concurrency: group: environment-stg-enhanced-webhooks-${{ github.ref }} @@ -30,4 +30,3 @@ jobs: npm ci npm run deploy:feature:stack -- -c env=dotcomstg -c customFeatureName=enhancedApp-dotcomstg auto-builder-stack-enhancedApp-dotcomstg-webhooks npm run deploy:feature:stack -- -c env=stg -c customFeatureName=enhancedApp-stg auto-builder-stack-enhancedApp-stg-webhooks - diff --git a/.github/workflows/deploy-stg-enhanced-worker.yml b/.github/workflows/deploy-stg-enhanced-worker.yml index 4690585f6..24000195c 100644 --- a/.github/workflows/deploy-stg-enhanced-worker.yml +++ b/.github/workflows/deploy-stg-enhanced-worker.yml @@ -4,7 +4,6 @@ on: branches: - 'main' - 'integration' - concurrency: group: environment-stg-enhanced-worker-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/update-feature-branch.yml b/.github/workflows/update-feature-branch.yml index b896e751d..1f16579d8 100644 --- a/.github/workflows/update-feature-branch.yml +++ b/.github/workflows/update-feature-branch.yml @@ -61,6 +61,12 @@ jobs: node_modules cdk-infra/node_modules key: ${{ github.head_ref }} + - name: Install Dependencies + if: steps.cache-restore.outputs.cache-hit != 'true' + run: | + npm ci + cd cdk-infra/ + npm ci - uses: dorny/paths-filter@v2 id: filter with: @@ -96,6 +102,12 @@ jobs: node_modules cdk-infra/node_modules key: ${{ github.head_ref }} + - name: Install Dependencies + if: steps.cache-restore.outputs.cache-hit != 'true' + run: | + npm ci + cd cdk-infra/ + npm ci - uses: dorny/paths-filter@v2 id: filter with: diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts index a761e742b..c97823694 100644 --- a/api/controllers/v2/github.ts +++ b/api/controllers/v2/github.ts @@ -1,12 +1,13 @@ import * as c from 'config'; import * as mongodb from 'mongodb'; import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda'; -import { PushEvent } from '@octokit/webhooks-types'; +import { PushEvent, WorkflowRunCompletedEvent } from '@octokit/webhooks-types'; import { JobRepository } from '../../../src/repositories/jobRepository'; import { ConsoleLogger } from '../../../src/services/logger'; import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRepository'; -import { EnhancedJob, JobStatus } from '../../../src/entities/job'; +import { ProjectsRepository } from '../../../src/repositories/projectsRepository'; +import { EnhancedJob, EnhancedPayload, JobStatus } from '../../../src/entities/job'; import { markBuildArtifactsForDeletion, validateJsonWebhook } from '../../handlers/github'; import { DocsetsRepository } from '../../../src/repositories/docsetsRepository'; import { getMonorepoPaths } from '../../../src/monorepo'; @@ -14,26 +15,25 @@ import { getUpdatedFilePaths } from '../../../src/monorepo/utils/path-utils'; import { ReposBranchesDocsetsDocument } from '../../../modules/persistence/src/services/metadata/repos_branches'; import { MONOREPO_NAME } from '../../../src/monorepo/utils/monorepo-constants'; +const SMOKETEST_SITES = [ + 'docs-landing', + 'cloud-docs', + 'docs-realm', + 'docs', + 'docs-atlas-cli', + 'docs-node', + 'docs-app-services', +]; + +//EnhancedPayload and EnhancedJob are used here for both githubPush (feature branch) events as well as productionDeploy(smoke test deploy) events for typing purposes async function prepGithubPushPayload( - githubEvent: PushEvent, - repoBranchesRepository: RepoBranchesRepository, - prefix: string, - repoInfo: ReposBranchesDocsetsDocument, - directory?: string + githubEvent: PushEvent | WorkflowRunCompletedEvent, + payload: EnhancedPayload, + title: string ): Promise> { - const branch_name = githubEvent.ref.split('/')[2]; - const branch_info = await repoBranchesRepository.getRepoBranchAliases( - githubEvent.repository.name, - branch_name, - repoInfo.project - ); - const urlSlug = branch_info.aliasObject?.urlSlug ?? branch_name; - const project = repoInfo?.project ?? githubEvent.repository.name; - return { - title: githubEvent.repository.full_name, - user: githubEvent.pusher.name, - email: githubEvent.pusher.email ?? '', + title: title, + user: githubEvent.sender.login, status: JobStatus.inQueue, createdTime: new Date(), startTime: null, @@ -41,31 +41,85 @@ async function prepGithubPushPayload( priority: 1, error: {}, result: null, - payload: { - jobType: 'githubPush', - source: 'github', - action: 'push', - repoName: githubEvent.repository.name, - branchName: githubEvent.ref.split('/')[2], - isFork: githubEvent.repository.fork, - repoOwner: githubEvent.repository.owner.login, - url: githubEvent.repository.clone_url, - newHead: githubEvent.after, - urlSlug: urlSlug, - prefix: prefix, - project: project, - directory: directory, - }, + payload, logs: [], }; } -export const TriggerBuild = async (event: APIGatewayEvent): Promise => { +interface CreatePayloadProps { + repoName: string; + isSmokeTestDeploy?: boolean; + prefix: string; + repoBranchesRepository: RepoBranchesRepository; + repoInfo: ReposBranchesDocsetsDocument; + newHead?: string; + repoOwner?: string; + githubEvent?: PushEvent; + directory?: string; +} + +async function createPayload({ + repoName, + isSmokeTestDeploy = false, + prefix, + repoBranchesRepository, + repoInfo, + newHead, + repoOwner = '', + githubEvent, + directory, +}: CreatePayloadProps): Promise { + const source = 'github'; + const project = repoInfo?.project ?? repoName; + + let branchName: string; + let jobType: string; + let action: string; + let url: string; + + if (isSmokeTestDeploy) { + url = 'https://github.com/' + repoOwner + '/' + repoName; + action = 'automatedTest'; + jobType = 'productionDeploy'; + branchName = 'master'; + } else { + if (!githubEvent) { + throw new Error(`Non SmokeTest Deploy jobs must have a github Event`); + } + action = 'push'; + jobType = 'githubPush'; + branchName = githubEvent.ref.split('/')[2]; + url = githubEvent.repository?.clone_url; + newHead = githubEvent.after; + repoOwner = githubEvent.repository?.owner?.login; + } + + const branchInfo = await repoBranchesRepository.getRepoBranchAliases(repoName, branchName, repoInfo.project); + const urlSlug = branchInfo.aliasObject?.urlSlug ?? branchName; + + return { + jobType, + source, + action, + repoName, + repoOwner, + branchName, + project, + prefix, + urlSlug, + url, + newHead, + directory, + }; +} + +export const triggerSmokeTestAutomatedBuild = async (event: APIGatewayEvent): Promise => { const client = new mongodb.MongoClient(c.get('dbUrl')); await client.connect(); const db = client.db(c.get('dbName')); const consoleLogger = new ConsoleLogger(); const jobRepository = new JobRepository(db, c, consoleLogger); + const projectsRepository = new ProjectsRepository(client.db(process.env.METADATA_DB_NAME), c, consoleLogger); const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger); const docsetsRepository = new DocsetsRepository(db, c, consoleLogger); @@ -79,6 +133,132 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise('githubSecret'))) { + const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; + return { + statusCode: 401, + headers: { 'Content-Type': 'text/plain' }, + body: errMsg, + }; + } + + let body: WorkflowRunCompletedEvent; + try { + body = JSON.parse(event.body) as WorkflowRunCompletedEvent; + } catch (e) { + console.log('[TriggerBuild]: ERROR! Could not parse event.body', e); + return { + statusCode: 502, + headers: { 'Content-Type': 'text/plain' }, + body: ' ERROR! Could not parse event.body', + }; + } + + if (body.workflow_run.conclusion != 'success') + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: `Build on branch ${body.workflow_run.head_branch} is not complete and will not trigger smoke test site deployments`, + }; + + if (body.workflow_run.name != 'Deploy Staging ECS') + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: `Workflow ${body.workflow_run.name} completed successfully but only Deploy Staging ECS workflow completion will trigger smoke test site deployments`, + }; + + // if the build was not building main branch, no need for smoke test sites + if (body.workflow_run.head_branch != 'main' || body.repository.fork) { + console.log('Build was not on master branch in main repo, sites will not deploy as no smoke tests are needed'); + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: `Build on branch ${body.workflow_run.head_branch} will not trigger site deployments as it was not on main branch in upstream repo`, + }; + } + + //automated test builds will always deploy in dotcomstg + const env = 'dotcomstg'; + + async function createAndInsertJob() { + return await Promise.all( + SMOKETEST_SITES.map(async (repoName): Promise => { + const jobTitle = `Smoke Test ${repoName} site for commit ${body.workflow_run.head_sha} on docs-worker-pool main branch`; + let repoInfo, projectEntry, repoOwner; + try { + repoInfo = await docsetsRepository.getRepo(repoName); + projectEntry = await projectsRepository.getProjectEntry(repoInfo.project); + repoOwner = projectEntry.github.organization; + } catch (err) { + consoleLogger.error( + `Atlas Repo Information Error`, + `RepoInfo, projectEntry, or repoOwner not found for docs site ${repoName}. RepoInfo: ${repoInfo}, projectEntry: ${projectEntry}, repoOwner: ${repoOwner}` + ); + return err; + } + + const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; + const payload = await createPayload({ + repoName, + isSmokeTestDeploy: true, + prefix: jobPrefix, + repoBranchesRepository, + repoInfo, + repoOwner, + }); + + //add logic for getting master branch, latest stable branch + const job = await prepGithubPushPayload(body, payload, jobTitle); + + try { + consoleLogger.info(job.title, 'Creating Job'); + const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); + jobRepository.notify(jobId, c.get('jobUpdatesQueueUrl'), JobStatus.inQueue, 0); + consoleLogger.info(job.title, `Created Job ${jobId}`); + return jobId; + } catch (err) { + consoleLogger.error('TriggerBuildError', `${err} Error inserting job for ${repoName}`); + return err; + } + }) + ); + } + + try { + const returnVal = await createAndInsertJob(); + return { + statusCode: 202, + headers: { 'Content-Type': 'text/plain' }, + body: 'Smoke Test Jobs Queued with the following Job Ids ' + returnVal, + }; + } catch (err) { + return { + statusCode: 500, + headers: { 'Content-Type': 'text/plain' }, + body: err, + }; + } +}; + +export const TriggerBuild = async (event: APIGatewayEvent): Promise => { + const client = new mongodb.MongoClient(c.get('dbUrl')); + await client.connect(); + const db = client.db(c.get('dbName')); + const consoleLogger = new ConsoleLogger(); + const jobRepository = new JobRepository(db, c, consoleLogger); + const repoBranchesRepository = new RepoBranchesRepository(db, c, consoleLogger); + const docsetsRepository = new DocsetsRepository(db, c, consoleLogger); + + if (!event.body) { + const err = 'Trigger build does not have a body in event payload'; + return { + statusCode: 400, + headers: { 'Content-Type': 'text/plain' }, + body: err, + }; + } + if (!validateJsonWebhook(event, c.get('githubSecret'))) { const errMsg = "X-Hub-Signature incorrect. Github webhook token doesn't match"; return { @@ -110,9 +290,19 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise('env'); async function createAndInsertJob(path?: string) { - const repoInfo = await docsetsRepository.getRepo(body.repository.name, path); + const repo = body.repository; + const repoInfo = await docsetsRepository.getRepo(repo.name, path); const jobPrefix = repoInfo?.prefix ? repoInfo['prefix'][env] : ''; - const job = await prepGithubPushPayload(body, repoBranchesRepository, jobPrefix, repoInfo, path); + const jobTitle = repo.full_name; + const payload = await createPayload({ + repoName: repo.name, + prefix: jobPrefix, + repoBranchesRepository, + repoInfo, + githubEvent: body, + }); + + const job = await prepGithubPushPayload(body, payload, jobTitle); consoleLogger.info(job.title, 'Creating Job'); const jobId = await jobRepository.insertJob(job, c.get('jobsQueueUrl')); diff --git a/api/controllers/v2/slack.ts b/api/controllers/v2/slack.ts index 2cb5307bc..b6d76f74a 100644 --- a/api/controllers/v2/slack.ts +++ b/api/controllers/v2/slack.ts @@ -6,7 +6,7 @@ import { ConsoleLogger, ILogger } from '../../../src/services/logger'; import { SlackConnector } from '../../../src/services/slack'; import { JobRepository } from '../../../src/repositories/jobRepository'; import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda'; -import { JobStatus } from '../../../src/entities/job'; +import { EnhancedPayload, JobStatus } from '../../../src/entities/job'; import { buildEntitledBranchList, getQSString, @@ -256,7 +256,7 @@ function createPayload( }; } -function createJob(payload: any, jobTitle: string, jobUserName: string, jobUserEmail: string) { +function createJob(payload: EnhancedPayload, jobTitle: string, jobUserName: string, jobUserEmail: string) { return { title: jobTitle, user: jobUserName, diff --git a/cdk-infra/lib/constructs/api/webhook-api-construct.ts b/cdk-infra/lib/constructs/api/webhook-api-construct.ts index 52c71b95f..10862f6b8 100644 --- a/cdk-infra/lib/constructs/api/webhook-api-construct.ts +++ b/cdk-infra/lib/constructs/api/webhook-api-construct.ts @@ -74,6 +74,15 @@ export class WebhookApiConstruct extends Construct { timeout, }); + const githubSmokeTestBuildLambda = new NodejsFunction(this, 'githubSmokeTestBuildLambda', { + entry: `${HANDLERS_PATH}/github.ts`, + runtime, + handler: 'triggerSmokeTestAutomatedBuild', + bundling, + environment, + timeout, + }); + const githubDeleteArtifactsLambda = new NodejsFunction(this, 'githubDeleteArtifactsLambda', { entry: `${HANDLERS_PATH}/github.ts`, runtime, @@ -160,6 +169,11 @@ export class WebhookApiConstruct extends Construct { .addResource('build', { defaultCorsPreflightOptions }) .addMethod('POST', new LambdaIntegration(githubTriggerLambda)); + // add endpoint for automated testing + githubEndpointTrigger + .addResource('smoke-test-build', { defaultCorsPreflightOptions }) + .addMethod('POST', new LambdaIntegration(githubSmokeTestBuildLambda)); + githubEndpointTrigger .addResource('delete', { defaultCorsPreflightOptions }) .addMethod('POST', new LambdaIntegration(githubDeleteArtifactsLambda)); @@ -176,11 +190,13 @@ export class WebhookApiConstruct extends Construct { // grant permission for lambdas to enqueue messages to the jobs queue jobsQueue.grantSendMessages(slackTriggerLambda); + jobsQueue.grantSendMessages(githubSmokeTestBuildLambda); jobsQueue.grantSendMessages(githubTriggerLambda); jobsQueue.grantSendMessages(triggerLocalBuildLambda); // grant permission for lambdas to enqueue messages to the job updates queue jobUpdatesQueue.grantSendMessages(slackTriggerLambda); + jobUpdatesQueue.grantSendMessages(githubSmokeTestBuildLambda); jobUpdatesQueue.grantSendMessages(githubTriggerLambda); jobUpdatesQueue.grantSendMessages(triggerLocalBuildLambda); diff --git a/cdk-infra/lib/constructs/api/webhook-env-construct.ts b/cdk-infra/lib/constructs/api/webhook-env-construct.ts index 800b9277d..0c53cb4a7 100644 --- a/cdk-infra/lib/constructs/api/webhook-env-construct.ts +++ b/cdk-infra/lib/constructs/api/webhook-env-construct.ts @@ -40,8 +40,10 @@ export class WebhookEnvConstruct extends Construct { MONGO_ATLAS_URL: `mongodb+srv://${dbUsername}:${dbPassword}@${dbHost}/admin?retryWrites=true`, DB_NAME: dbName, SNOOTY_DB_NAME: snootyDbName, + METADATA_DB_NAME: 'docs_metadata', REPO_BRANCHES_COL_NAME: repoBranchesCollection, DOCSETS_COL_NAME: docsetsCollection, + PROJECTS_COL_NAME: 'projects', JOB_QUEUE_COL_NAME: jobCollection, NODE_CONFIG_DIR: './config', JOBS_QUEUE_URL: jobsQueue.queueUrl, diff --git a/cdk-infra/lib/constructs/worker/worker-construct.ts b/cdk-infra/lib/constructs/worker/worker-construct.ts index 8c00607dc..63f550305 100644 --- a/cdk-infra/lib/constructs/worker/worker-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-construct.ts @@ -39,7 +39,7 @@ export class WorkerConstruct extends Construct { const taskRoleSsmPolicyStatement = new PolicyStatement({ effect: Effect.ALLOW, - actions: ['ssm:GetParameter'], + actions: ['ssm:GetParameter', 'ssm:PutParameter'], resources: ['*'], }); diff --git a/cdk-infra/lib/constructs/worker/worker-env-construct.ts b/cdk-infra/lib/constructs/worker/worker-env-construct.ts index ff3e1f8e4..de91138ad 100644 --- a/cdk-infra/lib/constructs/worker/worker-env-construct.ts +++ b/cdk-infra/lib/constructs/worker/worker-env-construct.ts @@ -77,6 +77,7 @@ export class WorkerEnvConstruct extends Construct { MONGO_ATLAS_URL: `mongodb+srv://${dbUsername}:${dbPassword}@${dbHost}/admin?retryWrites=true`, DB_NAME: dbName, SNOOTY_DB_NAME: snootyDbName, + METADATA_DB_NAME: 'docs_metadata', JOBS_QUEUE_URL: jobsQueue.queueUrl, JOB_UPDATES_QUEUE_URL: jobUpdatesQueue.queueUrl, GITHUB_BOT_USERNAME: githubBotUsername, @@ -89,6 +90,7 @@ export class WorkerEnvConstruct extends Construct { REPO_BRANCHES_COL_NAME: repoBranchesCollection, DOCSETS_COL_NAME: docsetsCollection, JOB_QUEUE_COL_NAME: jobCollection, + PROJECTS_COL_NAME: 'projects', CDN_INVALIDATOR_SERVICE_URL: getCdnInvalidatorUrl(env), SEARCH_INDEX_BUCKET: 'docs-search-indexes-test', SEARCH_INDEX_FOLDER: getSearchIndexFolder(env), diff --git a/src/commands/src/shared/next-gen-deploy.ts b/src/commands/src/shared/next-gen-deploy.ts index 5d86baf73..d41cac516 100644 --- a/src/commands/src/shared/next-gen-deploy.ts +++ b/src/commands/src/shared/next-gen-deploy.ts @@ -48,7 +48,7 @@ export async function nextGenDeploy({ console.log( `COMMAND: yes | mut-publish public ${bucket} --prefix=${mutPrefix} --deploy --deployed-url-prefix=${url} --json --all-subdirectories --dry-run` ); - console.log(`${outputText}\n Hosted at ${url}/${mutPrefix}`); + console.log(`${outputText}\n Hosted at ${url}${mutPrefix}`); return { status: CommandExecutorResponseStatus.success, output: outputText, diff --git a/src/entities/job.ts b/src/entities/job.ts index eaceaba59..fcbbd5233 100644 --- a/src/entities/job.ts +++ b/src/entities/job.ts @@ -64,7 +64,7 @@ export type EnhancedPayload = { action: string; repoName: string; branchName: string; - isFork: boolean; + isFork?: boolean; isXlarge?: boolean | null; repoOwner: string; url: string; @@ -131,7 +131,7 @@ export type EnhancedJob = { buildCommands?: string[]; deployCommands?: string[]; invalidationStatusURL?: string | null; - email: string | null; // probably can be removed + email?: string | null; // probably can be removed comMessage?: string[] | null; purgedUrls?: string[] | null; shouldGenerateSearchManifest?: boolean; diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts index bf6548933..f0f8743cb 100644 --- a/src/job/jobHandler.ts +++ b/src/job/jobHandler.ts @@ -149,7 +149,7 @@ export abstract class JobHandler { @throwIfJobInterupted() private async constructPrefix(): Promise { const server_user = this._config.get('GATSBY_PARSER_USER'); - const pathPrefix = await this.getPathPrefix(); + const pathPrefix = this.getPathPrefix(); // TODO: Can empty string check be removed? if (pathPrefix || pathPrefix === '') { this.currJob.payload.pathPrefix = pathPrefix; diff --git a/src/job/productionJobHandler.ts b/src/job/productionJobHandler.ts index a7ed1ac28..6190cc018 100644 --- a/src/job/productionJobHandler.ts +++ b/src/job/productionJobHandler.ts @@ -181,7 +181,7 @@ export class ProductionJobHandler extends JobHandler { await this.jobRepository.insertInvalidationRequestStatusUrl(this.currJob._id, 'Invalidation Failed'); } } catch (error) { - await this.logger.save(this.currJob._id, error); + await this.logger.save(this.currJob._id, error.message); } } diff --git a/src/repositories/baseRepository.ts b/src/repositories/baseRepository.ts index 750a1f83f..564e9b268 100644 --- a/src/repositories/baseRepository.ts +++ b/src/repositories/baseRepository.ts @@ -71,7 +71,11 @@ export abstract class BaseRepository { ); } - protected async findOne(query: any, errorMsg: string, options: mongodb.FindOptions = {}): Promise { + protected async findOne( + query: any, + errorMsg: string, + options: mongodb.FindOptions = {} + ): Promise | null> { try { return await this.promiseTimeoutS( this._config.get('MONGO_TIMEOUT_S'), diff --git a/src/repositories/jobRepository.ts b/src/repositories/jobRepository.ts index 7712f15b3..d2e27da19 100644 --- a/src/repositories/jobRepository.ts +++ b/src/repositories/jobRepository.ts @@ -79,7 +79,7 @@ export class JobRepository extends BaseRepository { return jobIds; } - async getJobById(id: string): Promise { + async getJobById(id: string): Promise { const query = { _id: new objectId(id), }; diff --git a/src/repositories/projectsRepository.ts b/src/repositories/projectsRepository.ts new file mode 100644 index 000000000..08ebb4e19 --- /dev/null +++ b/src/repositories/projectsRepository.ts @@ -0,0 +1,22 @@ +import mongodb from 'mongodb'; +import { IConfig } from 'config'; +import { BaseRepository } from './baseRepository'; +import { ILogger } from '../services/logger'; + +//Project information from docs_metadata.projects for parser builds. + +export class ProjectsRepository extends BaseRepository { + constructor(db: mongodb.Db, config: IConfig, logger: ILogger) { + super(config, logger, 'ProjectsRepository', db.collection(process.env.PROJECTS_COL_NAME || '')); + } + + async getProjectEntry(name: string): Promise | null> { + const query = { name: name }; + const projectEntry = await this.findOne( + query, + `Mongo Timeout Error: Timedout while retrieving branches for ${name} + }` + ); + return projectEntry; + } +} diff --git a/src/repositories/repoEntitlementsRepository.ts b/src/repositories/repoEntitlementsRepository.ts index f9cdcde26..2e73f8623 100644 --- a/src/repositories/repoEntitlementsRepository.ts +++ b/src/repositories/repoEntitlementsRepository.ts @@ -73,7 +73,7 @@ export class RepoEntitlementsRepository extends BaseRepository { `Mongo Timeout Error: Timedout while retrieving entitlements for ${slackUserId}` ); // if user has specific entitlements - if ((entitlementsObject?.repos?.length ?? 0) > 0) { + if (entitlementsObject?.repos?.length) { return { repos: entitlementsObject.repos, github_username: entitlementsObject.github_username, From 1a2ff581eed2428c038d8f1fb7a7fa7dc84090c5 Mon Sep 17 00:00:00 2001 From: anabellabuckvar <41971124+anabellabuckvar@users.noreply.github.com> Date: Fri, 12 Apr 2024 12:35:56 -0400 Subject: [PATCH 4/4] DOP-4481: Test-deploy all docs site button (#1022) * DOP-4490 scrappy changes * DOP-4490 slack changes * DOP-4490 deploy to preprd * DOP-4490 fixing nits * DOP-4490 fixing find options * DOP-4490-b new branch because of cdk error * DOP-4490-b commenting out * DOP-4490-b trying to put section back in * DOP-4490-b trying to put button back in * DOP-4490-b button back in * DOP-4490-b value out * DOP-4490-b value in * DOP-4490-b return value early * DOP-4490-b removing entire button section * DOP-4490-b removing entire button section * DOP-4490-b returning early * DOP-4490-b adding logging * DOP-4490-b return empty deploy repo * DOP-4490-b more logging * DOP-4490-b commenting out * DOP-4490-b logging values * DOP-4490-b add await * DOP-4490-b add button back in * DOP-4490-b add confirm modal * DOP-4490-b remove confirm modal * DOP-4490-b return from deployRepo early * DOP-4490-b return from deployRepo early, but later than before * DOP-4490-b return from deployRepo early, but later than before * DOP-4490-b return from deployRepo earlier * DOP-4490-b test conditional rendering of deploy button * DOP-4490-b test conditional rendering of deploy button * DOP-4490-b test conditional rendering if user is admin * DOP-4490-b test conditional rendering deploy button * DOP-4490-b test conditional rendering deploy button * DOP-4490-b parsing button response * DOP-4490-b fixing admin function * DOP-4490-b parsing button payload * DOP-4490-b parsing button payload * DOP-4490-b logging parsed values * DOP-4490-b logging parsed values * DOP-4490-b logging parsed values * DOP-4490-b logging parsed values * DOP-4490-b logging parsed values * DOP-4490-b changing button block type * DOP-4490-b changing button block type back * DOP-4490-b changing button block type to actions * DOP-4490-b changing button block type to actions * DOP-4490-b changing button block type to actions * DOP-4490-b testing radio button * DOP-4490-b updating radio button * DOP-4490-b prod deployable repo branches * DOP-4490-b parse selection correctly * DOP-4490-b logging getProdDeployableRepoBranches * DOP-4490-b logging getProdDeployableRepoBranches * DOP-4490-b undoing recent changes * DOP-4490-b readding some changes * DOP-4490-b undoing last change * DOP-4490-b changing parse function * DOP-4490-b adding back in * DOP-4490-b fix typo * DOP-4490-b cleaning up * DOP-4490-b logging type * DOP-4490-b logging type again * DOP-4490-b remove from preprd * DOP-4490-b repo selection optional * DOP-4490-b repo selection optional checks * DOP-4490-b redeploy to preprd * DOP-4490-b fix error catching * DOP-4490-b fix error catching * DOP-4490-b remove from preprd * DOP-4490-b nits * DOP-4490-b PR review nits * DOP-4490-b PR review nits * DOP-4490-b nits * DOP-4490-b change how errors are thrown * DOP-4490-b fix try catch for parsing selection --- api/controllers/v1/slack.ts | 85 +++++++++++------ api/controllers/v2/slack.ts | 15 ++- src/job/stagingJobHandler.ts | 2 +- src/repositories/repoBranchesRepository.ts | 9 +- .../repoEntitlementsRepository.ts | 9 ++ src/services/slack.ts | 92 ++++++++++++++++--- 6 files changed, 165 insertions(+), 47 deletions(-) diff --git a/api/controllers/v1/slack.ts b/api/controllers/v1/slack.ts index 6f208b7e2..45484e78a 100644 --- a/api/controllers/v1/slack.ts +++ b/api/controllers/v1/slack.ts @@ -4,6 +4,7 @@ import { RepoEntitlementsRepository } from '../../../src/repositories/repoEntitl import { RepoBranchesRepository } from '../../../src/repositories/repoBranchesRepository'; import { ConsoleLogger, ILogger } from '../../../src/services/logger'; import { SlackConnector } from '../../../src/services/slack'; +import { APIGatewayEvent, APIGatewayProxyResult } from 'aws-lambda'; import { JobRepository } from '../../../src/repositories/jobRepository'; import { buildEntitledBranchList, @@ -13,13 +14,23 @@ import { prepResponse, } from '../../handlers/slack'; import { DocsetsRepository } from '../../../src/repositories/docsetsRepository'; +import { Payload } from '../../../src/entities/job'; -export const DisplayRepoOptions = async (event: any = {}, context: any = {}): Promise => { +export const DisplayRepoOptions = async (event: APIGatewayEvent): Promise => { const consoleLogger = new ConsoleLogger(); const slackConnector = new SlackConnector(consoleLogger, c); + if (!slackConnector.validateSlackRequest(event)) { return prepResponse(401, 'text/plain', 'Signature Mismatch, Authentication Failed!'); } + + if (!event.body) { + return { + statusCode: 400, + body: 'Event body is undefined', + }; + } + const client = new mongodb.MongoClient(c.get('dbUrl')); await client.connect(); const db = client.db(process.env.DB_NAME); @@ -34,8 +45,11 @@ export const DisplayRepoOptions = async (event: any = {}, context: any = {}): Pr : 'User is not entitled!'; return prepResponse(401, 'text/plain', response); } + + const isAdmin = await repoEntitlementRepository.getIsAdmin(key_val['user_id']); + const entitledBranches = await buildEntitledBranchList(entitlement, repoBranchesRepository); - const resp = await slackConnector.displayRepoOptions(entitledBranches, key_val['trigger_id']); + const resp = await slackConnector.displayRepoOptions(entitledBranches, key_val['trigger_id'], isAdmin); if (resp?.status == 200 && resp?.data) { return { statusCode: 200, @@ -71,22 +85,29 @@ export const getDeployableJobs = async ( ) => { const deployable = []; - for (let i = 0; i < values.repo_option.length; i++) { - let repoOwner: string, repoName: string, branchName: string, directory: string | undefined; - const splitValues = values.repo_option[i].value.split('/'); - - if (splitValues.length === 3) { - // e.g. mongodb/docs-realm/master => (owner/repo/branch) - [repoOwner, repoName, branchName] = splitValues; - } else if (splitValues.length === 4 && process.env.FEATURE_FLAG_MONOREPO_PATH === 'true') { - // e.g. 10gen/docs-monorepo/cloud-docs/master => (owner/monorepo/repoDirectory/branch) - [repoOwner, repoName, directory, branchName] = splitValues; + for (let i = 0; i < values?.repo_option?.length; i++) { + let jobTitle: string, repoOwner: string, repoName: string, branchName: string, directory: string | undefined; + if (values.deploy_option == 'deploy_all') { + repoOwner = 'mongodb'; + branchName = 'master'; + repoName = values.repo_option[i].repoName; + jobTitle = `Slack deploy: ${repoOwner}/${repoName}/${branchName}, by ${entitlement.github_username}`; } else { - throw Error('Selected entitlement value is configured incorrectly. Check user entitlements!'); + const splitValues = values.repo_option[i].value.split('/'); + jobTitle = `Slack deploy: ${values.repo_option[i].value}, by ${entitlement.github_username}`; + + if (splitValues.length === 3) { + // e.g. mongodb/docs-realm/master => (owner/repo/branch) + [repoOwner, repoName, branchName] = splitValues; + } else if (splitValues.length === 4 && process.env.FEATURE_FLAG_MONOREPO_PATH === 'true') { + // e.g. 10gen/docs-monorepo/cloud-docs/master => (owner/monorepo/repoDirectory/branch) + [repoOwner, repoName, directory, branchName] = splitValues; + } else { + throw Error('Selected entitlement value is configured incorrectly. Check user entitlements!'); + } } const hashOption = values?.hash_option ?? null; - const jobTitle = `Slack deploy: ${values.repo_option[i].value}, by ${entitlement.github_username}`; const jobUserName = entitlement.github_username; const jobUserEmail = entitlement?.email ?? ''; @@ -96,12 +117,11 @@ export const getDeployableJobs = async ( const branchObject = await repoBranchesRepository.getRepoBranchAliases(repoName, branchName, repoInfo.project); if (!branchObject?.aliasObject) continue; - // TODO: Create strong typing for these rather than comments - const publishOriginalBranchName = branchObject.aliasObject.publishOriginalBranchName; // bool - let aliases = branchObject.aliasObject.urlAliases; // array or null - let urlSlug = branchObject.aliasObject.urlSlug; // string or null, string must match value in urlAliases or gitBranchName - const isStableBranch = branchObject.aliasObject.isStableBranch; // bool or Falsey - aliases = aliases?.filter((a) => a); + const publishOriginalBranchName: boolean = branchObject.aliasObject.publishOriginalBranchName; + const aliases: string[] | null = branchObject.aliasObject.urlAliases; + let urlSlug: string = branchObject.aliasObject.urlSlug; // string or null, string must match value in urlAliases or gitBranchName + const isStableBranch = !!branchObject.aliasObject.isStableBranch; // bool or Falsey, add strong typing + if (!urlSlug || !urlSlug.trim()) { urlSlug = branchName; } @@ -118,12 +138,10 @@ export const getDeployableJobs = async ( urlSlug, false, false, - false, + isStableBranch, directory ); - newPayload.stable = !!isStableBranch; - if (!aliases || aliases.length === 0) { if (non_versioned) { newPayload.urlSlug = ''; @@ -164,7 +182,7 @@ export const getDeployableJobs = async ( return deployable; }; -export const DeployRepo = async (event: any = {}, context: any = {}): Promise => { +export const DeployRepo = async (event: any = {}): Promise => { const consoleLogger = new ConsoleLogger(); const slackConnector = new SlackConnector(consoleLogger, c); if (!slackConnector.validateSlackRequest(event)) { @@ -173,6 +191,7 @@ export const DeployRepo = async (event: any = {}, context: any = {}): Promise 0) { await deployRepo(deployable, consoleLogger, jobRepository, c.get('jobsQueueUrl')); } @@ -235,7 +266,7 @@ function createPayload( }; } -function createJob(payload: any, jobTitle: string, jobUserName: string, jobUserEmail: string) { +function createJob(payload: Payload, jobTitle: string, jobUserName: string, jobUserEmail: string) { return { title: jobTitle, user: jobUserName, diff --git a/api/controllers/v2/slack.ts b/api/controllers/v2/slack.ts index b6d76f74a..fb3ef632f 100644 --- a/api/controllers/v2/slack.ts +++ b/api/controllers/v2/slack.ts @@ -20,6 +20,10 @@ export const DisplayRepoOptions = async (event: APIGatewayEvent): Promise 0) { diff --git a/src/job/stagingJobHandler.ts b/src/job/stagingJobHandler.ts index e543da9fb..df1d772b4 100644 --- a/src/job/stagingJobHandler.ts +++ b/src/job/stagingJobHandler.ts @@ -49,7 +49,7 @@ export class StagingJobHandler extends JobHandler { prepDeployCommands(): void { this.currJob.deployCommands = [ `cd repos/${getDirectory(this.currJob)}`, - `make next-gen-stage${this.currJob.payload.pathPrefix ? `MUT_PREFIX=${this.currJob.payload.mutPrefix}` : ''}`, + `make next-gen-stage${this.currJob.payload.pathPrefix ? ` MUT_PREFIX=${this.currJob.payload.mutPrefix}` : ''}`, ]; } diff --git a/src/repositories/repoBranchesRepository.ts b/src/repositories/repoBranchesRepository.ts index f2764c3ef..20bce0aeb 100644 --- a/src/repositories/repoBranchesRepository.ts +++ b/src/repositories/repoBranchesRepository.ts @@ -1,4 +1,4 @@ -import { Db } from 'mongodb'; +import { Document, Db } from 'mongodb'; import { BaseRepository } from './baseRepository'; import { ILogger } from '../services/logger'; import { IConfig } from 'config'; @@ -32,6 +32,13 @@ export class RepoBranchesRepository extends BaseRepository { return repo?.['branches'] ?? []; } + async getProdDeployableRepoBranches(): Promise { + const reposArray = await this._collection + .aggregate([{ $match: { prodDeployable: true, internalOnly: false } }, { $project: { _id: 0, repoName: 1 } }]) + .toArray(); + return reposArray ?? []; + } + async getRepoBranchAliases(repoName: string, branchName: string, project: string): Promise { const returnObject = { status: 'failure' }; const aliasArray = await this._collection diff --git a/src/repositories/repoEntitlementsRepository.ts b/src/repositories/repoEntitlementsRepository.ts index 2e73f8623..7cb5c8aaa 100644 --- a/src/repositories/repoEntitlementsRepository.ts +++ b/src/repositories/repoEntitlementsRepository.ts @@ -47,6 +47,15 @@ export class RepoEntitlementsRepository extends BaseRepository { } } + async getIsAdmin(slackUserId: string): Promise { + const query = { slack_user_id: slackUserId }; + const entitlementsObject = await this.findOne( + query, + `Mongo Timeout Error: Timedout while retrieving entitlements for ${slackUserId}` + ); + return entitlementsObject?.admin; + } + async getGatsbySiteIdByGithubUsername(githubUsername: string): Promise { return this.getBuildHookByGithubUsername(githubUsername, 'gatsby_site_id'); } diff --git a/src/services/slack.ts b/src/services/slack.ts index 4370a8500..20803d268 100644 --- a/src/services/slack.ts +++ b/src/services/slack.ts @@ -2,6 +2,7 @@ import axios from 'axios'; import { ILogger } from './logger'; import { IConfig } from 'config'; import * as crypto from 'crypto'; +import { RepoBranchesRepository } from '../repositories/repoBranchesRepository'; export const axiosApi = axios.create(); function bufferEqual(a: Buffer, b: Buffer) { @@ -22,8 +23,8 @@ function timeSafeCompare(a: string, b: string) { export interface ISlackConnector { validateSlackRequest(payload: any): boolean; - displayRepoOptions(repos: Array, triggerId: string): Promise; - parseSelection(payload: any): any; + displayRepoOptions(repos: Array, triggerId: string, isAdmin: boolean): Promise; + parseSelection(payload: any, isAdmin: boolean, repoBranchesRepository: RepoBranchesRepository): any; sendMessage(message: any, user: string): Promise; } @@ -52,25 +53,44 @@ export class SlackConnector implements ISlackConnector { } return {}; } - parseSelection(stateValues: any): any { + + async parseSelection( + stateValues: any, + isAdmin: boolean, + repoBranchesRepository: RepoBranchesRepository + ): Promise { const values = {}; const inputMapping = { block_repo_option: 'repo_option', block_hash_option: 'hash_option', + block_deploy_option: 'deploy_option', }; + // if deploy all was selected: + if (stateValues['block_deploy_option']['deploy_option']?.selected_option?.value == 'deploy_all') { + if (!isAdmin) { + throw new Error('User is not an admin and therefore not entitled to deploy all repos'); + } + + values['deploy_option'] = 'deploy_all'; + values['repo_option'] = await repoBranchesRepository.getProdDeployableRepoBranches(); + return values; + } + + //if deploy indivual repos was selected: // get key and values to figure out what user wants to deploy for (const blockKey in inputMapping) { const blockInputKey = inputMapping[blockKey]; const stateValuesObj = stateValues[blockKey][blockInputKey]; - // selected value from dropdown - if (stateValuesObj?.selected_option?.value) { - values[blockInputKey] = stateValuesObj.selected_option.value; - } + // multi select is an array - else if (stateValuesObj?.selected_options && stateValuesObj.selected_options.length > 0) { + if (stateValuesObj?.selected_options?.length > 0) { values[blockInputKey] = stateValuesObj.selected_options; } + //return an error if radio button choice 'deploy individual repos' was selected but no repo was actually chosen + else if (blockInputKey == 'repo_option') { + throw new Error('Deploy individual repos was selected but no repo was actually chosen to be deployed'); + } // input value else if (stateValuesObj?.value) { values[blockInputKey] = stateValuesObj.value; @@ -99,8 +119,9 @@ export class SlackConnector implements ISlackConnector { return false; } - async displayRepoOptions(repos: string[], triggerId: string): Promise { - const repoOptView = this._buildDropdown(repos, triggerId); + async displayRepoOptions(repos: string[], triggerId: string, isAdmin: boolean): Promise { + const reposToShow = this._buildDropdown(repos); + const repoOptView = this._getDropDownView(triggerId, reposToShow, isAdmin); const slackToken = this._config.get('slackAuthToken'); const slackUrl = this._config.get('slackViewOpenUrl'); return await axiosApi.post(slackUrl, repoOptView, { @@ -110,8 +131,51 @@ export class SlackConnector implements ISlackConnector { }, }); } + private _getDropDownView(triggerId: string, repos: Array, isAdmin: boolean) { + const deployAll = isAdmin + ? { + type: 'section', + block_id: 'block_deploy_option', + text: { + type: 'plain_text', + text: 'How would you like to deploy docs sites?', + }, + accessory: { + type: 'radio_buttons', + action_id: 'deploy_option', + initial_option: { + value: 'deploy_individually', + text: { + type: 'plain_text', + text: 'Deploy individual repos', + }, + }, + options: [ + { + value: 'deploy_individually', + text: { + type: 'plain_text', + text: 'Deploy individual repos', + }, + }, + { + value: 'deploy_all', + text: { + type: 'plain_text', + text: 'Deploy all repos', + }, + }, + ], + }, + } + : { + type: 'section', + text: { + type: 'plain_text', + text: ' ', + }, + }; - private _getDropDownView(triggerId: string, repos: Array) { return { trigger_id: triggerId, view: { @@ -145,6 +209,7 @@ export class SlackConnector implements ISlackConnector { }, options: repos, }, + optional: true, label: { type: 'plain_text', text: 'Select Repo', @@ -168,12 +233,13 @@ export class SlackConnector implements ISlackConnector { text: 'Commit Hash', }, }, + deployAll, ], }, }; } - private _buildDropdown(branches: Array, triggerId: string): any { + private _buildDropdown(branches: Array): Array { let reposToShow: Array = []; branches.forEach((fullPath) => { const displayBranchPath = fullPath; @@ -208,6 +274,6 @@ export class SlackConnector implements ISlackConnector { .localeCompare(a.text.text.toString().replace(/\d+/g, (n) => +n + 100000)); }); - return this._getDropDownView(triggerId, reposToShow); + return reposToShow; } }