DOP-4209 (#958)

* experiments, added nextGenParse call to jobHandler * stage ecs - no makefiles * alter snooty version * next gen html * command key * parse * catch * logger * remove error * more logging * cwd * repoDir correct * quotations * successful parse? odd address * oas page build * xlarge * persistence module * quote * no parse * try catch parse * type error * no parse, use persistence and build * log errors * localApp working * remove /dist * add build deps & nextgenhtml * add logging to build deps * log html * remove build deps for envs * env vars * build deps before executeBuild * staging monorepo jobs * comment * localApp type cleanup * stage * deploy * clean fs * status * no event body * throw new error * commented out all unnecessary steps * explicitly call build steps * remove steps that should not be used * include job inqueue for error * log status * Empty-Commit * comment * used process.env to find URL and BUCKET * save localApp * uncomment prepNextGenBuild * remove throw error * not build on preprd * buildCommands array * log publish * use normal deploy * deploy * add cp commands * cd snooty * log outputs * ref repoDir rather than cwd * correct reposDir vs repoDir * add slash * add repos to prodFileName * log build deps * change prodFIleName to cwd/snooty * run parse * remove deploy * override build in stagingJobHandler * override build, set up debugger * vscode launch * remove dist * remove dist * use project * readd build deps * add cp and cd commands to nextGenHtml * cp correct paths * remove first cp and cd * remove cp * ref snooty filepath correctly * dockerfile snooty branch mm-log * remove first cp * cd .. * commands * stringify, logs * use chdir * cp second * log in clicommand * pass logger to mut publish * dotcomstg -> stg * checkout and pull branch * add pull repo and change clone * commented out incorrect code * local run works for both cloud-docs and monorepo/cloud-docs * log event and boyd * log out trigger build * log repo name and feat flag * feature flag * ssmprefix * env * dist * remove feature flag for feature branch build * log why no paths * change slash of path * clean * organize code * clean * redoc * fixes from merge * comment out builddeps, use redoc rc * duplicate clone * clean, get bucket and url * log env vars * pass logger to getEnvVar * takeover preprd * add to v1? * log which build * force directory * add directory to debug command and local build * remove from preprd * clean up * keep conditionals for buildCommands in normal build * further cleaning * remove logs * number of logs * curl into repoDir/targetDir * try new flow of logging * remove comments * allow output and error text to be returned from nextGenStage * clean logs * revert targetDir for downloadBuildDependencies * clean, wrapWithBenchmark * PR feedback * [DOP-4127]: Update dockerfile.local to have redoc installed properly * [DOP-4127]: Use new SQS queue URL * [DOP-4127]: Install redoc bundle * [DOP-4127]: Revert how redoc is installed * PR feedback, second round * replace useWithBenchmarks with isNextGen * source patchId from getBuildAndGetDependencies * [DOP-4204]: Update README for local Autobuilder (#954) * [DOP-4204]: Add help command * [DOP-4204]: Typo * [DOP-4204]: Formatting * [DOP-4024]: Update README.md * [DOP-4204]: Troubleshooting * [DOP-4204]: Fix typos * [DOP-4204]: Rename images and move into shared folder * [DOP-4204]: Update troubleshooting * added logging and error throwing in wrapWithBenchmark * conditionally write patchId and commitHash env vars * seeing if we have build dependencies * adding await * log * strringify * adding monorepo handling * snooty toml * logging * Adding slash * adding timeout to curl * adding logging * finally * bruh * L * oh well * adding repodir * adding monorepo support * log * cwd * adding to both curl and mkdir * adding logging * testing with executeclicommand not throwing * avoiding merge conflict * cleaning up * restructuring * [DOP-4269]: Refactor error handling * [DOP-4269]: Add text to error * consolidating * fixing merge stuff in readme * merge * async in dep helper * error message * nvm removing error msg * trying * seems like the best way for await idk * one more attempt * Nvm * not working * trying this out * nvm --------- Co-authored-by: Matt Meigs <[email protected]> Co-authored-by: branberry <[email protected]>
mongodb · Jan 22, 2024 · 834e691 · 834e691
1 parent effa134
commit 834e691
Show file tree

Hide file tree

Showing 8 changed files with 59 additions and 64 deletions.
diff --git a/Dockerfile.enhanced b/Dockerfile.enhanced
@@ -117,4 +117,4 @@ ENV OAS_MODULE_PATH=${WORK_DIRECTORY}/modules/oas-page-builder/index.js
 
 RUN mkdir repos && chmod 755 repos
 EXPOSE 3000
-CMD ["node", "enhanced/enhancedApp.js"]
+CMD ["node", "--enable-source-maps", "enhanced/enhancedApp.js"]
diff --git a/api/controllers/v2/github.ts b/api/controllers/v2/github.ts
@@ -138,6 +138,7 @@ export const TriggerBuild = async (event: APIGatewayEvent): Promise<APIGatewayPr
 
     /* Create and insert Job for each monorepo project that has changes */
     for (const path of monorepoPaths) {
+      consoleLogger.info(body.repository.full_name, `Create Job for Monorepo directory: /${path}`);
       // TODO: Deal with nested monorepo projects
       /* For now, we will ignore nested monorepo projects until necessary */
       if (path.split('/').length > 1) continue;

diff --git a/src/commands/index.ts b/src/commands/index.ts
@@ -1,17 +1,9 @@
-import { prepareBuildAndGetDependencies } from './src/helpers/dependency-helpers';
+import { prepareBuild } from './src/helpers/dependency-helpers';
 import { nextGenDeploy } from './src/shared/next-gen-deploy';
 import { nextGenHtml } from './src/shared/next-gen-html';
 import { nextGenParse } from './src/shared/next-gen-parse';
 import { nextGenStage } from './src/shared/next-gen-stage';
 import { oasPageBuild } from './src/shared/oas-page-build';
 import { persistenceModule } from './src/shared/persistence-module';
 
-export {
-  nextGenParse,
-  nextGenHtml,
-  nextGenStage,
-  persistenceModule,
-  oasPageBuild,
-  nextGenDeploy,
-  prepareBuildAndGetDependencies,
-};
+export { nextGenParse, nextGenHtml, nextGenStage, persistenceModule, oasPageBuild, nextGenDeploy, prepareBuild };
diff --git a/src/commands/src/helpers/dependency-helpers.ts b/src/commands/src/helpers/dependency-helpers.ts
@@ -43,16 +43,25 @@ async function createEnvProdFile({
   }
 }
 
-export async function downloadBuildDependencies(buildDependencies: BuildDependencies, repoName: string) {
+export async function downloadBuildDependencies(
+  buildDependencies: BuildDependencies,
+  repoName: string,
+  directory?: string
+) {
   const commands: string[] = [];
   await Promise.all(
     buildDependencies.map(async (dependencyInfo) => {
-      const repoDir = getRepoDir(repoName);
+      const repoDir = getRepoDir(repoName, directory);
       const targetDir = dependencyInfo.targetDir ?? repoDir;
+      let options = {};
+      if (targetDir != repoDir) {
+        options = { cwd: repoDir };
+      }
       try {
         await executeCliCommand({
           command: 'mkdir',
           args: ['-p', targetDir],
+          options: options,
         });
       } catch (error) {
         console.error(
@@ -63,36 +72,29 @@ export async function downloadBuildDependencies(buildDependencies: BuildDependen
       }
       commands.push(`mkdir -p ${targetDir}`);
       await Promise.all(
-        dependencyInfo.dependencies.map((dep) => {
+        dependencyInfo.dependencies.map(async (dep) => {
+          commands.push(`curl -SfL ${dep.url} -o ${targetDir}/${dep.filename}`);
           try {
-            executeCliCommand({
+            return await executeCliCommand({
               command: 'curl',
-              args: ['-SfL', dep.url, '-o', `${targetDir}/${dep.filename}`],
+              args: ['--max-time', '10', '-SfL', dep.url, '-o', `${targetDir}/${dep.filename}`],
+              options: options,
             });
           } catch (error) {
             console.error(
               `ERROR! Could not curl ${dep.url} into ${targetDir}/${dep.filename}. Dependency information: `,
               dependencyInfo
             );
           }
-          commands.push(`curl -SfL ${dep.url} -o ${targetDir}/${dep.filename}`);
         })
       );
     })
   );
   return commands;
 }
 
-export async function prepareBuildAndGetDependencies(
-  repoName: string,
-  projectName: string,
-  baseUrl: string,
-  buildDependencies: BuildDependencies,
-  directory?: string
-) {
+export async function prepareBuild(repoName: string, projectName: string, baseUrl: string, directory?: string) {
   const repoDir = getRepoDir(repoName, directory);
-  await downloadBuildDependencies(buildDependencies, repoName);
-  console.log('Downloaded Build dependencies');
 
   // doing these in parallel
   const commandPromises = [

diff --git a/src/commands/src/helpers/index.ts b/src/commands/src/helpers/index.ts
@@ -14,9 +14,11 @@ const EPIPE_SYSCALL = 'write';
 
 export class ExecuteCommandError extends Error {
   data: unknown;
-  constructor(message: string, data: unknown) {
+  exitCode: number | null;
+  constructor(message: string, exitCode: number | null, data?: unknown) {
     super(message);
     this.data = data;
+    this.exitCode = exitCode;
   }
 }
 
@@ -80,7 +82,7 @@ export async function executeAndPipeCommands(
         return;
       }
 
-      reject(new ExecuteCommandError('The first command stdin (cmdTo) failed', err));
+      reject(new ExecuteCommandError('The first command stdin (cmdTo) failed', err.errno, err));
       hasRejected = true;
     });
 
@@ -89,7 +91,7 @@ export async function executeAndPipeCommands(
     });
 
     cmdFrom.on('error', (err) => {
-      reject(new ExecuteCommandError('The first command (cmdTo) failed', err));
+      reject(new ExecuteCommandError('The first command (cmdTo) failed', 1, err));
       hasRejected = true;
     });
 
@@ -105,7 +107,7 @@ export async function executeAndPipeCommands(
     });
 
     cmdTo.on('error', (err) => {
-      reject(new ExecuteCommandError('The second command failed', err));
+      reject(new ExecuteCommandError('The second command failed', 1, err));
     });
 
     cmdTo.on('exit', (exitCode) => {
@@ -127,7 +129,13 @@ export async function executeAndPipeCommands(
           console.error('error', errorText.join(''));
         }
 
-        reject(new ExecuteCommandError('The command failed', { exitCode, outputText, errorText }));
+        reject(
+          new ExecuteCommandError('The command failed', exitCode, {
+            exitCode,
+            outputText: outputText.join(''),
+            errorText: errorText.join(''),
+          })
+        );
         return;
       }
 
@@ -177,7 +185,7 @@ export async function executeCliCommand({
 
     executedCommand.on('error', (err) => {
       console.log(`ERROR in executeCliCommand.\nCommand: ${command} ${args.join(' ')}\nError: ${err}`);
-      reject(new ExecuteCommandError('The command failed', err));
+      reject(new ExecuteCommandError('The command failed', 1, err));
     });
 
     executedCommand.on('close', (exitCode) => {
@@ -204,7 +212,11 @@ export async function executeCliCommand({
             Options provided: ${JSON.stringify(options, null, 4)}\n
             Stdout: ${outputText.join('')} \n
             Error: ${errorText.join('')}`,
-            exitCode
+            exitCode,
+            {
+              outputText: outputText.join(''),
+              errorText: errorText.join(''),
+            }
           )
         );
         return;

diff --git a/src/commands/src/shared/next-gen-parse.ts b/src/commands/src/shared/next-gen-parse.ts
@@ -1,7 +1,7 @@
 import path from 'path';
 import { Job } from '../../../entities/job';
 import { getDirectory } from '../../../job/jobHandler';
-import { CliCommandResponse, executeCliCommand } from '../helpers';
+import { CliCommandResponse, ExecuteCommandError, executeCliCommand } from '../helpers';
 
 const RSTSPEC_FLAG = '--rstspec=https://raw.githubusercontent.com/mongodb/snooty-parser/latest/snooty/rstspec.toml';
 interface NextGenParseParams {
@@ -37,6 +37,9 @@ export async function nextGenParse({ job, patchId, isProd }: NextGenParseParams)
     });
     return result;
   } catch (error) {
+    if (error instanceof ExecuteCommandError && error.exitCode !== 1) {
+      return error.data as CliCommandResponse;
+    }
     throw new Error(`next-gen-parse failed. \n ${error}`);
   }
 }
diff --git a/src/job/jobHandler.ts b/src/job/jobHandler.ts
@@ -14,13 +14,7 @@ import { IJobValidator } from './jobValidator';
 import { RepoEntitlementsRepository } from '../repositories/repoEntitlementsRepository';
 import { DocsetsRepository } from '../repositories/docsetsRepository';
 import { MONOREPO_NAME } from '../monorepo/utils/monorepo-constants';
-import {
-  nextGenHtml,
-  nextGenParse,
-  oasPageBuild,
-  persistenceModule,
-  prepareBuildAndGetDependencies,
-} from '../commands';
+import { nextGenHtml, nextGenParse, oasPageBuild, persistenceModule, prepareBuild } from '../commands';
 import { downloadBuildDependencies } from '../commands/src/helpers/dependency-helpers';
 import { CliCommandResponse } from '../commands/src/helpers';
 require('fs');
@@ -213,17 +207,14 @@ export abstract class JobHandler {
   }
 
   @throwIfJobInterupted()
-  private async getBuildDependencies() {
-    const buildDependencies = await this._repoBranchesRepo.getBuildDependencies(this.currJob.payload.repoName);
-    if (!buildDependencies) return [];
-    return buildDependencies;
-  }
-
-  @throwIfJobInterupted()
-  private async getAndBuildDependencies() {
-    const buildDependencies = await this.getBuildDependencies();
-    const commands = await downloadBuildDependencies(buildDependencies, this.currJob.payload.repoName);
-    this._logger.save(this._currJob._id, commands.join('\n'));
+  private async getAndDownloadBuildDependencies() {
+    const repoName = this.currJob.payload.repoName;
+    const directory = this.currJob.payload.repoName === MONOREPO_NAME ? this.currJob.payload.directory : undefined;
+    const buildDependencies = await this._repoBranchesRepo.getBuildDependencies(repoName, directory);
+    if (!buildDependencies) return;
+    await this._logger.save(this._currJob._id, 'Identified Build dependencies');
+    const commands = await downloadBuildDependencies(buildDependencies, this.currJob.payload.repoName, directory);
+    await this._logger.save(this._currJob._id, `${commands.join('\n')}`);
   }
 
   @throwIfJobInterupted()
@@ -550,7 +541,7 @@ export abstract class JobHandler {
     this._logger.save(this._currJob._id, 'Checked Commit');
     await this.pullRepo();
     this._logger.save(this._currJob._id, 'Pulled Repo');
-    await this.getAndBuildDependencies();
+    await this.getAndDownloadBuildDependencies();
     this._logger.save(this._currJob._id, 'Downloaded Build dependencies');
     this.prepBuildCommands();
     this._logger.save(this._currJob._id, 'Prepared Build commands');
@@ -581,8 +572,8 @@ export abstract class JobHandler {
     await this.setEnvironmentVariables();
     this.logger.save(job._id, 'Prepared Environment variables');
 
-    const buildDependencies = await this.getBuildDependencies();
-    this._logger.save(this._currJob._id, 'Identified Build dependencies');
+    await this.getAndDownloadBuildDependencies();
+    this._logger.save(this._currJob._id, 'Downloaded Build dependencies');
 
     const docset = await this._docsetsRepo.getRepo(this._currJob.payload.repoName, this._currJob.payload.directory);
     let env = this._config.get<string>('env');
@@ -591,16 +582,9 @@ export abstract class JobHandler {
     }
     const baseUrl = docset?.url?.[env] || 'https://mongodbcom-cdn.website.staging.corp.mongodb.com';
 
-    const { patchId } = await prepareBuildAndGetDependencies(
-      job.payload.repoName,
-      job.payload.project,
-      baseUrl,
-      buildDependencies,
-      job.payload.directory
-    );
+    const { patchId } = await prepareBuild(job.payload.repoName, job.payload.project, baseUrl, job.payload.directory);
     // Set patchId on payload for use in nextGenStage
     this._currJob.payload.patchId = patchId;
-    this._logger.save(this._currJob._id, 'Downloaded Build dependencies');
 
     let buildStepOutput: CliCommandResponse;
 

diff --git a/src/repositories/repoBranchesRepository.ts b/src/repositories/repoBranchesRepository.ts
@@ -9,8 +9,9 @@ export class RepoBranchesRepository extends BaseRepository {
     super(config, logger, 'RepoBranchesRepository', db.collection(config.get('repoBranchesCollection')));
   }
 
-  async getBuildDependencies(repoName: string): Promise<BuildDependencies> {
+  async getBuildDependencies(repoName: string, directoryName?: string): Promise<BuildDependencies> {
     const query = { repoName: repoName };
+    if (directoryName) query['directories.snooty_toml'] = `/${directoryName}`;
     const repo = await this.findOne(
       query,
       `Mongo Timeout Error: Timedout while retrieving build dependencies for ${repoName}`