-
Notifications
You must be signed in to change notification settings - Fork 73
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
DOP-4209 #958
DOP-4209 #958
Changes from 178 commits
f4b5188
8f941ba
706af67
1a63501
0be0b24
6c43e45
e733122
2442b6f
901a229
9aa2fd2
8bd6ade
c11687b
68eed83
e9d9b36
dc4c2a1
1873ce9
478ebf3
ab4f652
71751a4
38c0f7d
e313e5f
b97835c
97cbc8b
2832b0e
1d45d72
11bac2c
7aee7af
afadedc
e320a67
178e680
cf18066
3bc4550
781b7ab
bae5f35
53d68da
62a9747
733211c
2f3bebd
6b4dcbe
56b11c7
9c97ae1
4cb8bbb
f11e4b5
c3ae28e
e352d0b
b576633
c099d3b
a2e9a64
02aba79
7a10135
9e36e45
504cb84
0c8de69
62adea1
86dbf92
857a1db
d87b56e
cb8080e
01463ff
0a2c7b5
c0717cf
02b61af
a3cbc72
7d0ba08
375a97d
482f841
9f82e55
93fbc40
68137c4
be0d0a0
c386506
6534d77
d86040f
fe4ddca
8d7182f
312ba00
3c7156d
22d75fd
6ae841f
21207c7
e494c7f
8f48f51
b87d85c
f1b6e57
f48ac30
f24ef53
76a0677
901792c
5254b44
09cc84a
94312e8
f502182
52590c4
243ac89
32496d4
b770001
3667b5d
2650e8f
0108bc4
150b11f
39d84b8
a249c26
178a527
a6f3261
f0b535c
045180b
1d69643
7713c19
c6c5c69
f796c4a
ffbc331
39837b4
c3313a3
40e5c26
c25b533
248df20
d23da3c
c0c4c57
2adf502
4c8d4c3
64cf314
07967fb
2a8386b
f5dbead
07caee9
e45fce2
5c558da
4ac7b01
b57f0ac
493edaa
52b2f15
492d7c5
7f3299d
b8dec8c
bfe3cbd
93d2642
038337f
211d1c2
841e6f4
033d58a
1eaf867
c4052bb
64e6af4
d1367a7
e16a789
ffe258c
58b7c14
f7d968f
8e5e756
0f27590
cdaa1d8
341e1d3
41f10a7
9ef2582
57bbafd
f3b7748
04a687a
2a99aaa
845a0b7
1c59684
178cb7e
3c5ab4d
a9daaa8
cb63290
7b7901d
e444059
510172d
248257e
c901a06
419ced8
6218371
8181640
5ee7c73
749077c
313af9c
52f9ed7
6e7627d
97ae5f4
068c8ee
6af15c4
afd3064
b7e6bab
802340e
882dff2
4ef5cf8
6c05afa
f06608f
ce59317
a0d9a4b
da37185
20c2bc6
57c3fda
577fba4
fc56fc7
bb41a5f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -19,7 +19,7 @@ export class WebhookEnvConstruct extends Construct { | |
const ssmPrefix = getSsmPathPrefix(); | ||
const env = getEnv(); | ||
|
||
const featureFlagMonorepoPath = StringParameter.valueFromLookup(this, `${ssmPrefix}/flag/monorepo_path`); | ||
const featureFlagMonorepoPath = StringParameter.valueFromLookup(this, `${ssmPrefix}/DOP-4127/flag/monorepo_path`); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Just noting that this needs to be changed back before merge |
||
const dbName = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/dbname`); | ||
const snootyDbName = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/snooty`); | ||
const repoBranchesCollection = StringParameter.valueFromLookup(this, `${ssmPrefix}/atlas/collections/repo`); | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -50,7 +50,7 @@ export class WorkerEnvConstruct extends Construct { | |
// doing this for the time being, but I think we don't need to necessarily retrieve this from ssm for feature branches, nor would we want to in that case | ||
const previewBuildEnabled = StringParameter.valueFromLookup(this, `${ssmPrefix}/flag/preview_build/enabled`); | ||
const featureFlagUpdatePages = StringParameter.valueFromLookup(this, `${ssmPrefix}/flag/update_pages`); | ||
const featureFlagMonorepoPath = StringParameter.valueFromLookup(this, `${ssmPrefix}/flag/monorepo_path`); | ||
const featureFlagMonorepoPath = StringParameter.valueFromLookup(this, `${ssmPrefix}/DOP-4127/flag/monorepo_path`); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Note to change back before merge |
||
|
||
const entitlementCollection = StringParameter.valueFromLookup( | ||
this, | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -43,16 +43,25 @@ async function createEnvProdFile({ | |
} | ||
} | ||
|
||
export async function downloadBuildDependencies(buildDependencies: BuildDependencies, repoName: string) { | ||
export async function downloadBuildDependencies( | ||
buildDependencies: BuildDependencies, | ||
repoName: string, | ||
directory?: string | ||
) { | ||
const commands: string[] = []; | ||
await Promise.all( | ||
buildDependencies.map(async (dependencyInfo) => { | ||
const repoDir = getRepoDir(repoName); | ||
const repoDir = directory ? getRepoDir(repoName, directory) : getRepoDir(repoName); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No need to do this conditional! The
|
||
const targetDir = dependencyInfo.targetDir ?? repoDir; | ||
let options = {}; | ||
if (targetDir != repoDir) { | ||
options = { cwd: repoDir }; | ||
} | ||
try { | ||
await executeCliCommand({ | ||
command: 'mkdir', | ||
args: ['-p', targetDir], | ||
options: options, | ||
}); | ||
} catch (error) { | ||
console.error( | ||
|
@@ -63,17 +72,19 @@ export async function downloadBuildDependencies(buildDependencies: BuildDependen | |
} | ||
commands.push(`mkdir -p ${targetDir}`); | ||
await Promise.all( | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Note: I did not use There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What version of node are you using? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. v18.17.1! |
||
dependencyInfo.dependencies.map((dep) => { | ||
dependencyInfo.dependencies.map(async (dep) => { | ||
try { | ||
executeCliCommand({ | ||
command: 'curl', | ||
args: ['-SfL', dep.url, '-o', `${targetDir}/${dep.filename}`], | ||
args: ['--max-time', '10', '-SfL', dep.url, '-o', `${targetDir}/${dep.filename}`], | ||
options: options, | ||
}); | ||
} catch (error) { | ||
console.error( | ||
`ERROR! Could not curl ${dep.url} into ${targetDir}/${dep.filename}. Dependency information: `, | ||
dependencyInfo | ||
); | ||
return commands; | ||
} | ||
commands.push(`curl -SfL ${dep.url} -o ${targetDir}/${dep.filename}`); | ||
}) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hmm, interesting thing here. I could be wrong, but I believe that because we're returning a static value (even though it's a Promise because of the async keyword) it can return before the I think this can be slightly changed to do what you're trying to do and cleaner too. If you take away the I have an example below. Like this:
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. These are all great points, @mmeigs. The one small nit with the example though is that we won't hit this catch block, I think. This is because we return the promise, but we don't await it, not giving the promise the opportunity to settle before the catch block can be entered. I think we can add the This also has me thinking as well, do we want to do an There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Testing this out with some incorrect curl links but this approach still doesn't seem to catch the error 😞 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hmm, I wonder if doing const cliResult = await executeCliCommand(...);
return cliResult; in the try/catch? I wonder if return is sort of overwriting the There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this didn't work either 💔 i dropped a new ticket for this issue though! |
||
|
@@ -87,12 +98,9 @@ export async function prepareBuildAndGetDependencies( | |
repoName: string, | ||
projectName: string, | ||
baseUrl: string, | ||
buildDependencies: BuildDependencies, | ||
directory?: string | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If we're not getting dependencies in this function anymore, can we rename this function? |
||
) { | ||
const repoDir = getRepoDir(repoName, directory); | ||
await downloadBuildDependencies(buildDependencies, repoName); | ||
console.log('Downloaded Build dependencies'); | ||
|
||
// doing these in parallel | ||
const commandPromises = [ | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -214,16 +214,21 @@ export abstract class JobHandler { | |
|
||
@throwIfJobInterupted() | ||
private async getBuildDependencies() { | ||
const buildDependencies = await this._repoBranchesRepo.getBuildDependencies(this.currJob.payload.repoName); | ||
const repoName = this.currJob.payload.repoName; | ||
const directory = this.currJob.payload.repoName === MONOREPO_NAME ? this.currJob.payload.directory : undefined; | ||
const buildDependencies = await this._repoBranchesRepo.getBuildDependencies(repoName, directory); | ||
if (!buildDependencies) return []; | ||
await this._logger.save(this._currJob._id, 'Identified Build dependencies'); | ||
return buildDependencies; | ||
} | ||
|
||
@throwIfJobInterupted() | ||
private async getAndBuildDependencies() { | ||
private async getAndDownloadBuildDependencies() { | ||
const repoName = this.currJob.payload.repoName; | ||
const directory = this.currJob.payload.repoName === MONOREPO_NAME ? this.currJob.payload.directory : undefined; | ||
const buildDependencies = await this.getBuildDependencies(); | ||
const commands = await downloadBuildDependencies(buildDependencies, this.currJob.payload.repoName); | ||
this._logger.save(this._currJob._id, commands.join('\n')); | ||
const commands = await downloadBuildDependencies(buildDependencies, this.currJob.payload.repoName, directory); | ||
await this._logger.save(this._currJob._id, `${commands.join('\n')}`); | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We can combine these again. The reason to separate is gone now, and also it would combine the redundancy of accessing the repoName and directory twice in quick succession. |
||
|
||
@throwIfJobInterupted() | ||
|
@@ -550,7 +555,7 @@ export abstract class JobHandler { | |
this._logger.save(this._currJob._id, 'Checked Commit'); | ||
await this.pullRepo(); | ||
this._logger.save(this._currJob._id, 'Pulled Repo'); | ||
await this.getAndBuildDependencies(); | ||
await this.getAndDownloadBuildDependencies(); | ||
this._logger.save(this._currJob._id, 'Downloaded Build dependencies'); | ||
this.prepBuildCommands(); | ||
this._logger.save(this._currJob._id, 'Prepared Build commands'); | ||
|
@@ -581,8 +586,8 @@ export abstract class JobHandler { | |
await this.setEnvironmentVariables(); | ||
this.logger.save(job._id, 'Prepared Environment variables'); | ||
|
||
const buildDependencies = await this.getBuildDependencies(); | ||
this._logger.save(this._currJob._id, 'Identified Build dependencies'); | ||
await this.getAndDownloadBuildDependencies(); | ||
this._logger.save(this._currJob._id, 'Downloaded Build dependencies'); | ||
|
||
const docset = await this._docsetsRepo.getRepo(this._currJob.payload.repoName, this._currJob.payload.directory); | ||
let env = this._config.get<string>('env'); | ||
|
@@ -595,12 +600,10 @@ export abstract class JobHandler { | |
job.payload.repoName, | ||
job.payload.project, | ||
baseUrl, | ||
buildDependencies, | ||
job.payload.directory | ||
); | ||
// Set patchId on payload for use in nextGenStage | ||
this._currJob.payload.patchId = patchId; | ||
this._logger.save(this._currJob._id, 'Downloaded Build dependencies'); | ||
|
||
let buildStepOutput: CliCommandResponse; | ||
|
||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit: Make sure the merge finishes and gets rid of these characters
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Regarding the diff: The quotes are not necessary, but they are also not an issue. Either way is fine, but I would lean toward all the examples showing the same way for each input
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Just noting here that this entire file has multiple unfinished merge flags to take care of!