From acf8693e45687c9e3e9fd147ab9451dd06d78d80 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Mon, 30 Sep 2024 10:49:59 -0500 Subject: [PATCH] DEVPROD-11430 Use temporary aws creds for s3 uploads (#311) --- .evergreen/config.yml | 138 +++++++++++++++++++----------------------- 1 file changed, 61 insertions(+), 77 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 604ba3d8..e751271f 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -54,21 +54,18 @@ functions: export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration" export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin" - export UPLOAD_BUCKET="${project}" cat < expansion.yml CURRENT_VERSION: "$CURRENT_VERSION" DRIVERS_TOOLS: "$DRIVERS_TOOLS" MONGO_ORCHESTRATION_HOME: "$MONGO_ORCHESTRATION_HOME" MONGODB_BINARIES: "$MONGODB_BINARIES" - UPLOAD_BUCKET: "$UPLOAD_BUCKET" PROJECT_DIRECTORY: "$PROJECT_DIRECTORY" PREPARE_SHELL: | set -o errexit export DRIVERS_TOOLS="$DRIVERS_TOOLS" export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME" export MONGODB_BINARIES="$MONGODB_BINARIES" - export UPLOAD_BUCKET="$UPLOAD_BUCKET" export PROJECT_DIRECTORY="$PROJECT_DIRECTORY" export TMPDIR="$MONGO_ORCHESTRATION_HOME/db" export PATH="$MONGODB_BINARIES:$PATH" @@ -100,13 +97,17 @@ functions: echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" > $MONGO_ORCHESTRATION_HOME/orchestration.config "upload release": + - command: ec2.assume_role + params: + role_arn: ${assume_role_arn} - command: s3.put params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} local_file: ${project}.tar.gz - remote_file: ${UPLOAD_BUCKET}/${project}-${CURRENT_VERSION}.tar.gz - bucket: mciuploads + remote_file: ${project}-${CURRENT_VERSION}.tar.gz + bucket: ${aws_bucket} permissions: public-read content_type: ${content_type|application/x-gzip} @@ -122,28 +123,36 @@ functions: source_dir: ${PROJECT_DIRECTORY}/ include: - "./**" + - command: ec2.assume_role + params: + role_arn: ${assume_role_arn} - command: s3.put params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} local_file: ${build_id}.tar.gz - # Example: /mciuploads/${UPLOAD_BUCKET}/gcc49/9dfb7d741efbca16faa7859b9349d7a942273e43/debug-compile-nosasl-nossl/mongo_c_driver_releng_9dfb7d741efbca16faa7859b9349d7a942273e43_16_11_08_19_29_52.tar.gz - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${task_name}/${build_id}.tar.gz - bucket: mciuploads + # Example: ${aws_bucket}/gcc49/9dfb7d741efbca16faa7859b9349d7a942273e43/debug-compile-nosasl-nossl/mongo_c_driver_releng_9dfb7d741efbca16faa7859b9349d7a942273e43_16_11_08_19_29_52.tar.gz + remote_file: ${build_variant}/${revision}/${task_name}/${build_id}.tar.gz + bucket: ${aws_bucket} permissions: public-read content_type: ${content_type|application/x-gzip} "fetch build": + - command: ec2.assume_role + params: + role_arn: ${assume_role_arn} - command: shell.exec params: continue_on_err: true script: "set -o xtrace && rm -rf ${PROJECT_DIRECTORY}" - command: s3.get params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${BUILD_NAME}/${build_id}.tar.gz - bucket: mciuploads + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} + remote_file: ${build_variant}/${revision}/${BUILD_NAME}/${build_id}.tar.gz + bucket: ${aws_bucket} local_file: build.tar.gz - command: shell.exec params: @@ -169,45 +178,10 @@ functions: ${PREPARE_SHELL} [ -f ${PROJECT_DIRECTORY}/${file} ] && sh ${PROJECT_DIRECTORY}/${file} || echo "${PROJECT_DIRECTORY}/${file} not available, skipping" - "upload docs" : - - command: shell.exec - params: - silent: true - script: | - export AWS_ACCESS_KEY_ID=${aws_key} - export AWS_SECRET_ACCESS_KEY=${aws_secret} - aws s3 cp ${PROJECT_DIRECTORY}/doc/html s3://mciuploads/${UPLOAD_BUCKET}/docs/${CURRENT_VERSION} --recursive --acl public-read --region us-east-1 - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: ${PROJECT_DIRECTORY}/doc/html/index.html - remote_file: ${UPLOAD_BUCKET}/docs/${CURRENT_VERSION}/index.html - bucket: mciuploads - permissions: public-read - content_type: text/html - display_name: "Rendered docs" - - "upload coverage" : - - command: shell.exec - params: - silent: true - script: | - export AWS_ACCESS_KEY_ID=${aws_key} - export AWS_SECRET_ACCESS_KEY=${aws_secret} - aws s3 cp ${PROJECT_DIRECTORY}/coverage s3://mciuploads/${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/coverage/ --recursive --acl public-read --region us-east-1 - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: ${PROJECT_DIRECTORY}/coverage/index.html - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/coverage/index.html - bucket: mciuploads - permissions: public-read - content_type: text/html - display_name: "Coverage Report" - "upload scan artifacts" : + - command: ec2.assume_role + params: + role_arn: ${assume_role_arn} - command: shell.exec type: test params: @@ -221,22 +195,25 @@ functions: - command: shell.exec params: silent: true + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] script: | - export AWS_ACCESS_KEY_ID=${aws_key} - export AWS_SECRET_ACCESS_KEY=${aws_secret} - aws s3 cp ${PROJECT_DIRECTORY}/scan s3://mciuploads/${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/scan/ --recursive --acl public-read --region us-east-1 + aws s3 cp ${PROJECT_DIRECTORY}/scan s3://${aws_bucket}/${build_variant}/${revision}/${version_id}/${build_id}/scan/ --recursive --acl public-read --region us-east-1 - command: s3.put params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} local_file: ${PROJECT_DIRECTORY}/scan.html - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/scan/index.html - bucket: mciuploads + remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/scan/index.html + bucket: ${aws_bucket} permissions: public-read content_type: text/html display_name: "Scan Build Report" "upload mo artifacts": + - command: ec2.assume_role + params: + role_arn: ${assume_role_arn} - command: shell.exec params: script: | @@ -244,26 +221,31 @@ functions: find $MONGO_ORCHESTRATION_HOME -name \*.log | xargs tar czf mongodb-logs.tar.gz - command: s3.put params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} local_file: mongodb-logs.tar.gz - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-mongodb-logs.tar.gz - bucket: mciuploads + remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-mongodb-logs.tar.gz + bucket: ${aws_bucket} permissions: public-read content_type: ${content_type|application/x-gzip} display_name: "mongodb-logs.tar.gz" - command: s3.put params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} local_file: ${DRIVERS_TOOLS}/.evergreen/orchestration/server.log - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-orchestration.log - bucket: mciuploads + remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-orchestration.log + bucket: ${aws_bucket} permissions: public-read content_type: ${content_type|text/plain} display_name: "orchestration.log" "upload working dir": + - command: ec2.assume_role + params: + role_arn: ${assume_role_arn} - command: archive.targz_pack params: target: "working-dir.tar.gz" @@ -272,11 +254,12 @@ functions: - "./**" - command: s3.put params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} local_file: working-dir.tar.gz - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-working-dir.tar.gz - bucket: mciuploads + remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-working-dir.tar.gz + bucket: ${aws_bucket} permissions: public-read content_type: ${content_type|application/x-gzip} display_name: "working-dir.tar.gz" @@ -291,11 +274,12 @@ functions: - "*.lock" - command: s3.put params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} + aws_key: ${AWS_ACCESS_KEY_ID} + aws_secret: ${AWS_SECRET_ACCESS_KEY} + aws_session_token: ${AWS_SESSION_TOKEN} local_file: drivers-dir.tar.gz - remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-drivers-dir.tar.gz - bucket: mciuploads + remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-drivers-dir.tar.gz + bucket: ${aws_bucket} permissions: public-read content_type: ${content_type|application/x-gzip} display_name: "drivers-dir.tar.gz"