From 030a8354b594e6ba980adeedbf5d92e26d69c72f Mon Sep 17 00:00:00 2001 From: Jared Mariash <38887481+HugeIRL@users.noreply.github.com> Date: Wed, 13 Sep 2023 19:04:52 -0500 Subject: [PATCH] SUP-1372 fix compressed when ignore-missing is set (#91) * Updated README.md version numbers * Removed compression expression wrap * Updated compression test, add new compression ignore-missing test * Removed mocked glob pathing in upload compressed tests * Add clarification when using compressed with ignore-missing * Update test title --------- Co-authored-by: Pol (Paula) --- README.md | 28 ++++++------ hooks/post-command | 8 +++- tests/upload-compressed.bats | 86 +++++++++++++++++++++++++++++++----- 3 files changed, 95 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 72237c0..b5d4215 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ This functionality duplicates the [artifact_paths](https://buildkite.com/docs/pi steps: - command: ... plugins: - - artifacts#v1.9.0: + - artifacts#v1.9.1: upload: "log/**/*.log" ``` @@ -20,7 +20,7 @@ You can specify multiple files/globs to upload as artifacts: steps: - command: ... plugins: - - artifacts#v1.9.0: + - artifacts#v1.9.1: upload: [ "log/**/*.log", "debug/*.error" ] ``` @@ -30,7 +30,7 @@ And even rename them before uploading them (can not use globs here though, sorry steps: - command: ... plugins: - - artifacts#v1.9.0: + - artifacts#v1.9.1: upload: - from: log1.log to: log2.log @@ -47,7 +47,7 @@ eg: uploading a public file when using S3 steps: - command: ... plugins: - - artifacts#v1.9.0: + - artifacts#v1.9.1: upload: "coverage-report/**/*" s3-upload-acl: public-read ``` @@ -57,7 +57,7 @@ eg: uploading a private file when using GS steps: - command: ... plugins: - - artifacts#v1.9.0: + - artifacts#v1.9.1: upload: "coverage-report/**/*" gs-upload-acl: private ``` @@ -70,7 +70,7 @@ This downloads artifacts matching globs to the local filesystem. See [downloadin steps: - command: ... plugins: - - artifacts#v1.9.0: + - artifacts#v1.9.1: download: "log/**/*.log" ``` @@ -80,7 +80,7 @@ You can specify multiple files/patterns: steps: - command: ... plugins: - - artifacts#v1.9.0: + - artifacts#v1.9.1: download: [ "log/**/*.log", "debug/*.error" ] ``` @@ -90,7 +90,7 @@ Rename particular files after downloading them: steps: - command: ... plugins: - - artifacts#v1.9.0: + - artifacts#v1.9.1: download: - from: log1.log to: log2.log @@ -102,7 +102,7 @@ And even do so from different builds/steps: steps: - command: ... plugins: - - artifacts#v1.9.0: + - artifacts#v1.9.1: step: UUID-DEFAULT build: UUID-DEFAULT-2 download: @@ -145,7 +145,7 @@ When uploading, the file or directory specified in the `upload` option will be c steps: - command: ... plugins: - - artifacts#v1.9.0: + - artifacts#v1.9.1: upload: "log/my-folder" compressed: logs.zip ``` @@ -156,14 +156,14 @@ When downloading, this option states the actual name of the artifact to be downl steps: - command: ... plugins: - - artifacts#v1.9.0: + - artifacts#v1.9.1: download: "log/file.log" compressed: logs.tgz ``` ### `ignore-missing` (optional, boolean) -If set to `true`, it will ignore errors caused when calling `buildkite-agent artifact` to prevent failures if you expect artifacts not to be present in some situations. +If set to `true`, it will ignore errors caused when calling `buildkite-agent artifact` to prevent failures if you expect artifacts not to be present in some situations. When using the `compressed` property, it will ignore compressing the artifacts that are not present. ### `skip-on-status` (optional, integer or array of integers, uploads only) @@ -177,7 +177,7 @@ Skip uploading if the main command failed with exit code 147: steps: - command: ... plugins: - - artifacts#v1.9.0: + - artifacts#v1.9.1: upload: "log/*.log" skip-on-status: 147 ``` @@ -188,7 +188,7 @@ Alternatively, skip artifact uploading on exit codes 1 and 5: steps: - command: ... plugins: - - artifacts#v1.9.0: + - artifacts#v1.9.1: upload: "log/*.log" skip-on-status: - 1 diff --git a/hooks/post-command b/hooks/post-command index 357c1c4..7082535 100755 --- a/hooks/post-command +++ b/hooks/post-command @@ -128,8 +128,12 @@ if [[ "${SINGULAR_UPLOAD_OBJECT}" == "true" ]]; then if [[ "${COMPRESSED}" == "true" ]]; then echo "~~~ Compressing ${path} to ${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}" - "${compress[@]}" "${path}" - path=" ${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}" + if [[ ! -e "${path}" ]]; then + echo "+++ 🚨 Unable to compress artifact, '${path}' may not exist or is an empty directory" + else + "${compress[@]}" "${path}" + path=" ${BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED}" + fi fi echo "~~~ Uploading artifacts ${EXTRA_MESSAGE}" diff --git a/tests/upload-compressed.bats b/tests/upload-compressed.bats index 3ef2d9b..2ef73d9 100644 --- a/tests/upload-compressed.bats +++ b/tests/upload-compressed.bats @@ -5,6 +5,34 @@ load "${BATS_PLUGIN_PATH}/load.bash" # Uncomment to enable stub debug output: # export BUILDKITE_AGENT_STUB_DEBUG=/dev/tty +@test "Compression fails when there is nothing to compress" { + export BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD="non_existent_file.txt" + export BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED="file.zip" + + run "$PWD/hooks/post-command" + + assert_failure + assert_output --partial "Unable to compress artifact, 'non_existent_file.txt' may not exist or is an empty directory" + + unset BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD + unset BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED +} + +@test "Compression ignored when there is nothing to compress and ignore-missing is set to true" { + export BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD="non_existent_file.txt" + export BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED="file.zip" + export BUILDKITE_PLUGIN_ARTIFACTS_IGNORE_MISSING="true" + + run "$PWD/hooks/post-command" + + assert_success + assert_output --partial "Unable to compress artifact, 'non_existent_file.txt' may not exist or is an empty directory" + assert_output --partial "Ignoring error in upload of" + + unset BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD + unset BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED +} + @test "Invalid compressed format" { export BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD="file.log" export BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED="file.rar" @@ -20,9 +48,11 @@ load "${BATS_PLUGIN_PATH}/load.bash" } @test "Single value zip" { - export BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD="*.log" + export BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD="file.log" export BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED="file.zip" + touch "file.log" + stub buildkite-agent \ "artifact upload \* : echo uploaded \$3" @@ -32,7 +62,7 @@ load "${BATS_PLUGIN_PATH}/load.bash" run "$PWD/hooks/post-command" assert_success - assert_output --partial "Compressing *.log to file.zip" + assert_output --partial "Compressing file.log to file.zip" assert_output --partial "Uploading artifacts" assert_output --partial "uploaded file.zip" @@ -41,12 +71,16 @@ load "${BATS_PLUGIN_PATH}/load.bash" unset BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD unset BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED + + rm "file.log" } @test "Single value tgz" { - export BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD="*.log" + export BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD="file.log" export BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED="file.tgz" + touch "file.log" + stub buildkite-agent \ "artifact upload \* : echo uploaded \$3" @@ -56,7 +90,7 @@ load "${BATS_PLUGIN_PATH}/load.bash" run "$PWD/hooks/post-command" assert_success - assert_output --partial "Compressing *.log to file.tgz" + assert_output --partial "Compressing file.log to file.tgz" assert_output --partial "Uploading artifacts" assert_output --partial "uploaded file.tgz" @@ -65,6 +99,8 @@ load "${BATS_PLUGIN_PATH}/load.bash" unset BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD unset BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED + + rm "file.log" } @test "Single file zip with relocation" { @@ -138,10 +174,12 @@ load "${BATS_PLUGIN_PATH}/load.bash" } @test "Single value zip with job" { - export BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD="*.log" + export BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD="file.log" export BUILDKITE_PLUGIN_ARTIFACTS_JOB="12345" export BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED="file.zip" + touch "file.log" + stub buildkite-agent \ "artifact upload --job \* \* : echo uploaded \$5 with --job \$4" @@ -152,9 +190,9 @@ load "${BATS_PLUGIN_PATH}/load.bash" assert_success assert_output --partial "Uploading artifacts (extra args: '--job 12345')" - assert_output --partial "Compressing *.log to file.zip" + assert_output --partial "Compressing file.log to file.zip" assert_output --partial "uploaded file.zip" - refute_output --partial "uploaded *.log" + refute_output --partial "uploaded file.log" unstub buildkite-agent unstub zip @@ -162,13 +200,17 @@ load "${BATS_PLUGIN_PATH}/load.bash" unset BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD unset BUILDKITE_PLUGIN_ARTIFACTS_JOB unset BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED + + rm "file.log" } @test "Single value tgz with job" { - export BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD="*.log" + export BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD="file.log" export BUILDKITE_PLUGIN_ARTIFACTS_JOB="12345" export BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED="file.tgz" + touch "file.log" + stub buildkite-agent \ "artifact upload --job \* \* : echo uploaded \$5 with --job \$4" @@ -179,9 +221,9 @@ load "${BATS_PLUGIN_PATH}/load.bash" assert_success assert_output --partial "Uploading artifacts (extra args: '--job 12345')" - assert_output --partial "Compressing *.log to file.tgz" + assert_output --partial "Compressing file.log to file.tgz" assert_output --partial "uploaded file.tgz" - refute_output --partial "uploaded *.log" + refute_output --partial "uploaded file.log" unstub buildkite-agent unstub tar @@ -189,6 +231,8 @@ load "${BATS_PLUGIN_PATH}/load.bash" unset BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD unset BUILDKITE_PLUGIN_ARTIFACTS_JOB unset BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED + + rm "file.log" } @test "Multiple artifacts zip" { @@ -197,6 +241,10 @@ load "${BATS_PLUGIN_PATH}/load.bash" export BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_2="baz.log" export BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED="file.zip" + touch /tmp/foo.log + touch bar.log + touch baz.log + stub buildkite-agent \ "artifact upload \* : echo uploaded \$3" @@ -220,6 +268,10 @@ load "${BATS_PLUGIN_PATH}/load.bash" unset BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_1 unset BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_2 unset BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED + + rm /tmp/foo.log + rm bar.log + rm baz.log } @test "Multiple artifacts tgz" { @@ -228,6 +280,10 @@ load "${BATS_PLUGIN_PATH}/load.bash" export BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_2="baz.log" export BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED="file.tgz" + touch /tmp/foo.log + touch bar.log + touch baz.log + stub buildkite-agent \ "artifact upload \* : echo uploaded \$3" @@ -251,6 +307,10 @@ load "${BATS_PLUGIN_PATH}/load.bash" unset BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_1 unset BUILDKITE_PLUGIN_ARTIFACTS_UPLOAD_2 unset BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED + + rm /tmp/foo.log + rm bar.log + rm baz.log } @test "Multiple artifacs zip some relocation" { @@ -261,6 +321,8 @@ load "${BATS_PLUGIN_PATH}/load.bash" export BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED="file.zip" touch /tmp/foo.log + touch bar.log + touch baz.log stub buildkite-agent \ "artifact upload \* : echo uploaded \$3" @@ -282,6 +344,8 @@ load "${BATS_PLUGIN_PATH}/load.bash" assert [ -e /tmp/foo2.log ] assert [ ! -e /tmp/foo.log ] rm /tmp/foo2.log + rm bar.log + rm baz.log unstub buildkite-agent unstub zip @@ -365,4 +429,4 @@ load "${BATS_PLUGIN_PATH}/load.bash" unset BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_0_FROM unset BUILDKITE_PLUGIN_ARTIFACTS_DOWNLOAD_0_TO unset BUILDKITE_PLUGIN_ARTIFACTS_COMPRESSED -} +} \ No newline at end of file