From 8be0120ccbf29c5565a49e9f48c78e498a054e6d Mon Sep 17 00:00:00 2001 From: Trevor Whitney Date: Tue, 19 Mar 2024 12:00:28 -0600 Subject: [PATCH 1/3] ci: track main of loki-release on main branch * this brings back the `check-format` task --- .github/jsonnetfile.json | 2 +- .github/jsonnetfile.lock.json | 4 +- .github/release-workflows.jsonnet | 93 ++++--- .../loki-release/workflows/build.libsonnet | 114 +++++++-- .../loki-release/workflows/common.libsonnet | 40 ++- .../loki-release/workflows/main.jsonnet | 139 ++++++++++- .../loki-release/workflows/release.libsonnet | 83 ++++++- .../workflows/validate-gel.libsonnet | 97 ++++++++ .../loki-release/workflows/validate.libsonnet | 50 ++-- .../loki-release/workflows/workflows.jsonnet | 13 +- .github/workflows/check.yml | 27 +- .github/workflows/minor-release-pr.yml | 232 ++++++++++-------- .github/workflows/patch-release-pr.yml | 232 ++++++++++-------- .github/workflows/release.yml | 99 +++++++- Makefile | 1 + 15 files changed, 900 insertions(+), 326 deletions(-) create mode 100644 .github/vendor/github.com/grafana/loki-release/workflows/validate-gel.libsonnet diff --git a/.github/jsonnetfile.json b/.github/jsonnetfile.json index cd4469eb6e50..605c762e9f8c 100644 --- a/.github/jsonnetfile.json +++ b/.github/jsonnetfile.json @@ -8,7 +8,7 @@ "subdir": "workflows" } }, - "version": "release-1.10.x" + "version": "main" } ], "legacyImports": true diff --git a/.github/jsonnetfile.lock.json b/.github/jsonnetfile.lock.json index ee1f7b9596b4..9b589d5b0246 100644 --- a/.github/jsonnetfile.lock.json +++ b/.github/jsonnetfile.lock.json @@ -8,8 +8,8 @@ "subdir": "workflows" } }, - "version": "c005223f58b83f288b655dde5bcfeff7490c7aa5", - "sum": "5K+r6Bsb8JMR1ytQjSObjvHFpH7SJBi5D4ysSwvC4/g=" + "version": "6c326b84f35d69f9d01732b2805eb645a52b69cc", + "sum": "Y5sjhGpO7ziNdhSBcF3tfYkZRk25kT5AfYTo/i+qmhM=" } ], "legacyImports": false diff --git a/.github/release-workflows.jsonnet b/.github/release-workflows.jsonnet index ae1f868fa651..a5f0dde9093a 100644 --- a/.github/release-workflows.jsonnet +++ b/.github/release-workflows.jsonnet @@ -1,49 +1,63 @@ local lokiRelease = import 'workflows/main.jsonnet'; local build = lokiRelease.build; + +local releaseLibRef = std.filter( + function(dep) dep.source.git.remote == 'https://github.com/grafana/loki-release.git', + (import 'jsonnetfile.json').dependencies +)[0].version; + +local checkTemplate = 'grafana/loki-release/.github/workflows/check.yml@%s' % releaseLibRef; + +local imageJobs = { + loki: build.image('loki', 'cmd/loki'), + fluentd: build.image('fluent-plugin-loki', 'clients/cmd/fluentd', platform=['linux/amd64']), + 'fluent-bit': build.image('fluent-bit-plugin-loki', 'clients/cmd/fluent-bit', platform=['linux/amd64']), + logstash: build.image('logstash-output-loki', 'clients/cmd/logstash', platform=['linux/amd64']), + logcli: build.image('logcli', 'cmd/logcli'), + 'loki-canary': build.image('loki-canary', 'cmd/loki-canary'), + 'loki-canary-boringcrypto': build.image('loki-canary-boringcrypto', 'cmd/loki-canary-boringcrypto'), + promtail: build.image('promtail', 'clients/cmd/promtail'), + querytee: build.image('loki-query-tee', 'cmd/querytee', platform=['linux/amd64']), +}; + +local buildImage = 'grafana/loki-build-image:0.33.0'; +local golangCiLintVersion = 'v1.55.1'; + +local imageBuildTimeoutMin = 40; +local imagePrefix = 'grafana'; + { 'patch-release-pr.yml': std.manifestYamlDoc( lokiRelease.releasePRWorkflow( - imageJobs={ - loki: build.image('loki', 'cmd/loki'), - fluentd: build.image('fluentd', 'clients/cmd/fluentd', platform=['linux/amd64']), - 'fluent-bit': build.image('fluent-bit', 'clients/cmd/fluent-bit', platform=['linux/amd64']), - logstash: build.image('logstash', 'clients/cmd/logstash', platform=['linux/amd64']), - logcli: build.image('logcli', 'cmd/logcli'), - 'loki-canary': build.image('loki-canary', 'cmd/loki-canary'), - 'loki-canary-boringcrypto': build.image('loki-canary-boringcrypto', 'cmd/loki-canary-boringcrypto'), - 'loki-operator': build.image('loki-operator', 'operator', context='release/operator', platform=['linux/amd64']), - promtail: build.image('promtail', 'clients/cmd/promtail'), - querytee: build.image('querytee', 'cmd/querytee', platform=['linux/amd64']), - }, branches=['release-[0-9]+.[0-9]+.x'], - checkTemplate='grafana/loki-release/.github/workflows/check.yml@release-1.10.x', - imagePrefix='grafana', + buildImage=buildImage, + checkTemplate=checkTemplate, + golangCiLintVersion=golangCiLintVersion, + imageBuildTimeoutMin=imageBuildTimeoutMin, + imageJobs=imageJobs, + imagePrefix=imagePrefix, + releaseLibRef=releaseLibRef, releaseRepo='grafana/loki', skipArm=false, skipValidation=false, + useGitHubAppToken=true, versioningStrategy='always-bump-patch', ), false, false ), 'minor-release-pr.yml': std.manifestYamlDoc( lokiRelease.releasePRWorkflow( - imageJobs={ - loki: build.image('loki', 'cmd/loki'), - fluentd: build.image('fluentd', 'clients/cmd/fluentd', platform=['linux/amd64']), - 'fluent-bit': build.image('fluent-bit', 'clients/cmd/fluent-bit', platform=['linux/amd64']), - logstash: build.image('logstash', 'clients/cmd/logstash', platform=['linux/amd64']), - logcli: build.image('logcli', 'cmd/logcli'), - 'loki-canary': build.image('loki-canary', 'cmd/loki-canary'), - 'loki-canary-boringcrypto': build.image('loki-canary-boringcrypto', 'cmd/loki-canary-boringcrypto'), - 'loki-operator': build.image('loki-operator', 'operator', context='release/operator', platform=['linux/amd64']), - promtail: build.image('promtail', 'clients/cmd/promtail'), - querytee: build.image('querytee', 'cmd/querytee', platform=['linux/amd64']), - }, branches=['k[0-9]+'], - checkTemplate='grafana/loki-release/.github/workflows/check.yml@release-1.10.x', - imagePrefix='grafana', + buildImage=buildImage, + checkTemplate=checkTemplate, + golangCiLintVersion=golangCiLintVersion, + imageBuildTimeoutMin=imageBuildTimeoutMin, + imageJobs=imageJobs, + imagePrefix=imagePrefix, + releaseLibRef=releaseLibRef, releaseRepo='grafana/loki', skipArm=false, skipValidation=false, + useGitHubAppToken=true, versioningStrategy='always-bump-minor', ), false, false ), @@ -52,7 +66,30 @@ local build = lokiRelease.build; branches=['release-[0-9]+.[0-9]+.x', 'k[0-9]+'], getDockerCredsFromVault=true, imagePrefix='grafana', + releaseLibRef=releaseLibRef, releaseRepo='grafana/loki', + useGitHubAppToken=false, ), false, false ), + 'check.yml': std.manifestYamlDoc({ + name: 'check', + on: { + pull_request: {}, + push: { + branches: ['main'], + }, + }, + jobs: { + check: { + uses: checkTemplate, + with: { + build_image: buildImage, + golang_ci_lint_version: golangCiLintVersion, + release_lib_ref: releaseLibRef, + skip_validation: false, + use_github_app_token: true, + }, + }, + }, + }), } diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet index cdd6b82463e4..1f0fd6382e81 100644 --- a/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet +++ b/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet @@ -8,6 +8,7 @@ local releaseLibStep = common.releaseLibStep; image: function( name, path, + dockerfile='Dockerfile', context='release', platform=[ 'linux/amd64', @@ -42,14 +43,18 @@ local releaseLibStep = common.releaseLibStep; |||), step.new('Build and export', 'docker/build-push-action@v5') - + step.withTimeoutMinutes(25) + + step.withTimeoutMinutes('${{ fromJSON(env.BUILD_TIMEOUT) }}') + step.withIf('${{ fromJSON(needs.version.outputs.pr_created) }}') + + step.withEnv({ + IMAGE_TAG: '${{ needs.version.outputs.version }}', + }) + step.with({ context: context, - file: 'release/%s/Dockerfile' % path, + file: 'release/%s/%s' % [path, dockerfile], platforms: '${{ matrix.platform }}', tags: '${{ env.IMAGE_PREFIX }}/%s:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}' % [name], outputs: 'type=docker,dest=release/images/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar' % name, + 'build-args': 'IMAGE_TAG=${{ needs.version.outputs.version }}', }), step.new('upload artifacts', 'google-github-actions/upload-cloud-storage@v2') + step.withIf('${{ fromJSON(needs.version.outputs.pr_created) }}') @@ -60,6 +65,47 @@ local releaseLibStep = common.releaseLibStep; }), ]), + + weeklyImage: function( + name, + path, + dockerfile='Dockerfile', + context='release', + platform=[ + 'linux/amd64', + 'linux/arm64', + 'linux/arm', + ] + ) + job.new() + + job.withSteps([ + common.fetchReleaseLib, + common.fetchReleaseRepo, + common.setupNode, + + step.new('Set up QEMU', 'docker/setup-qemu-action@v3'), + step.new('set up docker buildx', 'docker/setup-buildx-action@v3'), + step.new('Login to DockerHub (from vault)', 'grafana/shared-workflows/actions/dockerhub-login@main'), + + releaseStep('Get weekly version') + + step.withId('weekly-version') + + step.withRun(||| + echo "version=$(./tools/image-tag)" >> $GITHUB_OUTPUT + |||), + + step.new('Build and push', 'docker/build-push-action@v5') + + step.withTimeoutMinutes('${{ fromJSON(env.BUILD_TIMEOUT) }}') + + step.with({ + context: context, + file: 'release/%s/%s' % [path, dockerfile], + platforms: '%s' % std.join(',', platform), + push: true, + tags: '${{ env.IMAGE_PREFIX }}/%s:${{ steps.weekly-version.outputs.version }}' % [name], + 'build-args': 'IMAGE_TAG=${{ steps.weekly-version.outputs.version }}', + }), + ]), + + version: job.new() + job.withSteps([ @@ -67,6 +113,8 @@ local releaseLibStep = common.releaseLibStep; common.fetchReleaseRepo, common.setupNode, common.extractBranchName, + common.githubAppToken, + common.setToken, releaseLibStep('get release version') + step.withId('version') + step.withRun(||| @@ -75,12 +123,18 @@ local releaseLibStep = common.releaseLibStep; --consider-all-branches \ --dry-run \ --dry-run-output release.json \ + --group-pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ + --manifest-file .release-please-manifest.json \ + --pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ --release-type simple \ - --repo-url="${{ env.RELEASE_REPO }}" \ + --repo-url "${{ env.RELEASE_REPO }}" \ + --separate-pull-requests false \ --target-branch "${{ steps.extract_branch.outputs.branch }}" \ - --token="${{ secrets.GH_TOKEN }}" \ + --token "${{ steps.github_app_token.outputs.token }}" \ --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" + cat release.json + if [[ `jq length release.json` -gt 1 ]]; then echo 'release-please would create more than 1 PR, so cannot determine correct version' echo "pr_created=false" >> $GITHUB_OUTPUT @@ -102,11 +156,12 @@ local releaseLibStep = common.releaseLibStep; pr_created: '${{ steps.version.outputs.pr_created }}', }), - dist: function(buildImage, skipArm=true) + dist: function(buildImage, skipArm=true, useGCR=false, makeTargets=['dist', 'packages']) job.new() + job.withSteps([ common.fetchReleaseRepo, common.googleAuth, + common.setupGoogleCloudSdk, step.new('get nfpm signing keys', 'grafana/shared-workflows/actions/get-vault-secrets@main') + step.withId('get-secrets') + step.with({ @@ -117,6 +172,7 @@ local releaseLibStep = common.releaseLibStep; }), releaseStep('build artifacts') + + step.withIf('${{ fromJSON(needs.version.outputs.pr_created) }}') + step.withEnv({ BUILD_IN_CONTAINER: false, DRONE_TAG: '${{ needs.version.outputs.version }}', @@ -125,26 +181,34 @@ local releaseLibStep = common.releaseLibStep; SKIP_ARM: skipArm, }) //TODO: the workdir here is loki specific - + step.withRun(||| - cat < $NFPM_SIGNING_KEY_FILE - make dist packages - EOF - ||| % buildImage), - - step.new('upload build artifacts', 'google-github-actions/upload-cloud-storage@v2') + + step.withRun( + ( + if useGCR then ||| + gcloud auth configure-docker + ||| else '' + ) + + ||| + cat < $NFPM_SIGNING_KEY_FILE + make %s + EOF + ||| % [buildImage, std.join(' ', makeTargets)] + ), + + step.new('upload artifacts', 'google-github-actions/upload-cloud-storage@v2') + + step.withIf('${{ fromJSON(needs.version.outputs.pr_created) }}') + step.with({ path: 'release/dist', destination: 'loki-build-artifacts/${{ github.sha }}', //TODO: make bucket configurable diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/common.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/common.libsonnet index e3346f2bd5e4..48cca43890ed 100644 --- a/.github/vendor/github.com/grafana/loki-release/workflows/common.libsonnet +++ b/.github/vendor/github.com/grafana/loki-release/workflows/common.libsonnet @@ -61,8 +61,8 @@ withEnv: function(env) { env: env, }, - withSecrets: function(env) { - secrets: env, + withSecrets: function(secrets) { + secrets: secrets, }, }, @@ -86,6 +86,7 @@ + $.step.with({ repository: 'grafana/loki-release', path: 'lib', + ref: '${{ env.RELEASE_LIB_REF }}', }), setupNode: $.step.new('setup node', 'actions/setup-node@v4') @@ -121,4 +122,39 @@ + $.step.withRun(||| git config --global --add safe.directory "$GITHUB_WORKSPACE" |||), + + githubAppToken: $.step.new('get github app token', 'actions/github-app-token@v1') + + $.step.withId('get_github_app_token') + + $.step.withIf('${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}') + + $.step.with({ + 'app-id': '${{ secrets.APP_ID }}', + 'private-key': '${{ secrets.APP_PRIVATE_KEY }}', + // By setting owner, we should get access to all repositories in current owner's installation: https://github.com/marketplace/actions/create-github-app-token#create-a-token-for-all-repositories-in-the-current-owners-installation + owner: '${{ github.repository_owner }}', + }), + + setToken: $.step.new('set github token') + + $.step.withId('github_app_token') + + $.step.withRun(||| + if [[ "${USE_GITHUB_APP_TOKEN}" == "true" ]]; then + echo "token=${{ steps.get_github_app_token.outputs.token }}" >> $GITHUB_OUTPUT + else + echo "token=${{ secrets.GH_TOKEN }}" >> $GITHUB_OUTPUT + fi + |||), + + validationJob: function(useGCR=false) + $.job.new() + + $.job.withContainer({ + image: '${{ inputs.build_image }}', + } + if useGCR then { + credentials: { + username: '_json_key', + password: '${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}', + }, + } else {}) + + $.job.withEnv({ + BUILD_IN_CONTAINER: false, + SKIP_VALIDATION: '${{ inputs.skip_validation }}', + }), } diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/main.jsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/main.jsonnet index 0a033b81221f..a53d541a8a5a 100644 --- a/.github/vendor/github.com/grafana/loki-release/workflows/main.jsonnet +++ b/.github/vendor/github.com/grafana/loki-release/workflows/main.jsonnet @@ -5,16 +5,24 @@ build: import 'build.libsonnet', release: import 'release.libsonnet', validate: import 'validate.libsonnet', + validateGel: import 'validate-gel.libsonnet', releasePRWorkflow: function( branches=['release-[0-9]+.[0-9]+.x', 'k[0-9]+'], buildImage='grafana/loki-build-image:0.33.0', + changelogPath='CHANGELOG.md', checkTemplate='./.github/workflows/check.yml', + distMakeTargets=['dist', 'packages'], dockerUsername='grafana', + golangCiLintVersion='v1.55.1', + imageBuildTimeoutMin=25, imageJobs={}, imagePrefix='grafana', + releaseLibRef='main', releaseRepo='grafana/loki-release', - skipArm=true, + skipArm=false, skipValidation=false, + useGitHubAppToken=true, + useGCR=false, versioningStrategy='always-bump-patch', ) { name: 'create release PR', @@ -32,10 +40,14 @@ group: 'create-release-pr-${{ github.sha }}', }, env: { - RELEASE_REPO: releaseRepo, + BUILD_TIMEOUT: imageBuildTimeoutMin, + CHANGELOG_PATH: changelogPath, DOCKER_USERNAME: dockerUsername, IMAGE_PREFIX: imagePrefix, + RELEASE_LIB_REF: releaseLibRef, + RELEASE_REPO: releaseRepo, SKIP_VALIDATION: skipValidation, + USE_GITHUB_APP_TOKEN: useGitHubAppToken, VERSIONING_STRATEGY: versioningStrategy, }, local validationSteps = ['check'], @@ -43,20 +55,31 @@ check: {} + $.job.withUses(checkTemplate) + $.job.with({ skip_validation: skipValidation, - }), + build_image: buildImage, + golang_ci_lint_version: golangCiLintVersion, + release_lib_ref: releaseLibRef, + use_github_app_token: useGitHubAppToken, + }) + + if useGCR then $.job.withSecrets({ + GCS_SERVICE_ACCOUNT_KEY: '${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}', + }) else {}, version: $.build.version + $.common.job.withNeeds(validationSteps), - dist: $.build.dist(buildImage, skipArm) + $.common.job.withNeeds(['version']), + dist: $.build.dist(buildImage, skipArm, useGCR, distMakeTargets) + $.common.job.withNeeds(['version']), } + std.mapWithKey(function(name, job) job + $.common.job.withNeeds(['version']), imageJobs) + { local buildImageSteps = ['dist'] + std.objectFields(imageJobs), 'create-release-pr': $.release.createReleasePR + $.common.job.withNeeds(buildImageSteps), }, }, releaseWorkflow: function( - releaseRepo='grafana/loki-release', - dockerUsername='grafana', - imagePrefix='grafana', branches=['release-[0-9].[0-9].x', 'k[0-9]*'], - getDockerCredsFromVault=false + dockerUsername='grafanabot', + getDockerCredsFromVault=false, + imagePrefix='grafana', + publishBucket='', + publishToGCS=false, + releaseLibRef='main', + releaseRepo='grafana/loki-release', + useGitHubAppToken=true, ) { name: 'create release', on: { @@ -73,28 +96,114 @@ group: 'create-release-${{ github.sha }}', }, env: { - RELEASE_REPO: releaseRepo, IMAGE_PREFIX: imagePrefix, + RELEASE_LIB_REF: releaseLibRef, + RELEASE_REPO: releaseRepo, + USE_GITHUB_APP_TOKEN: useGitHubAppToken, + } + if publishToGCS then { + PUBLISH_BUCKET: publishBucket, + PUBLISH_TO_GCS: true, + } else { + PUBLISH_TO_GCS: false, }, jobs: { shouldRelease: $.release.shouldRelease, createRelease: $.release.createRelease, publishImages: $.release.publishImages(getDockerCredsFromVault, dockerUsername), + publishRelease: $.release.publishRelease, }, }, - check: function( - buildImage='grafana/loki-build-image:0.33.0', - ) { + check: { + name: 'check', + on: { + workflow_call: { + inputs: { + build_image: { + description: 'loki build image to use', + required: true, + type: 'string', + }, + skip_validation: { + default: false, + description: 'skip validation steps', + required: false, + type: 'boolean', + }, + golang_ci_lint_version: { + default: 'v1.55.1', + description: 'version of golangci-lint to use', + required: false, + type: 'string', + }, + release_lib_ref: { + default: 'main', + description: 'git ref of release library to use', + required: false, + type: 'string', + }, + use_github_app_token: { + default: true, + description: 'whether to use the GitHub App token for GH_TOKEN secret', + required: false, + type: 'boolean', + }, + }, + }, + }, + permissions: { + contents: 'write', + 'pull-requests': 'write', + 'id-token': 'write', + }, + concurrency: { + group: 'check-${{ github.sha }}', + }, + env: { + RELEASE_LIB_REF: '${{ inputs.release_lib_ref }}', + USE_GITHUB_APP_TOKEN: '${{ inputs.use_github_app_token }}', + }, + jobs: $.validate, + }, + checkGel: { name: 'check', on: { workflow_call: { inputs: { + build_image: { + description: 'loki build image to use', + required: true, + type: 'string', + }, skip_validation: { default: false, description: 'skip validation steps', required: false, type: 'boolean', }, + golang_ci_lint_version: { + default: 'v1.55.1', + description: 'version of golangci-lint to use', + required: false, + type: 'string', + }, + release_lib_ref: { + default: 'main', + description: 'git ref of release library to use', + required: false, + type: 'string', + }, + use_github_app_token: { + default: true, + description: 'whether to use the GitHub App token for GH_TOKEN secret', + required: false, + type: 'boolean', + }, + }, + secrets: { + GCS_SERVICE_ACCOUNT_KEY: { + description: 'GCS service account key', + required: true, + }, }, }, }, @@ -106,6 +215,10 @@ concurrency: { group: 'check-${{ github.sha }}', }, - jobs: $.validate(buildImage), + env: { + RELEASE_LIB_REF: '${{ inputs.release_lib_ref }}', + USE_GITHUB_APP_TOKEN: '${{ inputs.use_github_app_token }}', + }, + jobs: $.validateGel, }, } diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet index 6bf2daa8f033..96edd7f422db 100644 --- a/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet +++ b/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet @@ -19,6 +19,8 @@ local pullRequestFooter = 'Merging this PR will release the [artifacts](https:// common.fetchReleaseLib, common.setupNode, common.extractBranchName, + common.githubAppToken, + common.setToken, releaseLibStep('release please') + step.withId('release') @@ -30,19 +32,22 @@ local pullRequestFooter = 'Merging this PR will release the [artifacts](https:// //TODO backport action should not bring over autorelease: pending label + step.withRun(||| npm install - echo "Pull request footer: %s" npm exec -- release-please release-pr \ + --changelog-path "${CHANGELOG_PATH}" \ --consider-all-branches \ - --label "backport main,autorelease: pending,type/docs" \ + --group-pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ + --label "backport main,autorelease: pending,product-approved" \ + --manifest-file .release-please-manifest.json \ --pull-request-footer "%s" \ + --pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ --release-type simple \ --repo-url "${{ env.RELEASE_REPO }}" \ - --target-branch "${{ steps.extract_branch.outputs.branch }}" \ - --token "${{ secrets.GH_TOKEN }}" \ - --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" \ --separate-pull-requests false \ - --debug - ||| % [pullRequestFooter, pullRequestFooter]), + --target-branch "${{ steps.extract_branch.outputs.branch }}" \ + --token "${{ steps.github_app_token.outputs.token }}" \ + --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" + + ||| % pullRequestFooter), ]), shouldRelease: job.new() @@ -61,6 +66,8 @@ local pullRequestFooter = 'Merging this PR will release the [artifacts](https:// shouldRelease: '${{ steps.should_release.outputs.shouldRelease }}', sha: '${{ steps.should_release.outputs.sha }}', name: '${{ steps.should_release.outputs.name }}', + prNumber: '${{ steps.should_release.outputs.prNumber }}', + isLatest: '${{ steps.should_release.outputs.isLatest }}', branch: '${{ steps.extract_branch.outputs.branch }}', }), @@ -73,6 +80,8 @@ local pullRequestFooter = 'Merging this PR will release the [artifacts](https:// common.setupNode, common.googleAuth, common.setupGoogleCloudSdk, + common.githubAppToken, + common.setToken, // exits with code 1 if the url does not match // meaning there are no artifacts for that sha @@ -83,30 +92,64 @@ local pullRequestFooter = 'Merging this PR will release the [artifacts](https:// gsutil cp -r gs://loki-build-artifacts/${{ needs.shouldRelease.outputs.sha }}/dist . |||), + releaseStep('check if release exists') + + step.withId('check_release') + + step.withEnv({ + GH_TOKEN: '${{ steps.github_app_token.outputs.token }}', + }) + + step.withRun(||| + set +e + isDraft="$(gh release view --json="isDraft" --jq=".isDraft" ${{ needs.shouldRelease.outputs.name }} 2>&1)" + set -e + if [[ "$isDraft" == "release not found" ]]; then + echo "exists=false" >> $GITHUB_OUTPUT + else + echo "exists=true" >> $GITHUB_OUTPUT + fi + + if [[ "$isDraft" == "true" ]]; then + echo "draft=true" >> $GITHUB_OUTPUT + fi + |||), + releaseLibStep('create release') + step.withId('release') + + step.withIf('${{ !fromJSON(steps.check_release.outputs.exists) }}') + step.withRun(||| npm install npm exec -- release-please github-release \ --draft \ --release-type simple \ - --repo-url="${{ env.RELEASE_REPO }}" \ + --repo-url "${{ env.RELEASE_REPO }}" \ --target-branch "${{ needs.shouldRelease.outputs.branch }}" \ - --token="${{ secrets.GH_TOKEN }}" + --token "${{ steps.github_app_token.outputs.token }}" \ + --shas-to-tag "${{ needs.shouldRelease.outputs.prNumber }}:${{ needs.shouldRelease.outputs.sha }}" |||), releaseStep('upload artifacts') + step.withId('upload') + step.withEnv({ - GH_TOKEN: '${{ secrets.GH_TOKEN }}', + GH_TOKEN: '${{ steps.github_app_token.outputs.token }}', }) + step.withRun(||| - gh release upload ${{ needs.shouldRelease.outputs.name }} dist/* - gh release edit ${{ needs.shouldRelease.outputs.name }} --draft=false + gh release upload --clobber ${{ needs.shouldRelease.outputs.name }} dist/* |||), + + step.new('release artifacts', 'google-github-actions/upload-cloud-storage@v2') + + step.withIf('${{ fromJSON(env.PUBLISH_TO_GCS) }}') + + step.with({ + path: 'release/dist', + destination: '${{ env.PUBLISH_BUCKET }}', + parent: false, + process_gcloudignore: false, + }), ]) + job.withOutputs({ sha: '${{ needs.shouldRelease.outputs.sha }}', + name: '${{ needs.shouldRelease.outputs.name }}', + isLatest: '${{ needs.shouldRelease.outputs.isLatest }}', + draft: '${{ steps.check_release.outputs.draft }}', + exists: '${{ steps.check_release.outputs.exists }}', }), publishImages: function(getDockerCredsFromVault=false, dockerUsername='grafanabot') @@ -141,4 +184,20 @@ local pullRequestFooter = 'Merging this PR will release the [artifacts](https:// }), ] ), + + publishRelease: job.new() + + job.withNeeds(['createRelease', 'publishImages']) + + job.withSteps([ + common.fetchReleaseRepo, + common.githubAppToken, + common.setToken, + releaseStep('publish release') + + step.withIf('${{ !fromJSON(needs.createRelease.outputs.exists) || (needs.createRelease.outputs.draft && fromJSON(needs.createRelease.outputs.draft)) }}') + + step.withEnv({ + GH_TOKEN: '${{ steps.github_app_token.outputs.token }}', + }) + + step.withRun(||| + gh release edit ${{ needs.createRelease.outputs.name }} --draft=false --latest=${{ needs.createRelease.outputs.isLatest }} + |||), + ]), } diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/validate-gel.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/validate-gel.libsonnet new file mode 100644 index 000000000000..af741478a7cb --- /dev/null +++ b/.github/vendor/github.com/grafana/loki-release/workflows/validate-gel.libsonnet @@ -0,0 +1,97 @@ +local common = import 'common.libsonnet'; +local job = common.job; +local step = common.step; +local _validationJob = common.validationJob; + +local setupValidationDeps = function(job) job { + steps: [ + common.checkout, + common.fetchReleaseLib, + common.fixDubiousOwnership, + step.new('install dependencies') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.withRun(||| + apt update + apt install -qy tar xz-utils + |||), + step.new('install shellcheck', './lib/actions/install-binary') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.with({ + binary: 'shellcheck', + version: '0.9.0', + download_url: 'https://github.com/koalaman/shellcheck/releases/download/v${version}/shellcheck-v${version}.linux.x86_64.tar.xz', + tarball_binary_path: '*/${binary}', + smoke_test: '${binary} --version', + tar_args: 'xvf', + }), + step.new('install jsonnetfmt', './lib/actions/install-binary') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.with({ + binary: 'jsonnetfmt', + version: '0.18.0', + download_url: 'https://github.com/google/go-jsonnet/releases/download/v${version}/go-jsonnet_${version}_Linux_x86_64.tar.gz', + tarball_binary_path: '${binary}', + smoke_test: '${binary} --version', + }), + ] + job.steps, +}; + +local validationJob = _validationJob(true); + + +{ + local validationMakeStep = function(name, target) + step.new(name) + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.withRun(common.makeTarget(target)), + + test: setupValidationDeps( + validationJob + + job.withSteps([ + validationMakeStep('test', 'test'), + ]) + ), + + lint: setupValidationDeps( + validationJob + + job.withSteps( + [ + step.new('golangci-lint', 'golangci/golangci-lint-action@08e2f20817b15149a52b5b3ebe7de50aff2ba8c5') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.with({ + version: '${{ inputs.golang_ci_lint_version }}', + 'only-new-issues': true, + args: '--skip-files cmd/enterprise-logs/fips.go', + }), + ], + ) + ), + + check: setupValidationDeps( + validationJob + + job.withSteps([ + validationMakeStep('build enterprise logs', 'all'), + validationMakeStep('check docs', 'check-docs'), + ]) + { + steps+: [ + step.new('build docs website') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.withRun(||| + cat <> $GITHUB_OUTPUT working-directory: "release" + - id: "get_github_app_token" + if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}" + name: "get github app token" + uses: "actions/github-app-token@v1" + with: + app-id: "${{ secrets.APP_ID }}" + owner: "${{ github.repository_owner }}" + private-key: "${{ secrets.APP_PRIVATE_KEY }}" + - id: "github_app_token" + name: "set github token" + run: | + if [[ "${USE_GITHUB_APP_TOKEN}" == "true" ]]; then + echo "token=${{ steps.get_github_app_token.outputs.token }}" >> $GITHUB_OUTPUT + else + echo "token=${{ secrets.GH_TOKEN }}" >> $GITHUB_OUTPUT + fi - env: SHA: "${{ github.sha }}" id: "release" name: "release please" run: | npm install - echo "Pull request footer: Merging this PR will release the [artifacts](https://console.cloud.google.com/storage/browser/loki-build-artifacts/${SHA}) of ${SHA}" npm exec -- release-please release-pr \ + --changelog-path "${CHANGELOG_PATH}" \ --consider-all-branches \ - --label "backport main,autorelease: pending,type/docs" \ + --group-pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ + --label "backport main,autorelease: pending,product-approved" \ + --manifest-file .release-please-manifest.json \ --pull-request-footer "Merging this PR will release the [artifacts](https://console.cloud.google.com/storage/browser/loki-build-artifacts/${SHA}) of ${SHA}" \ + --pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ --release-type simple \ --repo-url "${{ env.RELEASE_REPO }}" \ - --target-branch "${{ steps.extract_branch.outputs.branch }}" \ - --token "${{ secrets.GH_TOKEN }}" \ - --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" \ --separate-pull-requests false \ - --debug + --target-branch "${{ steps.extract_branch.outputs.branch }}" \ + --token "${{ steps.github_app_token.outputs.token }}" \ + --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" + working-directory: "lib" dist: needs: @@ -78,6 +105,10 @@ jobs: uses: "google-github-actions/auth@v2" with: credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up Cloud SDK" + uses: "google-github-actions/setup-gcloud@v2" + with: + version: ">= 452.0.0" - id: "get-secrets" name: "get nfpm signing keys" uses: "grafana/shared-workflows/actions/get-vault-secrets@main" @@ -91,6 +122,7 @@ jobs: IMAGE_TAG: "${{ needs.version.outputs.version }}" NFPM_SIGNING_KEY_FILE: "nfpm-private-key.key" SKIP_ARM: false + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "build artifacts" run: | cat <> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT working-directory: "release" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - env: + IMAGE_TAG: "${{ needs.version.outputs.version }}" + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" - timeout-minutes: 25 + timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v5" with: + build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" file: "release/clients/cmd/fluent-bit/Dockerfile" - outputs: "type=docker,dest=release/images/fluent-bit-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + outputs: "type=docker,dest=release/images/fluent-bit-plugin-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" platforms: "${{ matrix.platform }}" - tags: "${{ env.IMAGE_PREFIX }}/fluent-bit:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + tags: "${{ env.IMAGE_PREFIX }}/fluent-bit-plugin-loki:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "upload artifacts" uses: "google-github-actions/upload-cloud-storage@v2" with: destination: "loki-build-artifacts/${{ github.sha }}/images" - path: "release/images/fluent-bit-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + path: "release/images/fluent-bit-plugin-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" process_gcloudignore: false strategy: fail-fast: true @@ -183,6 +220,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "pull code to release" uses: "actions/checkout@v4" @@ -210,22 +248,25 @@ jobs: echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT working-directory: "release" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - env: + IMAGE_TAG: "${{ needs.version.outputs.version }}" + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" - timeout-minutes: 25 + timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v5" with: + build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" file: "release/clients/cmd/fluentd/Dockerfile" - outputs: "type=docker,dest=release/images/fluentd-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + outputs: "type=docker,dest=release/images/fluent-plugin-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" platforms: "${{ matrix.platform }}" - tags: "${{ env.IMAGE_PREFIX }}/fluentd:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + tags: "${{ env.IMAGE_PREFIX }}/fluent-plugin-loki:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "upload artifacts" uses: "google-github-actions/upload-cloud-storage@v2" with: destination: "loki-build-artifacts/${{ github.sha }}/images" - path: "release/images/fluentd-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + path: "release/images/fluent-plugin-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" process_gcloudignore: false strategy: fail-fast: true @@ -241,6 +282,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "pull code to release" uses: "actions/checkout@v4" @@ -268,11 +310,14 @@ jobs: echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT working-directory: "release" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - env: + IMAGE_TAG: "${{ needs.version.outputs.version }}" + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" - timeout-minutes: 25 + timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v5" with: + build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" file: "release/cmd/logcli/Dockerfile" outputs: "type=docker,dest=release/images/logcli-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" @@ -301,6 +346,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "pull code to release" uses: "actions/checkout@v4" @@ -328,22 +374,25 @@ jobs: echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT working-directory: "release" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - env: + IMAGE_TAG: "${{ needs.version.outputs.version }}" + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" - timeout-minutes: 25 + timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v5" with: + build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" file: "release/clients/cmd/logstash/Dockerfile" - outputs: "type=docker,dest=release/images/logstash-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + outputs: "type=docker,dest=release/images/logstash-output-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" platforms: "${{ matrix.platform }}" - tags: "${{ env.IMAGE_PREFIX }}/logstash:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + tags: "${{ env.IMAGE_PREFIX }}/logstash-output-loki:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "upload artifacts" uses: "google-github-actions/upload-cloud-storage@v2" with: destination: "loki-build-artifacts/${{ github.sha }}/images" - path: "release/images/logstash-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + path: "release/images/logstash-output-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" process_gcloudignore: false strategy: fail-fast: true @@ -359,6 +408,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "pull code to release" uses: "actions/checkout@v4" @@ -386,11 +436,14 @@ jobs: echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT working-directory: "release" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - env: + IMAGE_TAG: "${{ needs.version.outputs.version }}" + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" - timeout-minutes: 25 + timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v5" with: + build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" file: "release/cmd/loki/Dockerfile" outputs: "type=docker,dest=release/images/loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" @@ -419,6 +472,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "pull code to release" uses: "actions/checkout@v4" @@ -446,11 +500,14 @@ jobs: echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT working-directory: "release" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - env: + IMAGE_TAG: "${{ needs.version.outputs.version }}" + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" - timeout-minutes: 25 + timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v5" with: + build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" file: "release/cmd/loki-canary/Dockerfile" outputs: "type=docker,dest=release/images/loki-canary-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" @@ -479,6 +536,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "pull code to release" uses: "actions/checkout@v4" @@ -506,11 +564,14 @@ jobs: echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT working-directory: "release" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - env: + IMAGE_TAG: "${{ needs.version.outputs.version }}" + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" - timeout-minutes: 25 + timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v5" with: + build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" file: "release/cmd/loki-canary-boringcrypto/Dockerfile" outputs: "type=docker,dest=release/images/loki-canary-boringcrypto-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" @@ -530,64 +591,6 @@ jobs: - "linux/amd64" - "linux/arm64" - "linux/arm" - loki-operator: - needs: - - "version" - runs-on: "ubuntu-latest" - steps: - - name: "pull release library code" - uses: "actions/checkout@v4" - with: - path: "lib" - repository: "grafana/loki-release" - - name: "pull code to release" - uses: "actions/checkout@v4" - with: - path: "release" - repository: "${{ env.RELEASE_REPO }}" - - name: "setup node" - uses: "actions/setup-node@v4" - with: - node-version: 20 - - name: "auth gcs" - uses: "google-github-actions/auth@v2" - with: - credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" - - name: "Set up QEMU" - uses: "docker/setup-qemu-action@v3" - - name: "set up docker buildx" - uses: "docker/setup-buildx-action@v3" - - id: "platform" - name: "parse image platform" - run: | - mkdir -p images - - platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" - echo "platform=${platform}" >> $GITHUB_OUTPUT - echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT - working-directory: "release" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" - name: "Build and export" - timeout-minutes: 25 - uses: "docker/build-push-action@v5" - with: - context: "release/operator" - file: "release/operator/Dockerfile" - outputs: "type=docker,dest=release/images/loki-operator-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" - platforms: "${{ matrix.platform }}" - tags: "${{ env.IMAGE_PREFIX }}/loki-operator:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" - name: "upload artifacts" - uses: "google-github-actions/upload-cloud-storage@v2" - with: - destination: "loki-build-artifacts/${{ github.sha }}/images" - path: "release/images/loki-operator-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" - process_gcloudignore: false - strategy: - fail-fast: true - matrix: - platform: - - "linux/amd64" promtail: needs: - "version" @@ -597,6 +600,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "pull code to release" uses: "actions/checkout@v4" @@ -624,11 +628,14 @@ jobs: echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT working-directory: "release" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - env: + IMAGE_TAG: "${{ needs.version.outputs.version }}" + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" - timeout-minutes: 25 + timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v5" with: + build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" file: "release/clients/cmd/promtail/Dockerfile" outputs: "type=docker,dest=release/images/promtail-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" @@ -657,6 +664,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "pull code to release" uses: "actions/checkout@v4" @@ -684,22 +692,25 @@ jobs: echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT working-directory: "release" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - env: + IMAGE_TAG: "${{ needs.version.outputs.version }}" + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" - timeout-minutes: 25 + timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v5" with: + build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" file: "release/cmd/querytee/Dockerfile" - outputs: "type=docker,dest=release/images/querytee-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + outputs: "type=docker,dest=release/images/loki-query-tee-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" platforms: "${{ matrix.platform }}" - tags: "${{ env.IMAGE_PREFIX }}/querytee:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + tags: "${{ env.IMAGE_PREFIX }}/loki-query-tee:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "upload artifacts" uses: "google-github-actions/upload-cloud-storage@v2" with: destination: "loki-build-artifacts/${{ github.sha }}/images" - path: "release/images/querytee-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + path: "release/images/loki-query-tee-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" process_gcloudignore: false strategy: fail-fast: true @@ -718,6 +729,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "pull code to release" uses: "actions/checkout@v4" @@ -733,6 +745,22 @@ jobs: run: | echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT working-directory: "release" + - id: "get_github_app_token" + if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}" + name: "get github app token" + uses: "actions/github-app-token@v1" + with: + app-id: "${{ secrets.APP_ID }}" + owner: "${{ github.repository_owner }}" + private-key: "${{ secrets.APP_PRIVATE_KEY }}" + - id: "github_app_token" + name: "set github token" + run: | + if [[ "${USE_GITHUB_APP_TOKEN}" == "true" ]]; then + echo "token=${{ steps.get_github_app_token.outputs.token }}" >> $GITHUB_OUTPUT + else + echo "token=${{ secrets.GH_TOKEN }}" >> $GITHUB_OUTPUT + fi - id: "version" name: "get release version" run: | @@ -741,12 +769,18 @@ jobs: --consider-all-branches \ --dry-run \ --dry-run-output release.json \ + --group-pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ + --manifest-file .release-please-manifest.json \ + --pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ --release-type simple \ - --repo-url="${{ env.RELEASE_REPO }}" \ + --repo-url "${{ env.RELEASE_REPO }}" \ + --separate-pull-requests false \ --target-branch "${{ steps.extract_branch.outputs.branch }}" \ - --token="${{ secrets.GH_TOKEN }}" \ + --token "${{ steps.github_app_token.outputs.token }}" \ --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" + cat release.json + if [[ `jq length release.json` -gt 1 ]]; then echo 'release-please would create more than 1 PR, so cannot determine correct version' echo "pr_created=false" >> $GITHUB_OUTPUT diff --git a/.github/workflows/patch-release-pr.yml b/.github/workflows/patch-release-pr.yml index 411fff87d410..cf5119cf4301 100644 --- a/.github/workflows/patch-release-pr.yml +++ b/.github/workflows/patch-release-pr.yml @@ -1,16 +1,24 @@ concurrency: group: "create-release-pr-${{ github.sha }}" env: + BUILD_TIMEOUT: 40 + CHANGELOG_PATH: "CHANGELOG.md" DOCKER_USERNAME: "grafana" IMAGE_PREFIX: "grafana" + RELEASE_LIB_REF: "main" RELEASE_REPO: "grafana/loki" SKIP_VALIDATION: false + USE_GITHUB_APP_TOKEN: true VERSIONING_STRATEGY: "always-bump-patch" jobs: check: - uses: "grafana/loki-release/.github/workflows/check.yml@release-1.10.x" + uses: "grafana/loki-release/.github/workflows/check.yml@main" with: + build_image: "grafana/loki-build-image:0.33.0" + golang_ci_lint_version: "v1.55.1" + release_lib_ref: "main" skip_validation: false + use_github_app_token: true create-release-pr: needs: - "dist" @@ -21,7 +29,6 @@ jobs: - "loki" - "loki-canary" - "loki-canary-boringcrypto" - - "loki-operator" - "promtail" - "querytee" runs-on: "ubuntu-latest" @@ -35,6 +42,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "setup node" uses: "actions/setup-node@v4" @@ -45,24 +53,43 @@ jobs: run: | echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT working-directory: "release" + - id: "get_github_app_token" + if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}" + name: "get github app token" + uses: "actions/github-app-token@v1" + with: + app-id: "${{ secrets.APP_ID }}" + owner: "${{ github.repository_owner }}" + private-key: "${{ secrets.APP_PRIVATE_KEY }}" + - id: "github_app_token" + name: "set github token" + run: | + if [[ "${USE_GITHUB_APP_TOKEN}" == "true" ]]; then + echo "token=${{ steps.get_github_app_token.outputs.token }}" >> $GITHUB_OUTPUT + else + echo "token=${{ secrets.GH_TOKEN }}" >> $GITHUB_OUTPUT + fi - env: SHA: "${{ github.sha }}" id: "release" name: "release please" run: | npm install - echo "Pull request footer: Merging this PR will release the [artifacts](https://console.cloud.google.com/storage/browser/loki-build-artifacts/${SHA}) of ${SHA}" npm exec -- release-please release-pr \ + --changelog-path "${CHANGELOG_PATH}" \ --consider-all-branches \ - --label "backport main,autorelease: pending,type/docs" \ + --group-pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ + --label "backport main,autorelease: pending,product-approved" \ + --manifest-file .release-please-manifest.json \ --pull-request-footer "Merging this PR will release the [artifacts](https://console.cloud.google.com/storage/browser/loki-build-artifacts/${SHA}) of ${SHA}" \ + --pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ --release-type simple \ --repo-url "${{ env.RELEASE_REPO }}" \ - --target-branch "${{ steps.extract_branch.outputs.branch }}" \ - --token "${{ secrets.GH_TOKEN }}" \ - --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" \ --separate-pull-requests false \ - --debug + --target-branch "${{ steps.extract_branch.outputs.branch }}" \ + --token "${{ steps.github_app_token.outputs.token }}" \ + --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" + working-directory: "lib" dist: needs: @@ -78,6 +105,10 @@ jobs: uses: "google-github-actions/auth@v2" with: credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up Cloud SDK" + uses: "google-github-actions/setup-gcloud@v2" + with: + version: ">= 452.0.0" - id: "get-secrets" name: "get nfpm signing keys" uses: "grafana/shared-workflows/actions/get-vault-secrets@main" @@ -91,6 +122,7 @@ jobs: IMAGE_TAG: "${{ needs.version.outputs.version }}" NFPM_SIGNING_KEY_FILE: "nfpm-private-key.key" SKIP_ARM: false + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "build artifacts" run: | cat <> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT working-directory: "release" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - env: + IMAGE_TAG: "${{ needs.version.outputs.version }}" + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" - timeout-minutes: 25 + timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v5" with: + build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" file: "release/clients/cmd/fluent-bit/Dockerfile" - outputs: "type=docker,dest=release/images/fluent-bit-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + outputs: "type=docker,dest=release/images/fluent-bit-plugin-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" platforms: "${{ matrix.platform }}" - tags: "${{ env.IMAGE_PREFIX }}/fluent-bit:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + tags: "${{ env.IMAGE_PREFIX }}/fluent-bit-plugin-loki:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "upload artifacts" uses: "google-github-actions/upload-cloud-storage@v2" with: destination: "loki-build-artifacts/${{ github.sha }}/images" - path: "release/images/fluent-bit-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + path: "release/images/fluent-bit-plugin-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" process_gcloudignore: false strategy: fail-fast: true @@ -183,6 +220,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "pull code to release" uses: "actions/checkout@v4" @@ -210,22 +248,25 @@ jobs: echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT working-directory: "release" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - env: + IMAGE_TAG: "${{ needs.version.outputs.version }}" + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" - timeout-minutes: 25 + timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v5" with: + build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" file: "release/clients/cmd/fluentd/Dockerfile" - outputs: "type=docker,dest=release/images/fluentd-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + outputs: "type=docker,dest=release/images/fluent-plugin-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" platforms: "${{ matrix.platform }}" - tags: "${{ env.IMAGE_PREFIX }}/fluentd:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + tags: "${{ env.IMAGE_PREFIX }}/fluent-plugin-loki:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "upload artifacts" uses: "google-github-actions/upload-cloud-storage@v2" with: destination: "loki-build-artifacts/${{ github.sha }}/images" - path: "release/images/fluentd-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + path: "release/images/fluent-plugin-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" process_gcloudignore: false strategy: fail-fast: true @@ -241,6 +282,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "pull code to release" uses: "actions/checkout@v4" @@ -268,11 +310,14 @@ jobs: echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT working-directory: "release" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - env: + IMAGE_TAG: "${{ needs.version.outputs.version }}" + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" - timeout-minutes: 25 + timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v5" with: + build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" file: "release/cmd/logcli/Dockerfile" outputs: "type=docker,dest=release/images/logcli-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" @@ -301,6 +346,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "pull code to release" uses: "actions/checkout@v4" @@ -328,22 +374,25 @@ jobs: echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT working-directory: "release" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - env: + IMAGE_TAG: "${{ needs.version.outputs.version }}" + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" - timeout-minutes: 25 + timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v5" with: + build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" file: "release/clients/cmd/logstash/Dockerfile" - outputs: "type=docker,dest=release/images/logstash-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + outputs: "type=docker,dest=release/images/logstash-output-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" platforms: "${{ matrix.platform }}" - tags: "${{ env.IMAGE_PREFIX }}/logstash:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + tags: "${{ env.IMAGE_PREFIX }}/logstash-output-loki:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "upload artifacts" uses: "google-github-actions/upload-cloud-storage@v2" with: destination: "loki-build-artifacts/${{ github.sha }}/images" - path: "release/images/logstash-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + path: "release/images/logstash-output-loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" process_gcloudignore: false strategy: fail-fast: true @@ -359,6 +408,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "pull code to release" uses: "actions/checkout@v4" @@ -386,11 +436,14 @@ jobs: echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT working-directory: "release" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - env: + IMAGE_TAG: "${{ needs.version.outputs.version }}" + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" - timeout-minutes: 25 + timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v5" with: + build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" file: "release/cmd/loki/Dockerfile" outputs: "type=docker,dest=release/images/loki-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" @@ -419,6 +472,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "pull code to release" uses: "actions/checkout@v4" @@ -446,11 +500,14 @@ jobs: echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT working-directory: "release" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - env: + IMAGE_TAG: "${{ needs.version.outputs.version }}" + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" - timeout-minutes: 25 + timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v5" with: + build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" file: "release/cmd/loki-canary/Dockerfile" outputs: "type=docker,dest=release/images/loki-canary-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" @@ -479,6 +536,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "pull code to release" uses: "actions/checkout@v4" @@ -506,11 +564,14 @@ jobs: echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT working-directory: "release" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - env: + IMAGE_TAG: "${{ needs.version.outputs.version }}" + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" - timeout-minutes: 25 + timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v5" with: + build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" file: "release/cmd/loki-canary-boringcrypto/Dockerfile" outputs: "type=docker,dest=release/images/loki-canary-boringcrypto-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" @@ -530,64 +591,6 @@ jobs: - "linux/amd64" - "linux/arm64" - "linux/arm" - loki-operator: - needs: - - "version" - runs-on: "ubuntu-latest" - steps: - - name: "pull release library code" - uses: "actions/checkout@v4" - with: - path: "lib" - repository: "grafana/loki-release" - - name: "pull code to release" - uses: "actions/checkout@v4" - with: - path: "release" - repository: "${{ env.RELEASE_REPO }}" - - name: "setup node" - uses: "actions/setup-node@v4" - with: - node-version: 20 - - name: "auth gcs" - uses: "google-github-actions/auth@v2" - with: - credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" - - name: "Set up QEMU" - uses: "docker/setup-qemu-action@v3" - - name: "set up docker buildx" - uses: "docker/setup-buildx-action@v3" - - id: "platform" - name: "parse image platform" - run: | - mkdir -p images - - platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" - echo "platform=${platform}" >> $GITHUB_OUTPUT - echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT - working-directory: "release" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" - name: "Build and export" - timeout-minutes: 25 - uses: "docker/build-push-action@v5" - with: - context: "release/operator" - file: "release/operator/Dockerfile" - outputs: "type=docker,dest=release/images/loki-operator-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" - platforms: "${{ matrix.platform }}" - tags: "${{ env.IMAGE_PREFIX }}/loki-operator:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" - name: "upload artifacts" - uses: "google-github-actions/upload-cloud-storage@v2" - with: - destination: "loki-build-artifacts/${{ github.sha }}/images" - path: "release/images/loki-operator-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" - process_gcloudignore: false - strategy: - fail-fast: true - matrix: - platform: - - "linux/amd64" promtail: needs: - "version" @@ -597,6 +600,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "pull code to release" uses: "actions/checkout@v4" @@ -624,11 +628,14 @@ jobs: echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT working-directory: "release" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - env: + IMAGE_TAG: "${{ needs.version.outputs.version }}" + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" - timeout-minutes: 25 + timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v5" with: + build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" file: "release/clients/cmd/promtail/Dockerfile" outputs: "type=docker,dest=release/images/promtail-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" @@ -657,6 +664,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "pull code to release" uses: "actions/checkout@v4" @@ -684,22 +692,25 @@ jobs: echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT working-directory: "release" - - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - env: + IMAGE_TAG: "${{ needs.version.outputs.version }}" + if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" - timeout-minutes: 25 + timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v5" with: + build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" file: "release/cmd/querytee/Dockerfile" - outputs: "type=docker,dest=release/images/querytee-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + outputs: "type=docker,dest=release/images/loki-query-tee-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" platforms: "${{ matrix.platform }}" - tags: "${{ env.IMAGE_PREFIX }}/querytee:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + tags: "${{ env.IMAGE_PREFIX }}/loki-query-tee:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "upload artifacts" uses: "google-github-actions/upload-cloud-storage@v2" with: destination: "loki-build-artifacts/${{ github.sha }}/images" - path: "release/images/querytee-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + path: "release/images/loki-query-tee-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" process_gcloudignore: false strategy: fail-fast: true @@ -718,6 +729,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "pull code to release" uses: "actions/checkout@v4" @@ -733,6 +745,22 @@ jobs: run: | echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT working-directory: "release" + - id: "get_github_app_token" + if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}" + name: "get github app token" + uses: "actions/github-app-token@v1" + with: + app-id: "${{ secrets.APP_ID }}" + owner: "${{ github.repository_owner }}" + private-key: "${{ secrets.APP_PRIVATE_KEY }}" + - id: "github_app_token" + name: "set github token" + run: | + if [[ "${USE_GITHUB_APP_TOKEN}" == "true" ]]; then + echo "token=${{ steps.get_github_app_token.outputs.token }}" >> $GITHUB_OUTPUT + else + echo "token=${{ secrets.GH_TOKEN }}" >> $GITHUB_OUTPUT + fi - id: "version" name: "get release version" run: | @@ -741,12 +769,18 @@ jobs: --consider-all-branches \ --dry-run \ --dry-run-output release.json \ + --group-pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ + --manifest-file .release-please-manifest.json \ + --pull-request-title-pattern "chore\${scope}: release\${component} \${version}" \ --release-type simple \ - --repo-url="${{ env.RELEASE_REPO }}" \ + --repo-url "${{ env.RELEASE_REPO }}" \ + --separate-pull-requests false \ --target-branch "${{ steps.extract_branch.outputs.branch }}" \ - --token="${{ secrets.GH_TOKEN }}" \ + --token "${{ steps.github_app_token.outputs.token }}" \ --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" + cat release.json + if [[ `jq length release.json` -gt 1 ]]; then echo 'release-please would create more than 1 PR, so cannot determine correct version' echo "pr_created=false" >> $GITHUB_OUTPUT diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 64970d1bd719..69909c242ebe 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,13 +2,20 @@ concurrency: group: "create-release-${{ github.sha }}" env: IMAGE_PREFIX: "grafana" + PUBLISH_TO_GCS: false + RELEASE_LIB_REF: "main" RELEASE_REPO: "grafana/loki" + USE_GITHUB_APP_TOKEN: false jobs: createRelease: if: "${{ fromJSON(needs.shouldRelease.outputs.shouldRelease) }}" needs: - "shouldRelease" outputs: + draft: "${{ steps.check_release.outputs.draft }}" + exists: "${{ steps.check_release.outputs.exists }}" + isLatest: "${{ needs.shouldRelease.outputs.isLatest }}" + name: "${{ needs.shouldRelease.outputs.name }}" sha: "${{ needs.shouldRelease.outputs.sha }}" runs-on: "ubuntu-latest" steps: @@ -21,6 +28,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "setup node" uses: "actions/setup-node@v4" @@ -34,30 +42,73 @@ jobs: uses: "google-github-actions/setup-gcloud@v2" with: version: ">= 452.0.0" + - id: "get_github_app_token" + if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}" + name: "get github app token" + uses: "actions/github-app-token@v1" + with: + app-id: "${{ secrets.APP_ID }}" + owner: "${{ github.repository_owner }}" + private-key: "${{ secrets.APP_PRIVATE_KEY }}" + - id: "github_app_token" + name: "set github token" + run: | + if [[ "${USE_GITHUB_APP_TOKEN}" == "true" ]]; then + echo "token=${{ steps.get_github_app_token.outputs.token }}" >> $GITHUB_OUTPUT + else + echo "token=${{ secrets.GH_TOKEN }}" >> $GITHUB_OUTPUT + fi - name: "download binaries" run: | echo "downloading binaries to $(pwd)/dist" gsutil cp -r gs://loki-build-artifacts/${{ needs.shouldRelease.outputs.sha }}/dist . working-directory: "release" + - env: + GH_TOKEN: "${{ steps.github_app_token.outputs.token }}" + id: "check_release" + name: "check if release exists" + run: | + set +e + isDraft="$(gh release view --json="isDraft" --jq=".isDraft" ${{ needs.shouldRelease.outputs.name }} 2>&1)" + set -e + if [[ "$isDraft" == "release not found" ]]; then + echo "exists=false" >> $GITHUB_OUTPUT + else + echo "exists=true" >> $GITHUB_OUTPUT + fi + + if [[ "$isDraft" == "true" ]]; then + echo "draft=true" >> $GITHUB_OUTPUT + fi + working-directory: "release" - id: "release" + if: "${{ !fromJSON(steps.check_release.outputs.exists) }}" name: "create release" run: | npm install npm exec -- release-please github-release \ --draft \ --release-type simple \ - --repo-url="${{ env.RELEASE_REPO }}" \ + --repo-url "${{ env.RELEASE_REPO }}" \ --target-branch "${{ needs.shouldRelease.outputs.branch }}" \ - --token="${{ secrets.GH_TOKEN }}" + --token "${{ steps.github_app_token.outputs.token }}" \ + --shas-to-tag "${{ needs.shouldRelease.outputs.prNumber }}:${{ needs.shouldRelease.outputs.sha }}" working-directory: "lib" - env: - GH_TOKEN: "${{ secrets.GH_TOKEN }}" + GH_TOKEN: "${{ steps.github_app_token.outputs.token }}" id: "upload" name: "upload artifacts" run: | - gh release upload ${{ needs.shouldRelease.outputs.name }} dist/* - gh release edit ${{ needs.shouldRelease.outputs.name }} --draft=false + gh release upload --clobber ${{ needs.shouldRelease.outputs.name }} dist/* working-directory: "release" + - if: "${{ fromJSON(env.PUBLISH_TO_GCS) }}" + name: "release artifacts" + uses: "google-github-actions/upload-cloud-storage@v2" + with: + destination: "${{ env.PUBLISH_BUCKET }}" + parent: false + path: "release/dist" + process_gcloudignore: false publishImages: needs: - "createRelease" @@ -67,6 +118,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - name: "auth gcs" uses: "google-github-actions/auth@v2" @@ -91,10 +143,46 @@ jobs: with: imageDir: "images" imagePrefix: "${{ env.IMAGE_PREFIX }}" + publishRelease: + needs: + - "createRelease" + - "publishImages" + runs-on: "ubuntu-latest" + steps: + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - id: "get_github_app_token" + if: "${{ fromJSON(env.USE_GITHUB_APP_TOKEN) }}" + name: "get github app token" + uses: "actions/github-app-token@v1" + with: + app-id: "${{ secrets.APP_ID }}" + owner: "${{ github.repository_owner }}" + private-key: "${{ secrets.APP_PRIVATE_KEY }}" + - id: "github_app_token" + name: "set github token" + run: | + if [[ "${USE_GITHUB_APP_TOKEN}" == "true" ]]; then + echo "token=${{ steps.get_github_app_token.outputs.token }}" >> $GITHUB_OUTPUT + else + echo "token=${{ secrets.GH_TOKEN }}" >> $GITHUB_OUTPUT + fi + - env: + GH_TOKEN: "${{ steps.github_app_token.outputs.token }}" + if: "${{ !fromJSON(needs.createRelease.outputs.exists) || (needs.createRelease.outputs.draft && fromJSON(needs.createRelease.outputs.draft)) }}" + name: "publish release" + run: | + gh release edit ${{ needs.createRelease.outputs.name }} --draft=false --latest=${{ needs.createRelease.outputs.isLatest }} + working-directory: "release" shouldRelease: outputs: branch: "${{ steps.extract_branch.outputs.branch }}" + isLatest: "${{ steps.should_release.outputs.isLatest }}" name: "${{ steps.should_release.outputs.name }}" + prNumber: "${{ steps.should_release.outputs.prNumber }}" sha: "${{ steps.should_release.outputs.sha }}" shouldRelease: "${{ steps.should_release.outputs.shouldRelease }}" runs-on: "ubuntu-latest" @@ -108,6 +196,7 @@ jobs: uses: "actions/checkout@v4" with: path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" repository: "grafana/loki-release" - id: "extract_branch" name: "extract branch name" diff --git a/Makefile b/Makefile index 4789bf7e319d..07aaf9c58d8c 100644 --- a/Makefile +++ b/Makefile @@ -870,4 +870,5 @@ scan-vulnerabilities: trivy snyk .PHONY: release-workflows release-workflows: + pushd $(CURDIR)/.github && jb update && popd jsonnet -SJ .github/vendor -m .github/workflows .github/release-workflows.jsonnet From d501b5b1252cd4c19c8c679e4922594a0e24da63 Mon Sep 17 00:00:00 2001 From: Trevor Whitney Date: Tue, 19 Mar 2024 12:05:30 -0600 Subject: [PATCH 2/3] ci: bump release code to bring back check-format --- .github/jsonnetfile.lock.json | 4 ++-- .../loki-release/workflows/validate.libsonnet | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/jsonnetfile.lock.json b/.github/jsonnetfile.lock.json index 9b589d5b0246..7d51f277c194 100644 --- a/.github/jsonnetfile.lock.json +++ b/.github/jsonnetfile.lock.json @@ -8,8 +8,8 @@ "subdir": "workflows" } }, - "version": "6c326b84f35d69f9d01732b2805eb645a52b69cc", - "sum": "Y5sjhGpO7ziNdhSBcF3tfYkZRk25kT5AfYTo/i+qmhM=" + "version": "d9bfe17ed706fdfbd02445a576eb820dea4811a3", + "sum": "RCFmsb2FD3ZQmjSVfAo+5/GNi2mit/T9BZ474iE2u2o=" } ], "legacyImports": false diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/validate.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/validate.libsonnet index bde54efd6d3a..28df05dd215d 100644 --- a/.github/vendor/github.com/grafana/loki-release/workflows/validate.libsonnet +++ b/.github/vendor/github.com/grafana/loki-release/workflows/validate.libsonnet @@ -65,12 +65,12 @@ local validationJob = _validationJob(false); validationMakeStep('lint', 'lint'), validationMakeStep('lint jsonnet', 'lint-jsonnet'), validationMakeStep('lint scripts', 'lint-scripts'), - // step.new('format') - // + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') - // + step.withRun(||| - // git fetch origin - // make check-format - // |||), + step.new('check format') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.withRun(||| + git fetch origin + make check-format + |||), ] + [ step.new('golangci-lint', 'golangci/golangci-lint-action@08e2f20817b15149a52b5b3ebe7de50aff2ba8c5') + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') From b4ad28cdad3bfdecb1d8855431b230a1705c0779 Mon Sep 17 00:00:00 2001 From: Trevor Whitney Date: Tue, 19 Mar 2024 15:16:44 -0600 Subject: [PATCH 3/3] fix: OTLP integration tests --- integration/cluster/cluster.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/integration/cluster/cluster.go b/integration/cluster/cluster.go index 7e978b84eb32..c7a0ba2d17dd 100644 --- a/integration/cluster/cluster.go +++ b/integration/cluster/cluster.go @@ -62,6 +62,11 @@ limits_config: ingestion_burst_size_mb: 50 reject_old_samples: false allow_structured_metadata: true + otlp_config: + resource_attributes: + attributes_config: + - action: index_label + attributes: ["service.name"] storage_config: named_stores: