From 474031588aa442fcd590896267f5fa3a633b6d15 Mon Sep 17 00:00:00 2001 From: Madhav Jivrajani Date: Wed, 15 May 2024 16:07:29 +0530 Subject: [PATCH 1/2] .*: sync go toolchain version and add ability to verify versions This commit adds a script to sync the version present in .go-version across all go.mod files as the toolchain directive. As part of that, this commit also modifies go.mod files that did not have synced toolchain directives. Additionally, this also adds a script to verify all toolchain and go directives against the version present in .go-version as follows: (1) The go directive <= version in .go-version (2) The toolchain directive == version in .go-version This script runs as part of the `make verify` target, making it run as a presbumit by default. Signed-off-by: Madhav Jivrajani --- .github/workflows/static-analysis.yaml | 32 ++++++++++++++++ Makefile | 15 ++++++++ api/go.mod | 2 + client/pkg/go.mod | 2 + client/v2/go.mod | 2 + client/v3/go.mod | 2 + etcdctl/go.mod | 2 + etcdutl/go.mod | 2 + go.mod | 2 + pkg/go.mod | 2 + raft/go.mod | 2 + scripts/sync_go_toolchain_directive.sh | 14 +++++++ scripts/verify_go_versions.sh | 53 ++++++++++++++++++++++++++ server/go.mod | 2 + tests/go.mod | 2 + tools/mod/go.mod | 2 + 16 files changed, 138 insertions(+) create mode 100644 .github/workflows/static-analysis.yaml create mode 100755 scripts/sync_go_toolchain_directive.sh create mode 100755 scripts/verify_go_versions.sh diff --git a/.github/workflows/static-analysis.yaml b/.github/workflows/static-analysis.yaml new file mode 100644 index 00000000000..03e4deb0ffd --- /dev/null +++ b/.github/workflows/static-analysis.yaml @@ -0,0 +1,32 @@ +--- +name: Static Analysis +on: [push, pull_request] +permissions: read-all +jobs: + run: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # v4.1.5 + - id: goversion + run: echo "goversion=$(cat .go-version)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1 + with: + go-version: ${{ steps.goversion.outputs.goversion }} + - run: | + set -euo pipefail + + make verify + - run: | + set -euo pipefail + + make fix + + DIFF=$(git status --porcelain) + + if [ -n "$DIFF" ]; then + echo "These files were modified:" + echo + echo "$DIFF" + echo + exit 1 + fi diff --git a/Makefile b/Makefile index b3f3268273e..8cfd790e002 100644 --- a/Makefile +++ b/Makefile @@ -580,3 +580,18 @@ gofail-enable: install-gofail .PHONY: gofail-disable gofail-disable: install-gofail PASSES="toggle_failpoints" ./test.sh + +.PHONY: verify +verify: verify-go-versions + +.PHONY: verify-go-versions +verify-go-versions: + ./scripts/verify_go_versions.sh + +.PHONY: fix +fix: sync-toolchain-directive + ./scripts/fix.sh + +.PHONY: sync-toolchain-directive +sync-toolchain-directive: + ./scripts/sync_go_toolchain_directive.sh diff --git a/api/go.mod b/api/go.mod index 370bb4f862e..8bdab35a904 100644 --- a/api/go.mod +++ b/api/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/api/v3 go 1.21 +toolchain go1.21.10 + require ( github.com/coreos/go-semver v0.3.0 github.com/gogo/protobuf v1.3.2 diff --git a/client/pkg/go.mod b/client/pkg/go.mod index 8168a25efc5..b51579ca19a 100644 --- a/client/pkg/go.mod +++ b/client/pkg/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/client/pkg/v3 go 1.21 +toolchain go1.21.10 + require ( github.com/coreos/go-systemd/v22 v22.3.2 github.com/stretchr/testify v1.8.4 diff --git a/client/v2/go.mod b/client/v2/go.mod index 388647b265e..ba1847a6b31 100644 --- a/client/v2/go.mod +++ b/client/v2/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/client/v2 go 1.21 +toolchain go1.21.10 + require ( github.com/json-iterator/go v1.1.11 github.com/modern-go/reflect2 v1.0.1 diff --git a/client/v3/go.mod b/client/v3/go.mod index bb805cca3bb..5fee7c2a999 100644 --- a/client/v3/go.mod +++ b/client/v3/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/client/v3 go 1.21 +toolchain go1.21.10 + require ( github.com/dustin/go-humanize v1.0.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 diff --git a/etcdctl/go.mod b/etcdctl/go.mod index 8db91967308..436d5ab77a3 100644 --- a/etcdctl/go.mod +++ b/etcdctl/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/etcdctl/v3 go 1.21 +toolchain go1.21.10 + require ( github.com/bgentry/speakeasy v0.1.0 github.com/dustin/go-humanize v1.0.0 diff --git a/etcdutl/go.mod b/etcdutl/go.mod index e1c68a3acbd..668098c4392 100644 --- a/etcdutl/go.mod +++ b/etcdutl/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/etcdutl/v3 go 1.21 +toolchain go1.21.10 + replace ( go.etcd.io/etcd/api/v3 => ../api go.etcd.io/etcd/client/pkg/v3 => ../client/pkg diff --git a/go.mod b/go.mod index 8f3d4f9900d..2e1aa9351bb 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/v3 go 1.21 +toolchain go1.21.10 + replace ( go.etcd.io/etcd/api/v3 => ./api go.etcd.io/etcd/client/pkg/v3 => ./client/pkg diff --git a/pkg/go.mod b/pkg/go.mod index 6cc805ea618..e4c5973403c 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/pkg/v3 go 1.21 +toolchain go1.21.10 + require ( github.com/creack/pty v1.1.11 github.com/dustin/go-humanize v1.0.0 diff --git a/raft/go.mod b/raft/go.mod index 42d2c417adb..c0c425b3f62 100644 --- a/raft/go.mod +++ b/raft/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/raft/v3 go 1.21 +toolchain go1.21.10 + require ( github.com/cockroachdb/datadriven v1.0.2 github.com/gogo/protobuf v1.3.2 diff --git a/scripts/sync_go_toolchain_directive.sh b/scripts/sync_go_toolchain_directive.sh new file mode 100755 index 00000000000..643138e1d0a --- /dev/null +++ b/scripts/sync_go_toolchain_directive.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# This script looks at the version present in the .go-version file and treats +# that to be the value of the toolchain directive that go should use. It then +# updates the toolchain directives of all go.mod files to reflect this version. +# +# We do this to ensure that .go-version acts as the source of truth for go versions. + +set -euo pipefail + +source ./scripts/test_lib.sh + +TARGET_GO_VERSION="${TARGET_GO_VERSION:-"$(cat "${ETCD_ROOT_DIR}/.go-version")"}" +find . -name 'go.mod' -exec go mod edit -toolchain=go"${TARGET_GO_VERSION}" {} \; diff --git a/scripts/verify_go_versions.sh b/scripts/verify_go_versions.sh new file mode 100755 index 00000000000..11d34f92a51 --- /dev/null +++ b/scripts/verify_go_versions.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# This script verifies that the value of the toolchain directive in the +# go.mod files always match that of the .go-version file to ensure that +# we accidentally don't test and release with differing versions of Go. + +set -euo pipefail + +source ./scripts/test_lib.sh + +target_go_version="${target_go_version:-"$(cat "${ETCD_ROOT_DIR}/.go-version")"}" +log_info "expected go toolchain directive: go${target_go_version}" +log_info + +toolchain_out_of_sync="false" +go_line_violation="false" + +# verify_go_versions takes a go.mod filepath as an argument +# and checks if: +# (1) go directive <= version in .go-version +# (2) toolchain directive == version in .go-version +function verify_go_versions() { + # shellcheck disable=SC2086 + toolchain_version="$(go mod edit -json $1 | jq -r .Toolchain)" + # shellcheck disable=SC2086 + go_line_version="$(go mod edit -json $1 | jq -r .Go)" + if [[ "go${target_go_version}" != "${toolchain_version}" ]]; then + log_error "go toolchain directive out of sync for $1, got: ${toolchain_version}" + toolchain_out_of_sync="true" + fi + if ! printf '%s\n' "${go_line_version}" "${target_go_version}" | sort --check=silent --version-sort; then + log_error "go directive in $1 is greater than maximum allowed: go${target_go_version}" + go_line_violation="true" + fi +} + +while read -r mod; do + verify_go_versions "${mod}"; +done < <(find . -name 'go.mod') + +if [[ "${toolchain_out_of_sync}" == "true" ]]; then + log_error + log_error "Please run scripts/sync_go_toolchain_directive.sh or update .go-version to rectify this error" +fi + +if [[ "${go_line_violation}" == "true" ]]; then + log_error + log_error "Please update .go-version to rectify this error, any go directive should be <= .go-version" +fi + +if [[ "${go_line_violation}" == "true" ]] || [[ "${toolchain_out_of_sync}" == "true" ]]; then + exit 1 +fi diff --git a/server/go.mod b/server/go.mod index 4b2e16eb528..85fa045e73e 100644 --- a/server/go.mod +++ b/server/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/server/v3 go 1.21 +toolchain go1.21.10 + require ( github.com/coreos/go-semver v0.3.0 github.com/coreos/go-systemd/v22 v22.3.2 diff --git a/tests/go.mod b/tests/go.mod index bd3ad9d9438..61088e66f44 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/tests/v3 go 1.21 +toolchain go1.21.10 + replace ( go.etcd.io/etcd/api/v3 => ../api go.etcd.io/etcd/client/pkg/v3 => ../client/pkg diff --git a/tools/mod/go.mod b/tools/mod/go.mod index c48837c3705..dcd20c77cf6 100644 --- a/tools/mod/go.mod +++ b/tools/mod/go.mod @@ -2,6 +2,8 @@ module go.etcd.io/etcd/tools/v3 go 1.21 +toolchain go1.21.10 + require ( github.com/alexkohler/nakedret v1.0.0 github.com/chzchzchz/goword v0.0.0-20170907005317-a9744cb52b03 From 3c990bc43bf7a92d04ad96026ab350d255e8b447 Mon Sep 17 00:00:00 2001 From: Madhav Jivrajani Date: Wed, 15 May 2024 16:08:09 +0530 Subject: [PATCH 2/2] scripts: default to using .go-version's version for tests and builds Additionally, provide ability to opt-out of the .go-version and use a custom one via env vars: FORCE_HOST_GO and GO_VERSION. Signed-off-by: Madhav Jivrajani --- scripts/test_lib.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/test_lib.sh b/scripts/test_lib.sh index fc5985af8b2..58651670384 100644 --- a/scripts/test_lib.sh +++ b/scripts/test_lib.sh @@ -384,3 +384,27 @@ function git_assert_branch_in_sync { log_warning "Cannot verify consistency with the origin, as git is on detached branch." fi } + +# The version present in the .go-verion is the default version that test and build scripts will use. +# However, it is possible to control the version that should be used with the help of env vars: +# - FORCE_HOST_GO: if set to a non-empty value, use the version of go installed in system's $PATH. +# - GO_VERSION: desired version of go to be used, might differ from what is present in .go-version. +# If empty, the value defaults to the version in .go-version. +function determine_go_version { + # Borrowing from how Kubernetes does this: + # https://github.com/kubernetes/kubernetes/blob/17854f0e0a153b06f9d0db096e2cd8ab2fa89c11/hack/lib/golang.sh#L510-L520 + # + # default GO_VERSION to content of .go-version + GO_VERSION="${GO_VERSION:-"$(cat "${ETCD_ROOT_DIR}/.go-version")"}" + if [ "${GOTOOLCHAIN:-auto}" != 'auto' ]; then + # no-op, just respect GOTOOLCHAIN + : + elif [ -n "${FORCE_HOST_GO:-}" ]; then + export GOTOOLCHAIN='local' + else + GOTOOLCHAIN="go${GO_VERSION}" + export GOTOOLCHAIN + fi +} + +determine_go_version