From b76418776dc562c8f442be2ab3e5840935026cc6 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Wed, 31 Jan 2024 18:58:43 +0700 Subject: [PATCH] chore(ci): Bump Rust Versions & Alpine Versions, add ADR for rust-versions (#180) * chore(cat-ci): Bup rust tooling dependencies, and add wit-bindgen-cli as a tool to the rust base image * chore(cat-ci): Bump all possible alpine container versions to 3.19 * fix(docs): Fix CIP header in docs. * docs(ADR): Add adr about rust-version * docs(ADR): Add scope section to ADR to show they apply to all subordinate repos, unless otherwise defined. * fix(rust): Fix wit-bindgen-cli binary name in install * docs(cspell): fix spelling * fix(postgresql): sqlfluff not a standard alpine package, but still install system wide. * fix(rust): Fix command lines for `cargo modules` due to tool version changes. * fix(postgresql): fix string concat generating rust build graphs * docs(cspell): fix spelling * feat(rust): Allow BSD-2-Clause license in dependencies for Rust --- Earthfile | 3 +- cli/Earthfile | 2 +- docs/src/appendix/earthly.md | 6 +-- .../0001-arch-std.md | 4 ++ .../09_architecture_decisions/0002-adr.md | 4 ++ .../0003-language.md | 4 ++ .../0004-spelling.md | 4 ++ .../09_architecture_decisions/0005-rust.md | 4 ++ .../0006-rust-cargo-lock.md | 6 ++- .../0007-minimum-rust-version-supported.md | 50 +++++++++++++++++++ docs/src/guides/languages/bash.md | 4 +- docs/src/guides/languages/go.md | 4 +- earthly/bash/Earthfile | 2 +- earthly/docs/Earthfile | 1 - earthly/docs/common/overrides/cip.html | 8 +-- earthly/postgresql/Earthfile | 5 +- earthly/python/Earthfile | 2 +- earthly/rust/Earthfile | 20 ++++---- earthly/rust/scripts/std_build.py | 46 +++++++++++++---- earthly/rust/stdcfgs/deny.toml | 1 + examples/go/Earthfile | 4 +- examples/rust/deny.toml | 1 + tools/fetcher/Earthfile | 2 +- tools/updater/Earthfile | 2 +- utilities/dbviz/deny.toml | 1 + 25 files changed, 148 insertions(+), 42 deletions(-) create mode 100644 docs/src/architecture/09_architecture_decisions/0007-minimum-rust-version-supported.md diff --git a/Earthfile b/Earthfile index 5c7e8c21d..b1f9b62a2 100644 --- a/Earthfile +++ b/Earthfile @@ -20,7 +20,7 @@ check-spelling: # check-bash - test all bash files lint properly according to shellcheck. check-bash: - FROM alpine:3.18 + FROM alpine:3.19 DO ./earthly/bash+SHELLCHECK --src=. @@ -28,7 +28,6 @@ check-bash: repo-docs: # Create artifacts of extra files we embed inside the documentation when its built. FROM scratch - #FROM alpine:3.18 WORKDIR /repo COPY --dir *.md LICENSE-APACHE LICENSE-MIT . diff --git a/cli/Earthfile b/cli/Earthfile index a623c5621..d113cb663 100644 --- a/cli/Earthfile +++ b/cli/Earthfile @@ -1,5 +1,5 @@ VERSION --global-cache 0.7 -FROM golang:1.20-alpine3.18 +FROM golang:1.21-alpine3.19 # cspell: words onsi ldflags extldflags diff --git a/docs/src/appendix/earthly.md b/docs/src/appendix/earthly.md index b729cd9d6..46a8ca2f1 100644 --- a/docs/src/appendix/earthly.md +++ b/docs/src/appendix/earthly.md @@ -81,7 +81,7 @@ VERSION --global-cache 0.7 # This defines the "schema version" that this Earthf deps: # A target can be thought of as a group of container image layers (think of Docker multi-stage builds) # For this target, we start by deriving from an image which contains the Go tooling we need - FROM golang:1.20-alpine3.18 + FROM golang:1.21-alpine3.19 # Earthly has a 1:1 relationship with most Dockerfile commands, but there are a few exceptions WORKDIR /work @@ -107,7 +107,7 @@ Each target then specifies one or more commands that create the image layers ass VERSION --global-cache 0.7 deps: - FROM golang:1.20-alpine3.18 + FROM golang:1.21-alpine3.19 WORKDIR /work # These commands work identical to their Dockerfile equivalent @@ -182,7 +182,7 @@ build: docker: # Here we inherit from a "fresh" minimal alpine version - FROM alpine:3.18 + FROM alpine:3.19 WORKDIR /app # By default, we'll output this image with the 'latest' tag, but this can be diff --git a/docs/src/architecture/09_architecture_decisions/0001-arch-std.md b/docs/src/architecture/09_architecture_decisions/0001-arch-std.md index 1c0535cff..c6b3dcbf9 100644 --- a/docs/src/architecture/09_architecture_decisions/0001-arch-std.md +++ b/docs/src/architecture/09_architecture_decisions/0001-arch-std.md @@ -44,6 +44,10 @@ If we don't: * Difficult to ensure the necessary information is captured. * Difficult to iterate and be agile. +## Scope + +This ADR applies to all projects which consume `Catalyst-CI` unless they define an ADR specific to that project. + ## More Information * [arc42] diff --git a/docs/src/architecture/09_architecture_decisions/0002-adr.md b/docs/src/architecture/09_architecture_decisions/0002-adr.md index 1566338b9..380c9b8e8 100644 --- a/docs/src/architecture/09_architecture_decisions/0002-adr.md +++ b/docs/src/architecture/09_architecture_decisions/0002-adr.md @@ -36,6 +36,10 @@ This risk can be mitigated because the plugin is simple, and it would be easy fo * ADR become easier for people to author. * This should assist in making the team more pro-active in their creation and maintenance. +## Scope + +This ADR applies to all projects which consume `Catalyst-CI` unless they define an ADR specific to that project. + ## More Information * [arc42] diff --git a/docs/src/architecture/09_architecture_decisions/0003-language.md b/docs/src/architecture/09_architecture_decisions/0003-language.md index 84b0f2473..85e6dab88 100644 --- a/docs/src/architecture/09_architecture_decisions/0003-language.md +++ b/docs/src/architecture/09_architecture_decisions/0003-language.md @@ -33,6 +33,10 @@ and considerations of the audience for the project the primary language of the p Having a uniform language for the project makes it easier for people to interact on a common basis. +## Scope + +This ADR applies to all projects which consume `Catalyst-CI` unless they define an ADR specific to that project. + ## More Information * [Why English is the Lingua Franca of Programming](https://ystudios.com/insights-passion/codelanguage) diff --git a/docs/src/architecture/09_architecture_decisions/0004-spelling.md b/docs/src/architecture/09_architecture_decisions/0004-spelling.md index 258553075..a08f70547 100644 --- a/docs/src/architecture/09_architecture_decisions/0004-spelling.md +++ b/docs/src/architecture/09_architecture_decisions/0004-spelling.md @@ -46,6 +46,10 @@ By using automation we ensure this consistency, regardless of a contributors pro `cspell` +## Scope + +This ADR applies to all projects which consume `Catalyst-CI` unless they define an ADR specific to that project. + ## More Information * [CSpell] diff --git a/docs/src/architecture/09_architecture_decisions/0005-rust.md b/docs/src/architecture/09_architecture_decisions/0005-rust.md index a1a443f6a..b14641f98 100644 --- a/docs/src/architecture/09_architecture_decisions/0005-rust.md +++ b/docs/src/architecture/09_architecture_decisions/0005-rust.md @@ -79,6 +79,10 @@ make it a compelling choice for backend systems programming. The language emphasizes backward compatibility and provides a strong commitment to avoiding breaking changes. This ensures that code written in Rust today will continue to work in the future. +## Scope + +This ADR applies to all projects which consume `Catalyst-CI` unless they define an ADR specific to that project. + ## More Information * [Rust](https://www.rust-lang.org/) diff --git a/docs/src/architecture/09_architecture_decisions/0006-rust-cargo-lock.md b/docs/src/architecture/09_architecture_decisions/0006-rust-cargo-lock.md index 3a5628c37..3da358d40 100644 --- a/docs/src/architecture/09_architecture_decisions/0006-rust-cargo-lock.md +++ b/docs/src/architecture/09_architecture_decisions/0006-rust-cargo-lock.md @@ -3,7 +3,7 @@ adr: author: Steven Johnson created: 09-Jan-2024 - status: draft + status: accepted extends: - 0005-rust tags: @@ -38,6 +38,10 @@ We forget to introduce `cargo.lock` on our binaries when we approach release. This should make it a little easier to iterate with less issues caused by out of date `cargo.lock` files finding there way into CI. +## Scope + +This ADR applies to all projects which consume `Catalyst-CI` unless they define an ADR specific to that project. + ## More Information * [Cargo/toml vs Cargo.lock] diff --git a/docs/src/architecture/09_architecture_decisions/0007-minimum-rust-version-supported.md b/docs/src/architecture/09_architecture_decisions/0007-minimum-rust-version-supported.md new file mode 100644 index 000000000..38a815f49 --- /dev/null +++ b/docs/src/architecture/09_architecture_decisions/0007-minimum-rust-version-supported.md @@ -0,0 +1,50 @@ +--- + title: 0007 Rust Version configuration in `cargo.toml` + adr: + author: Steven Johnson + created: 21-Jan-2024 + status: accepted + extends: + - 0005-rust + tags: + - Rust +--- + +## Context + +In the Rust`cargo.toml` it is possible to specify a minimum version of Rust supported by a Crate/Application. +This rust projects which consume crates to ensure they or on a supported version. +There should be a policy about How many old versions of Rust is supported by our project. + +## Assumptions + +This ADR is deliberately limited to the initial bring up phase of our projects, and subject to review. + +## Decision + +We will not use the `rust-version` feature of `cargo.toml` during initial bring up. +We have not defined a maximum range of valid Rust versions, and always build ONLY with the version defined in `rust-toolchain.toml`. + +Currently the ONLY supported rust version is the one specified by `rust-toolchain.toml`. + +If at a later time, a range of rust versions is decided to be supported then: + +* This ADR will be obsoleted by a new one which defines that range of supported versions. +* The allowable range will need to be enforced in CI to ensure a `Cargo.toml` file does not specify the wrong thing. +* All Rust versions in that range will need to be tested in CI to ensure they are properly supported. + +## Risks + +None, this decision is subject to review at any time. + +## Consequences + +Development should be easier, and CI faster as we are specifically locked to a single Rust toolchain version. + +## Scope + +This ADR applies to all projects which consume `Catalyst-CI` unless they define an ADR specific to that project. + +## More Information + +* [`rust-version` field](https://doc.rust-lang.org/cargo/reference/manifest.html#the-rust-version-field) diff --git a/docs/src/guides/languages/bash.md b/docs/src/guides/languages/bash.md index 2f9948af9..0c690ca36 100644 --- a/docs/src/guides/languages/bash.md +++ b/docs/src/guides/languages/bash.md @@ -40,13 +40,13 @@ All that needs to happen is the following be added to the `Earthfile` in the roo ```Earthfile # Internal: shell-check - test all bash files against our shell check rules. shell-check: - FROM alpine:3.18 + FROM alpine:3.19 DO github.com/input-output-hk/catalyst-ci/earthly/bash:vx.y.z+SHELLCHECK --src=. # check all repo wide checks are run from here check: - FROM alpine:3.18 + FROM alpine:3.19 # Lint all bash files. BUILD +shell-check diff --git a/docs/src/guides/languages/go.md b/docs/src/guides/languages/go.md index 1d35e90df..0668d7302 100644 --- a/docs/src/guides/languages/go.md +++ b/docs/src/guides/languages/go.md @@ -50,7 +50,7 @@ VERSION --global-cache 0.7 deps: # This target is used to install external Go dependencies. - FROM golang:1.20-alpine3.18 + FROM golang:1.21-alpine3.19 WORKDIR /work # Any build dependencies should also be captured in this target. @@ -192,7 +192,7 @@ publish-example: # This target is called by CI when publishing images. It should use the # `SAVE IMAGE` command to save the image which is then picked up by the CI. # Note that we start from a "fresh" base image. - FROM alpine:3.18 + FROM alpine:3.19 WORKDIR /app ARG tag=latest # Prefer to use `latest` by default, the CI will override this. diff --git a/earthly/bash/Earthfile b/earthly/bash/Earthfile index 93617567f..189a2d70e 100644 --- a/earthly/bash/Earthfile +++ b/earthly/bash/Earthfile @@ -3,7 +3,7 @@ VERSION --global-cache --use-function-keyword 0.7 # Internal: builder creates a container we can use to execute shellcheck builder: - FROM alpine:3.18 + FROM alpine:3.19 RUN apk add --no-cache \ bash \ diff --git a/earthly/docs/Earthfile b/earthly/docs/Earthfile index 74b8a16e0..6802dc60f 100644 --- a/earthly/docs/Earthfile +++ b/earthly/docs/Earthfile @@ -56,7 +56,6 @@ deps: # Documentation files used across all documentation builds. common: FROM scratch - #FROM alpine:3.18 COPY --dir common/includes common/macros common/overrides common/std-theme.yml /std diff --git a/earthly/docs/common/overrides/cip.html b/earthly/docs/common/overrides/cip.html index 33e90e4a8..36f1432e3 100644 --- a/earthly/docs/common/overrides/cip.html +++ b/earthly/docs/common/overrides/cip.html @@ -20,16 +20,16 @@ Authors: {{ page.meta.Authors }}
{% endif %} {% if page.meta.Implementors %} - Status: {{ page.meta.Implementors }}
+ Implementors: {{ page.meta.Implementors }}
{% endif %} {% if page.meta.Discussions %} - Status: {{ page.meta.Discussions }}
+ Discussions: {{ page.meta.Discussions }}
{% endif %} {% if page.meta.Created %} - Status: {{ page.meta.Created }}
+ Created: {{ page.meta.Created }}
{% endif %} {% if page.meta.License %} - Status: {{ page.meta.License }}
+ License: {{ page.meta.License }}
{% endif %}
{% endif %} diff --git a/earthly/postgresql/Earthfile b/earthly/postgresql/Earthfile index 465a66fea..2adaefc00 100644 --- a/earthly/postgresql/Earthfile +++ b/earthly/postgresql/Earthfile @@ -4,7 +4,7 @@ VERSION --global-cache --use-function-keyword 0.7 # cspell: words colordiff psycopg dbviz postgres-base: - FROM postgres:16.0-alpine3.18 + FROM postgres:16.1-alpine3.19 WORKDIR /root @@ -28,7 +28,8 @@ postgres-base: RUN fc-cache -f # Install SQLFluff - RUN pip3 install sqlfluff==2.3.5 + # Not provided by Alpine, so we force pip3 to install it system wide (--break-system-packages) + RUN pip3 install sqlfluff==2.3.5 --break-system-packages RUN sqlfluff version # Get refinery diff --git a/earthly/python/Earthfile b/earthly/python/Earthfile index dbe99a9a0..ea8ac6049 100644 --- a/earthly/python/Earthfile +++ b/earthly/python/Earthfile @@ -4,7 +4,7 @@ VERSION --global-cache --use-function-keyword 0.7 # cspell: words libgcc python-base: - FROM python:3.12-alpine3.18 + FROM python:3.12-alpine3.19 # Install extra packages we will need to support plugins. RUN apk add --no-cache \ diff --git a/earthly/rust/Earthfile b/earthly/rust/Earthfile index e89930a69..c1d77dd19 100644 --- a/earthly/rust/Earthfile +++ b/earthly/rust/Earthfile @@ -1,7 +1,7 @@ # Common Rust UDCs and Builders. VERSION --global-cache --use-function-keyword 0.7 -# cspell: words rustup miri ripgrep colordiff stdcfgs toolset depgraph lcov psycopg +# cspell: words rustup miri ripgrep colordiff stdcfgs toolset depgraph lcov psycopg bindgen # cspell: words TARGETPLATFORM TARGETOS TARGETARCH TARGETVARIANT USERPLATFORM USEROS USERARCH USERVARIANT # Base Rustup build container. @@ -21,7 +21,7 @@ rust-base: # The ACTUAL version of rust that will be used, and available targets # is controlled by a `rust-toolchain.toml` file when the `SETUP` UDC is run. # HOWEVER, It is enforced that the rust version in `rust-toolchain.toml` MUST match this version. - FROM rust:1.75-alpine3.18 + FROM rust:1.75-alpine3.19 RUN echo "TARGETPLATFORM = $TARGETPLATFORM"; \ echo "TARGETOS = $TARGETOS"; \ @@ -69,14 +69,16 @@ rust-base: # Install tools we use commonly with `cargo`. # Note, we disable static compiles for tools, specifically, as its not required. # These tools are not artifacts and we do not use them in production. - RUN cargo install cargo-nextest --version=0.9.59 && \ + RUN cargo install cargo-nextest --version=0.9.67 && \ cargo install cargo-machete --version=0.6.0 && \ - cargo install refinery_cli --version=0.8.11 && \ - cargo install cargo-deny --version=0.14.3 && \ - cargo install cargo-modules --version=0.10.2 && \ - cargo install cargo-depgraph --version=1.5.0 && \ - cargo install cargo-llvm-cov --version=0.5.39 && \ - cargo install --git https://github.com/bytecodealliance/wasmtime --tag v16.0.0 verify-component-adapter + cargo install refinery_cli --version=0.8.12 && \ + cargo install cargo-deny --version=0.14.10 && \ + cargo install cargo-modules --version=0.13.5 && \ + cargo install cargo-depgraph --version=1.6.0 && \ + cargo install cargo-llvm-cov --version=0.6.4 && \ + cargo install wasm-tools --version=1.0.57 && \ + cargo install --git https://github.com/bytecodealliance/wasmtime --tag v17.0.0 verify-component-adapter && \ + cargo install --git https://github.com/bytecodealliance/wit-bindgen --rev 442f005 wit-bindgen-cli SAVE ARTIFACT $CARGO_HOME/bin/refinery refinery diff --git a/earthly/rust/scripts/std_build.py b/earthly/rust/scripts/std_build.py index 06d1ac122..764561e01 100755 --- a/earthly/rust/scripts/std_build.py +++ b/earthly/rust/scripts/std_build.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# cspell: words lcov depgraph readelf +# cspell: words lcov depgraph readelf sysroot import python.exec_manager as exec_manager import argparse @@ -114,13 +114,35 @@ def cargo_depgraph(results: exec_manager.Results): ) ) +COMMON_CARGO_MODULES_ORPHANS = ( + "NO_COLOR=1 " + + "cargo modules orphans --all-features " + + "--deny --cfg-test " +) +COMMON_CARGO_MODULES_STRUCTURE = ( + "NO_COLOR=1 " + + "cargo modules structure --no-fns --all-features " +) +COMMON_CARGO_MODULES_DEPENDENCIES = ( + "NO_COLOR=1 " + + "cargo modules dependencies --all-features " + + "--no-externs --no-fns --no-sysroot --no-traits --no-types --no-uses " +) def cargo_modules_lib(results: exec_manager.Results, lib: str): + # Check if we have any Orphans. + results.add( + exec_manager.cli_run( + COMMON_CARGO_MODULES_ORPHANS + + f"--package '{lib}' --lib", + name=f"Checking Orphans for {lib}", + ) + ) + # Generate tree results.add( exec_manager.cli_run( - "NO_COLOR=1 " - + "cargo modules generate tree --orphans --types --traits --tests --all-features " + COMMON_CARGO_MODULES_STRUCTURE + f"--package '{lib}' --lib > 'target/doc/{lib}.lib.modules.tree' ", name=f"Generate Module Trees for {lib}", ) @@ -128,8 +150,7 @@ def cargo_modules_lib(results: exec_manager.Results, lib: str): # Generate graph results.add( exec_manager.cli_run( - "NO_COLOR=1 " - + "cargo modules generate graph --all-features --modules " + COMMON_CARGO_MODULES_DEPENDENCIES + f"--package '{lib}' --lib > 'target/doc/{lib}.lib.modules.dot' ", name=f"Generate Module Graphs for {lib}", ) @@ -137,11 +158,19 @@ def cargo_modules_lib(results: exec_manager.Results, lib: str): def cargo_modules_bin(results: exec_manager.Results, package: str, bin: str): + # Check if we have any Orphans. + results.add( + exec_manager.cli_run( + COMMON_CARGO_MODULES_ORPHANS + + f"--package '{package}' --bin '{bin}'", + name=f"Checking Orphans for {package}/{bin}", + ) + ) + # Generate tree results.add( exec_manager.cli_run( - "NO_COLOR=1 " - + "cargo modules generate tree --orphans --types --traits --tests --all-features " + COMMON_CARGO_MODULES_STRUCTURE + f"--package '{package}' --bin '{bin}' > 'target/doc/{package}.{bin}.bin.modules.tree' ", name=f"Generate Module Trees for {package}/{bin}", ) @@ -149,8 +178,7 @@ def cargo_modules_bin(results: exec_manager.Results, package: str, bin: str): # Generate graph results.add( exec_manager.cli_run( - "NO_COLOR=1 " - + "cargo modules generate graph --all-features --modules " + COMMON_CARGO_MODULES_DEPENDENCIES + f"--package '{package}' --bin '{bin}' > 'target/doc/{package}.{bin}.bin.modules.dot' ", name=f"Generate Module Graphs for {package}/{bin}", ) diff --git a/earthly/rust/stdcfgs/deny.toml b/earthly/rust/stdcfgs/deny.toml index 4ac843fb5..886341af4 100644 --- a/earthly/rust/stdcfgs/deny.toml +++ b/earthly/rust/stdcfgs/deny.toml @@ -108,6 +108,7 @@ allow = [ "Apache-2.0", "Unicode-DFS-2016", "BSD-3-Clause", + "BSD-2-Clause", "BlueOak-1.0.0", "Apache-2.0 WITH LLVM-exception" ] diff --git a/examples/go/Earthfile b/examples/go/Earthfile index 8e102e6d9..4e49e9df7 100644 --- a/examples/go/Earthfile +++ b/examples/go/Earthfile @@ -7,7 +7,7 @@ VERSION --global-cache --use-function-keyword 0.7 deps: # This target is used to install external Go dependencies. - FROM golang:1.20-alpine3.18 + FROM golang:1.21-alpine3.19 WORKDIR /work # Any build dependencies should also be captured in this target. @@ -66,7 +66,7 @@ publish-example: # This target is called by CI when publishing images. It should use the # `SAVE IMAGE` command to save the image which is then picked up by the CI. # Note that we start from a "fresh" base image. - FROM alpine:3.18 + FROM alpine:3.19 WORKDIR /app ARG tag=latest # Prefer to use `latest` by default, the CI will override this. diff --git a/examples/rust/deny.toml b/examples/rust/deny.toml index 926f2ef43..41f4c10fd 100644 --- a/examples/rust/deny.toml +++ b/examples/rust/deny.toml @@ -108,6 +108,7 @@ allow = [ "Apache-2.0", "Unicode-DFS-2016", "BSD-3-Clause", + "BSD-2-Clause", "BlueOak-1.0.0", "Apache-2.0 WITH LLVM-exception" ] diff --git a/tools/fetcher/Earthfile b/tools/fetcher/Earthfile index 226b1605e..90746b251 100644 --- a/tools/fetcher/Earthfile +++ b/tools/fetcher/Earthfile @@ -1,5 +1,5 @@ VERSION 0.7 -FROM golang:1.20-alpine3.18 +FROM golang:1.21-alpine3.19 # cspell: words onsi ldflags extldflags diff --git a/tools/updater/Earthfile b/tools/updater/Earthfile index 7e6868e1b..a9c127cc4 100644 --- a/tools/updater/Earthfile +++ b/tools/updater/Earthfile @@ -1,5 +1,5 @@ VERSION 0.7 -FROM golang:1.21-alpine3.18 +FROM golang:1.21-alpine3.19 # cspell: words onsi ldflags extldflags diff --git a/utilities/dbviz/deny.toml b/utilities/dbviz/deny.toml index 926f2ef43..41f4c10fd 100644 --- a/utilities/dbviz/deny.toml +++ b/utilities/dbviz/deny.toml @@ -108,6 +108,7 @@ allow = [ "Apache-2.0", "Unicode-DFS-2016", "BSD-3-Clause", + "BSD-2-Clause", "BlueOak-1.0.0", "Apache-2.0 WITH LLVM-exception" ]