From 44212409bca42dd7d93eab0fff412e9d98ebac07 Mon Sep 17 00:00:00 2001 From: Shadaj Laddad Date: Sun, 7 Jan 2024 21:09:34 -0800 Subject: [PATCH] ci: consolidate tasks and use sccache and nextest --- .cargo/config.toml | 3 + .config/nextest.toml | 18 ++ .github/workflows/ci.yml | 168 ++++++++++--------- Cargo.lock | 47 +----- Cargo.toml | 8 - benches/Cargo.toml | 2 +- hydro_deploy/core/src/hydroflow_crate/mod.rs | 3 +- hydroflow/Cargo.toml | 2 +- hydroflow/tests/surface_scheduling.rs | 21 +-- 9 files changed, 127 insertions(+), 145 deletions(-) create mode 100644 .config/nextest.toml diff --git a/.cargo/config.toml b/.cargo/config.toml index 8fef47325299..2cd1fe84d567 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -9,3 +9,6 @@ rustflags = [ ## TODO(mingwei): Need rust-analyzer support: # "-Aclippy::uninlined-format-args", ] + +[target.x86_64-pc-windows-msvc] +linker = "rust-lld.exe" diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 000000000000..f72ea7b4b02e --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,18 @@ +[test-groups] +serial-integration = { max-threads = 1 } + +[[profile.default.overrides]] +filter = 'binary(deadlock_detector)' +test-group = 'serial-integration' + +[[profile.default.overrides]] +filter = 'binary(two_pc)' +test-group = 'serial-integration' + +[[profile.default.overrides]] +filter = 'binary(kvs)' +test-group = 'serial-integration' + +[[profile.default.overrides]] +filter = 'binary(kvs_replicated)' +test-group = 'serial-integration' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c3e83ca2158..3fa7dac9136c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: cancel_others: "true" check: - name: Check + name: Lint and Check if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.event_name != 'pull_request' }} timeout-minutes: 15 needs: pre_job @@ -40,7 +40,15 @@ jobs: # resolve to 'nothing' so we will exclude 'nothing'. https://stackoverflow.com/a/73822998 - rust_release: ${{ (needs.pre_job.outputs.should_skip != 'true' && 'nothing') || 'pinned-nightly' }} - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-nightly' }} - - os: ${{ (github.event_name != 'pull_request' && 'nothing') || 'windows-latest' }} + + env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + CARGO_PROFILE_DEV_STRIP: "debuginfo" + CARGO_PROFILE_TEST_STRIP: "debuginfo" + CARGO_PROFILE_RELEASE_STRIP: "debuginfo" steps: - name: Checkout sources @@ -53,6 +61,22 @@ jobs: toolchain: nightly override: ${{ matrix.rust_release == 'latest-nightly' }} + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 + + - name: Run cargo fmt + if: ${{ matrix.os == 'ubuntu-latest' }} + uses: actions-rs/cargo@v1 + with: + command: fmt + args: --all -- --check + + - name: Run cargo clippy + uses: actions-rs/cargo@v1 + with: + command: clippy + args: --all-targets --features python -- -D warnings + - name: Run cargo check uses: actions-rs/cargo@v1 with: @@ -108,12 +132,6 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest] - target_selection: - - "--lib --bins" - - "--examples" - - "--tests" - - "--benches" - - "--doc" rust_release: [pinned-nightly, latest-nightly] exclude: # For non-pull requests, event_name != 'pull_request' will be true, and 'nothing' is @@ -121,7 +139,15 @@ jobs: # resolve to 'nothing' so we will exclude 'nothing'. https://stackoverflow.com/a/73822998 - rust_release: ${{ (needs.pre_job.outputs.should_skip != 'true' && 'nothing') || 'pinned-nightly' }} - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-nightly' }} - - os: ${{ (github.event_name != 'pull_request' && 'nothing') || 'windows-latest' }} + + env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + CARGO_PROFILE_DEV_STRIP: "debuginfo" + CARGO_PROFILE_TEST_STRIP: "debuginfo" + CARGO_PROFILE_RELEASE_STRIP: "debuginfo" steps: - name: Checkout sources @@ -134,101 +160,75 @@ jobs: toolchain: nightly override: ${{ matrix.rust_release == 'latest-nightly' }} - - name: Run cargo test on target_selection - uses: actions-rs/cargo@v1 - with: - command: test - args: --profile cidev --no-fail-fast --features python ${{ matrix.target_selection }} + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@v0.0.3 - test-wasm: - name: Test Suite (WebAssembly) - if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.event_name != 'pull_request' }} - timeout-minutes: 15 - needs: pre_job - runs-on: ubuntu-latest - strategy: - matrix: - rust_release: [pinned-nightly, latest-nightly] - exclude: - # For non-pull requests, event_name != 'pull_request' will be true, and 'nothing' is - # truthy, so the entire && operator will resolve to 'nothing'. Then the || operator will - # resolve to 'nothing' so we will exclude 'nothing'. https://stackoverflow.com/a/73822998 - - rust_release: ${{ (needs.pre_job.outputs.should_skip != 'true' && 'nothing') || 'pinned-nightly' }} - - rust_release: ${{ (github.event_name != 'pull_request' && 'nothing') || 'latest-nightly' }} - - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - - name: Install nightly toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly - target: wasm32-unknown-unknown - override: ${{ matrix.rust_release == 'latest-nightly' }} + - name: Install cargo-nextest (linux) + if: ${{ matrix.os == 'ubuntu-latest' }} + run: curl -LsSf https://get.nexte.st/latest/linux | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin - - name: Get wasm-bindgen version - id: wasm-bindgen-version - run: echo "VERSION=$(cargo pkgid wasm-bindgen-shared | cut -d '@' -f2)" >> "$GITHUB_OUTPUT" + - name: Install cargo-nextest (windows) + if: ${{ matrix.os == 'windows-latest' }} + shell: bash + run: curl -LsSf https://get.nexte.st/latest/windows-tar | tar zxf - -C ${CARGO_HOME:-~/.cargo}/bin - - name: Install WebAssembly test runner + - name: Run cargo nextest on all targets uses: actions-rs/cargo@v1 with: - command: install - args: wasm-bindgen-cli@${{ steps.wasm-bindgen-version.outputs.VERSION }} + command: nextest + args: run --no-fail-fast --features python --all-targets - - name: Run cargo test + - name: Run doctests uses: actions-rs/cargo@v1 - env: - CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: wasm-bindgen-test-runner with: command: test - args: -p hydroflow --target wasm32-unknown-unknown --tests --no-fail-fast - - test-cli: - name: Test CLI - if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.event_name != 'pull_request' }} - timeout-minutes: 25 - needs: pre_job - runs-on: ubuntu-latest - - steps: - - name: Checkout sources - uses: actions/checkout@v3 - - - name: Install nightly toolchain - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: nightly + args: --no-fail-fast --features python --doc - name: Install Python uses: actions/setup-python@v4 with: python-version: '3.11' - - name: Build CLI + - name: Build CLI (Linux) + if: ${{ matrix.os == 'ubuntu-latest' }} run: | cd hydro_deploy/hydro_cli python -m venv .venv source .venv/bin/activate pip install maturin maturin develop + + - name: Build CLI (Windows) + if: ${{ matrix.os == 'windows-latest' }} + run: | + cd hydro_deploy/hydro_cli + python -m venv .venv + .venv/Scripts/activate + pip install maturin + maturin develop - - name: Run Python tests + - name: Run Python tests (Linux) + if: ${{ matrix.os == 'ubuntu-latest' }} run: | - ulimit -c unlimited cd hydro_deploy/hydro_cli source .venv/bin/activate cd python_tests pip install -r requirements.txt RUST_BACKTRACE=1 pytest *.py + + - name: Run Python tests (Windows) + if: ${{ matrix.os == 'windows-latest' }} + run: | + cd hydro_deploy/hydro_cli + .venv/Scripts/activate + cd python_tests + pip install -r requirements.txt + RUST_BACKTRACE=1 pytest *.py - lints: - name: Lints + test-wasm: + name: Test Suite (WebAssembly) if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.event_name != 'pull_request' }} - timeout-minutes: 10 + timeout-minutes: 15 needs: pre_job runs-on: ubuntu-latest strategy: @@ -250,20 +250,26 @@ jobs: with: profile: minimal toolchain: nightly + target: wasm32-unknown-unknown override: ${{ matrix.rust_release == 'latest-nightly' }} - components: rustfmt, clippy - - name: Run cargo fmt + - name: Get wasm-bindgen version + id: wasm-bindgen-version + run: echo "VERSION=$(cargo pkgid wasm-bindgen-shared | cut -d '@' -f2)" >> "$GITHUB_OUTPUT" + + - name: Install WebAssembly test runner uses: actions-rs/cargo@v1 with: - command: fmt - args: --all -- --check + command: install + args: wasm-bindgen-cli@${{ steps.wasm-bindgen-version.outputs.VERSION }} - - name: Run cargo clippy + - name: Run cargo test uses: actions-rs/cargo@v1 + env: + CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: wasm-bindgen-test-runner with: - command: clippy - args: --all-targets --features python -- -D warnings + command: test + args: -p hydroflow --target wasm32-unknown-unknown --tests --no-fail-fast build-website: name: Build Website diff --git a/Cargo.lock b/Cargo.lock index e8fe6a0fc3c9..e957f6f99a9b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -599,23 +599,11 @@ dependencies = [ "atty", "bitflags 1.3.2", "strsim 0.8.0", - "textwrap 0.11.0", + "textwrap", "unicode-width", "vec_map", ] -[[package]] -name = "clap" -version = "3.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" -dependencies = [ - "bitflags 1.3.2", - "clap_lex 0.2.4", - "indexmap 1.9.3", - "textwrap 0.16.0", -] - [[package]] name = "clap" version = "4.4.11" @@ -634,7 +622,7 @@ checksum = "a216b506622bb1d316cd51328dce24e07bdff4a6128a47c7e7fad11878d5adbb" dependencies = [ "anstream", "anstyle", - "clap_lex 0.6.0", + "clap_lex", "strsim 0.10.0", ] @@ -650,15 +638,6 @@ dependencies = [ "syn 2.0.42", ] -[[package]] -name = "clap_lex" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" -dependencies = [ - "os_str_bytes", -] - [[package]] name = "clap_lex" version = "0.6.0" @@ -771,20 +750,20 @@ dependencies = [ [[package]] name = "criterion" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" +checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" dependencies = [ "anes", - "atty", "cast", "ciborium", - "clap 3.2.25", + "clap 4.4.11", "criterion-plot", "futures", + "is-terminal", "itertools", - "lazy_static", "num-traits", + "once_cell", "oorandom", "plotters", "rayon", @@ -2122,12 +2101,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "os_str_bytes" -version = "6.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" - [[package]] name = "overload" version = "0.1.1" @@ -3218,12 +3191,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "textwrap" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" - [[package]] name = "thiserror" version = "1.0.51" diff --git a/Cargo.toml b/Cargo.toml index 01e134cc4f62..4a0e3e86fde0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,14 +38,6 @@ strip = true # Strip symbols from the binary opt-level = 3 lto = "fat" -# There is only around 19GB of free space on the github free default runners -# without stripping the debug info, the build artefacts take up about 19GB -# with stripping the debug info, they take up about 5GB, so in ci we can use this profile to -# avoid running out of disk space. -[profile.cidev] -inherits = "dev" -strip = "debuginfo" - [profile.profile] inherits = "release" debug = 2 diff --git a/benches/Cargo.toml b/benches/Cargo.toml index f58847eb0b75..83875ab262e3 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -10,7 +10,7 @@ license = "Apache-2.0" [dependencies] [dev-dependencies] -criterion = { version = "0.4", features = [ "async_tokio", "html_reports" ] } +criterion = { version = "0.5", features = [ "async_tokio", "html_reports" ] } hydroflow = { path = "../hydroflow" } lazy_static = "1.4.0" # pprof = { version = "0.6", features = [ "flamegraph", "criterion" ] } diff --git a/hydro_deploy/core/src/hydroflow_crate/mod.rs b/hydro_deploy/core/src/hydroflow_crate/mod.rs index a60ee8f66e95..ff500694e904 100644 --- a/hydro_deploy/core/src/hydroflow_crate/mod.rs +++ b/hydro_deploy/core/src/hydroflow_crate/mod.rs @@ -132,7 +132,8 @@ mod tests { let service = deployment.add_service( HydroflowCrate::new("../hydro_cli_examples", localhost.clone()) - .example("panic_program"), + .example("panic_program") + .profile("dev"), ); deployment.deploy().await.unwrap(); diff --git a/hydroflow/Cargo.toml b/hydroflow/Cargo.toml index 811dfbeb15fd..7f2afd6a0350 100644 --- a/hydroflow/Cargo.toml +++ b/hydroflow/Cargo.toml @@ -89,4 +89,4 @@ zipf = "7.0.0" [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] # Rayon (rust data-parallelism library) does not compile on WASM. -criterion = { version = "0.4", features = [ "async_tokio", "html_reports" ] } +criterion = { version = "0.5", features = [ "async_tokio", "html_reports" ] } diff --git a/hydroflow/tests/surface_scheduling.rs b/hydroflow/tests/surface_scheduling.rs index 4638ac941d82..39c28861c0b9 100644 --- a/hydroflow/tests/surface_scheduling.rs +++ b/hydroflow/tests/surface_scheduling.rs @@ -47,18 +47,14 @@ pub fn test_tick_loop() { async fn test_persist_stratum_run_available() -> Result<(), Box> { let (out_send, out_recv) = hydroflow::util::unbounded_channel(); - let handle = tokio::task::spawn_blocking(|| { - let mut df = hydroflow_syntax! { - a = source_iter([0]) - -> persist() - -> next_stratum() - -> for_each(|x| out_send.send(x).unwrap()); - }; - assert_graphvis_snapshots!(df); - df.run_available(); - }); - tokio::time::sleep(std::time::Duration::from_millis(200)).await; - handle.abort(); + let mut df = hydroflow_syntax! { + a = source_iter([0]) + -> persist() + -> next_stratum() + -> for_each(|x| out_send.send(x).unwrap()); + }; + assert_graphvis_snapshots!(df); + df.run_available(); let seen: Vec<_> = hydroflow::util::collect_ready_async(out_recv).await; rassert_eq!( @@ -68,7 +64,6 @@ async fn test_persist_stratum_run_available() -> Result<(), Box> { seen.len() )?; - handle.await.unwrap(); Ok(()) }