From ff82b8fb44d79871013e72aaf5d5a96fefcbd146 Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Tue, 5 Mar 2024 16:05:40 -0600 Subject: [PATCH 1/2] an assortment of CI bumps (#752) * an assortment of CI bumps * codecov token * test nightly on ubuntu-latest --- .github/workflows/benchmark.yml | 2 +- .github/workflows/current.yml | 8 +++++--- .github/workflows/docs-cleanup.yml | 2 +- .github/workflows/documenter.yml | 4 ++-- .github/workflows/minimum.yml | 8 +++++--- .github/workflows/nightly.yml | 8 +++++--- .github/workflows/style.yml | 2 +- 7 files changed, 20 insertions(+), 14 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index a2c31ae77..7d01e0cdb 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v1 with: version: "1" diff --git a/.github/workflows/current.yml b/.github/workflows/current.yml index f11a2eee5..97c6742ec 100644 --- a/.github/workflows/current.yml +++ b/.github/workflows/current.yml @@ -22,9 +22,9 @@ jobs: matrix: julia-version: [1] julia-arch: [x64] - os: [ubuntu-20.04] + os: [ubuntu-22.04] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.julia-version }} @@ -36,7 +36,9 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v4 with: flags: current files: lcov.info + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/docs-cleanup.yml b/.github/workflows/docs-cleanup.yml index e34565255..6b3692734 100644 --- a/.github/workflows/docs-cleanup.yml +++ b/.github/workflows/docs-cleanup.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout gh-pages branch - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: ref: gh-pages - name: Delete preview and history + push changes diff --git a/.github/workflows/documenter.yml b/.github/workflows/documenter.yml index 3ae86a5e6..a09104383 100644 --- a/.github/workflows/documenter.yml +++ b/.github/workflows/documenter.yml @@ -19,9 +19,9 @@ on: jobs: Documenter: name: Documentation - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v1 with: version: 1.8 diff --git a/.github/workflows/minimum.yml b/.github/workflows/minimum.yml index 9d4d90f83..528e2c4cd 100644 --- a/.github/workflows/minimum.yml +++ b/.github/workflows/minimum.yml @@ -22,9 +22,9 @@ jobs: matrix: julia-version: [1.8] julia-arch: [x64] - os: [ubuntu-20.04, macos-11, windows-2019] + os: [ubuntu-22.04, macos-11, windows-2019] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.julia-version }} @@ -36,7 +36,9 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v4 with: flags: minimum files: lcov.info + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 862873e1f..883418267 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -22,9 +22,9 @@ jobs: matrix: julia-version: [nightly] julia-arch: [x64] - os: [ubuntu-20.04] + os: [ubuntu-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v1 with: version: ${{ matrix.julia-version }} @@ -36,7 +36,9 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - uses: julia-actions/julia-processcoverage@v1 - - uses: codecov/codecov-action@v2 + - uses: codecov/codecov-action@v4 with: flags: nightly files: lcov.info + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index b671568fa..35597612f 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -26,7 +26,7 @@ jobs: - uses: julia-actions/setup-julia@latest with: version: ${{ matrix.julia-version }} - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Instantiate `format` environment and format run: | julia --project=format -e 'using Pkg; Pkg.instantiate()' From cfd302310ba66ad5008e97738bd6710cf5a7509a Mon Sep 17 00:00:00 2001 From: Phillip Alday Date: Tue, 5 Mar 2024 16:18:20 -0600 Subject: [PATCH 2/2] remove Pkg.add from Benchmark (#711) * remove Pkg.add from Benchmark * update * use median * disable retune * increase time tolerance * robustify * try forcing tuning * only run benchmark ci manually --- .github/workflows/benchmark.yml | 21 +++++++++++---------- benchmark/Project.toml | 2 +- benchmark/benchmarks.jl | 4 ++-- benchmark/run.jl | 9 +++++---- benchmark/tune.json | 1 + 5 files changed, 20 insertions(+), 17 deletions(-) create mode 100644 benchmark/tune.json diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 7d01e0cdb..6e73ae482 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -3,16 +3,17 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true on: - pull_request: - branches: - - main - paths-ignore: - - 'LICENSE.md' - - 'README.md' - - 'docs/**' - - 'issues/**' - - 'format/**' - - 'test/**' + workflow_dispatch: +# pull_request: +# branches: +# - main +# paths-ignore: +# - 'LICENSE.md' +# - 'README.md' +# - 'docs/**' +# - 'issues/**' +# - 'format/**' +# - 'test/**' jobs: benchmarks: runs-on: ubuntu-latest diff --git a/benchmark/Project.toml b/benchmark/Project.toml index e4293dceb..4d99f1bda 100644 --- a/benchmark/Project.toml +++ b/benchmark/Project.toml @@ -2,7 +2,7 @@ BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" MixedModels = "ff71e718-51f3-5ec2-a782-8ffcbfa3c316" PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d" -StatsModels = "3eaba693-59b7-5ba5-a881-562e759f1c8d" +Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" [compat] BenchmarkTools = "1" diff --git a/benchmark/benchmarks.jl b/benchmark/benchmarks.jl index 626146397..4c05d5c0d 100644 --- a/benchmark/benchmarks.jl +++ b/benchmark/benchmarks.jl @@ -1,4 +1,4 @@ -using BenchmarkTools, MixedModels, StatsModels +using BenchmarkTools, MixedModels using MixedModels: dataset const SUITE = BenchmarkGroup() @@ -85,7 +85,7 @@ function fitbobyqa(dsnm::Symbol, i::Integer) return fit(MixedModel, fms[dsnm][i], dataset(dsnm); contrasts, progress=false) end -# these tests are so fast that they can be very noisy because the denominator is so small, +# these tests are so fast that they can be very noisy because the denominator is so small, # so we disable them by default for auto-benchmarking # SUITE["simplescalar"] = BenchmarkGroup(["single", "simple", "scalar"]) # for (ds, i) in [ diff --git a/benchmark/run.jl b/benchmark/run.jl index ba2290db7..44a84d5ef 100644 --- a/benchmark/run.jl +++ b/benchmark/run.jl @@ -1,9 +1,10 @@ using Pkg Pkg.develop(PackageSpec(path=dirname(@__DIR__))) Pkg.instantiate() -using PkgBenchmark, MixedModels -# explicit `Pkg.add` is a crutch until we've got a good base on main +using PkgBenchmark, MixedModels, Statistics # Pkg.update() allows us to benchmark even when dependencies/compat requirements change -juliacmd = `$(Base.julia_cmd()) -O3 -e "using Pkg; Pkg.update(); Pkg.add([\"BenchmarkTools\", \"StatsModels\"])"` +juliacmd = `$(Base.julia_cmd()) -O3 -e "using Pkg; Pkg.update()"` config = BenchmarkConfig(; id="origin/HEAD", juliacmd) -export_markdown("benchmark.md", judge(MixedModels, config; verbose=true)) +# for many of the smaller models, we get a lot of noise at the default 5% tolerance +# TODO: specify a tune.json with per model time tolerances +export_markdown("benchmark.md", judge(MixedModels, config; verbose=true, retune=false, f=median, judgekwargs=(;time_tolerance=0.1, memory_tolerance=0.05))) diff --git a/benchmark/tune.json b/benchmark/tune.json new file mode 100644 index 000000000..969ce60ed --- /dev/null +++ b/benchmark/tune.json @@ -0,0 +1 @@ +[{"Julia":"1.9.2","BenchmarkTools":"1.0.0"},[["BenchmarkGroup",{"data":{"singlevector":["BenchmarkGroup",{"data":{"sleepstudy:3":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"sleepstudy:2":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"sleepstudy:4":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}]},"tags":["single","vector"]}],"crossed":["BenchmarkGroup",{"data":{"kb07:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"ml1m:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"insteval:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"insteval:2":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"machines:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"mrk17_exp1:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"penicillin:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}]},"tags":["multiple","crossed","scalar"]}],"crossedvector":["BenchmarkGroup",{"data":{"kb07:2":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"d3:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"kb07:3":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"mrk17_exp1:2":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}]},"tags":["multiple","crossed","vector"]}],"nested":["BenchmarkGroup",{"data":{"pastes:2":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}]},"tags":["multiple","nested","scalar"]}]},"tags":[]}]]] \ No newline at end of file