Merge branch 'main' into pa/pca_precomp

.github/workflows/benchmark.yml

@@ -3,23 +3,24 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 on:
-  pull_request:
-    branches:
-      - main
-    paths-ignore:
-      - 'LICENSE.md'
-      - 'README.md'
-      - 'docs/**'
-      - 'issues/**'
-      - 'format/**'
-      - 'test/**'
+    workflow_dispatch:
+#   pull_request:
+#     branches:
+#       - main
+#     paths-ignore:
+#       - 'LICENSE.md'
+#       - 'README.md'
+#       - 'docs/**'
+#       - 'issues/**'
+#       - 'format/**'
+#       - 'test/**'
 jobs:
     benchmarks:
         runs-on: ubuntu-latest
         strategy:
           fail-fast: false
         steps:
-          - uses: actions/checkout@v3
+          - uses: actions/checkout@v4
           - uses: julia-actions/setup-julia@v1
             with:
               version: "1"

.github/workflows/current.yml

@@ -22,9 +22,9 @@ jobs:
       matrix:
         julia-version: [1]
         julia-arch: [x64]
-        os: [ubuntu-20.04]
+        os: [ubuntu-22.04]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v1
         with:
           version: ${{ matrix.julia-version }}
@@ -36,7 +36,9 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - uses: julia-actions/julia-processcoverage@v1
-      - uses: codecov/codecov-action@v2
+      - uses: codecov/codecov-action@v4
         with:
           flags: current
           files: lcov.info
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/docs-cleanup.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout gh-pages branch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           ref: gh-pages
       - name: Delete preview and history + push changes

.github/workflows/documenter.yml

@@ -19,9 +19,9 @@ on:
 jobs:
   Documenter:
     name: Documentation
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v1
         with:
           version: 1.8

.github/workflows/minimum.yml

@@ -22,9 +22,9 @@ jobs:
       matrix:
         julia-version: [1.8]
         julia-arch: [x64]
-        os: [ubuntu-20.04, macos-11, windows-2019]
+        os: [ubuntu-22.04, macos-11, windows-2019]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v1
         with:
           version: ${{ matrix.julia-version }}
@@ -36,7 +36,9 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - uses: julia-actions/julia-processcoverage@v1
-      - uses: codecov/codecov-action@v2
+      - uses: codecov/codecov-action@v4
         with:
           flags: minimum
           files: lcov.info
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/nightly.yml

@@ -22,9 +22,9 @@ jobs:
       matrix:
         julia-version: [nightly]
         julia-arch: [x64]
-        os: [ubuntu-20.04]
+        os: [ubuntu-latest]
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: julia-actions/setup-julia@v1
         with:
           version: ${{ matrix.julia-version }}
@@ -36,7 +36,9 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - uses: julia-actions/julia-processcoverage@v1
-      - uses: codecov/codecov-action@v2
+      - uses: codecov/codecov-action@v4
         with:
           flags: nightly
           files: lcov.info
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/style.yml

@@ -26,7 +26,7 @@ jobs:
       - uses: julia-actions/setup-julia@latest
         with:
           version: ${{ matrix.julia-version }}
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
       - name: Instantiate `format` environment and format
         run: |
           julia --project=format -e 'using Pkg; Pkg.instantiate()'

benchmark/Project.toml

@@ -2,7 +2,7 @@
 BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
 MixedModels = "ff71e718-51f3-5ec2-a782-8ffcbfa3c316"
 PkgBenchmark = "32113eaa-f34f-5b0d-bd6c-c81e245fc73d"
-StatsModels = "3eaba693-59b7-5ba5-a881-562e759f1c8d"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 
 [compat]
 BenchmarkTools = "1"

benchmark/benchmarks.jl

@@ -1,4 +1,4 @@
-using BenchmarkTools, MixedModels, StatsModels
+using BenchmarkTools, MixedModels
 using MixedModels: dataset
 
 const SUITE = BenchmarkGroup()
@@ -85,7 +85,7 @@ function fitbobyqa(dsnm::Symbol, i::Integer)
     return fit(MixedModel, fms[dsnm][i], dataset(dsnm); contrasts, progress=false)
 end
 
-# these tests are so fast that they can be very noisy because the denominator is so small, 
+# these tests are so fast that they can be very noisy because the denominator is so small,
 # so we disable them by default for auto-benchmarking
 # SUITE["simplescalar"] = BenchmarkGroup(["single", "simple", "scalar"])
 # for (ds, i) in [

benchmark/run.jl

@@ -1,9 +1,10 @@
 using Pkg
 Pkg.develop(PackageSpec(path=dirname(@__DIR__)))
 Pkg.instantiate()
-using PkgBenchmark, MixedModels
-# explicit `Pkg.add` is a crutch until we've got a good base on main
+using PkgBenchmark, MixedModels, Statistics
 # Pkg.update() allows us to benchmark even when dependencies/compat requirements change
-juliacmd = `$(Base.julia_cmd()) -O3 -e "using Pkg; Pkg.update(); Pkg.add([\"BenchmarkTools\", \"StatsModels\"])"`
+juliacmd = `$(Base.julia_cmd()) -O3 -e "using Pkg; Pkg.update()"`
 config = BenchmarkConfig(; id="origin/HEAD", juliacmd)
-export_markdown("benchmark.md", judge(MixedModels, config; verbose=true))
+# for many of the smaller models, we get a lot of noise at the default 5% tolerance
+# TODO: specify a tune.json with per model time tolerances
+export_markdown("benchmark.md", judge(MixedModels, config; verbose=true, retune=false, f=median, judgekwargs=(;time_tolerance=0.1, memory_tolerance=0.05)))

benchmark/tune.json

@@ -0,0 +1 @@
+[{"Julia":"1.9.2","BenchmarkTools":"1.0.0"},[["BenchmarkGroup",{"data":{"singlevector":["BenchmarkGroup",{"data":{"sleepstudy:3":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"sleepstudy:2":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"sleepstudy:4":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}]},"tags":["single","vector"]}],"crossed":["BenchmarkGroup",{"data":{"kb07:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"ml1m:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"insteval:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"insteval:2":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"machines:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"mrk17_exp1:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"penicillin:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}]},"tags":["multiple","crossed","scalar"]}],"crossedvector":["BenchmarkGroup",{"data":{"kb07:2":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"d3:1":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"kb07:3":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}],"mrk17_exp1:2":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}]},"tags":["multiple","crossed","vector"]}],"nested":["BenchmarkGroup",{"data":{"pastes:2":["Parameters",{"gctrial":true,"time_tolerance":0.05,"samples":10000,"evals":1,"gcsample":false,"seconds":5.0,"overhead":0.0,"memory_tolerance":0.01}]},"tags":["multiple","nested","scalar"]}]},"tags":[]}]]]