Add a history_key keyword to keep separate test histories

docs/src/index.md

@@ -108,6 +108,10 @@ Pkg.test("MyPackage"; test_args=`--verbose --jobs=4 integration`)
 Tests run concurrently in isolated worker processes, each inside own module.
 `ParallelTestRunner` records historical tests duration for each package, so that in subsequent runs long-running tests are executed first, to improve load balancing.
 See [Choosing the Number of Jobs](@ref) for how the number of workers is chosen.
+The history is shared by all runs of the package on a machine and updated during the run, in
+batches of a few tests, so concurrent runs can share it safely. When the same suite is run in
+configurations with different timings (for example on a CPU and on a GPU) pass
+`history_key = "gpu"` to `runtests` to keep a separate history per configuration.
 
 ### Serial Test Support
 

src/ParallelTestRunner.jl

@@ -571,14 +571,19 @@ end
 Base.isless(a::TestHistoryEntry, b::TestHistoryEntry) = a.failed == b.failed ? a.duration < b.duration : a.failed < b.failed
 
 # Historical test duration database
-function get_history_file(mod::Module)
+function get_history_file(mod::Module, history_key::Union{Nothing, AbstractString} = nothing)
     # History file version. Change when modifying the history format
     hist_ver = "v2"
     scratch_dir = @get_scratch!("durations")
-    return joinpath(scratch_dir, "v$(VERSION.major).$(VERSION.minor)", hist_ver, "$(nameof(mod)).jls")
+    name = string(nameof(mod))
+    if history_key !== nothing
+        isempty(history_key) && throw(ArgumentError("history_key must not be empty"))
+        name *= "-" * replace(history_key, r"[^\w.-]" => "_")
+    end
+    return joinpath(scratch_dir, "v$(VERSION.major).$(VERSION.minor)", hist_ver, "$name.jls")
 end
-function load_test_history(mod::Module)
-    history_file = get_history_file(mod)
+function load_test_history(mod::Module, history_key = nothing)
+    history_file = get_history_file(mod, history_key)
     if isfile(history_file)
         try
             return deserialize(history_file)::Tuple{Dict{String, Float64}, Set{String}}
@@ -627,11 +632,11 @@ set of failing tests. Entries of tests not mentioned are left as they are, so co
 on the same machine only ever update their own tests.
 """
 function update_test_history!(mod::Module, durations::Dict{String, Float64},
-                              passed::Set{String}, failed::Set{String})
-    history_file = get_history_file(mod)
+                              passed::Set{String}, failed::Set{String}; history_key = nothing)
+    history_file = get_history_file(mod, history_key)
     try
         with_history_lock(history_file) do
-            stored_durations, stored_failures = load_test_history(mod)
+            stored_durations, stored_failures = load_test_history(mod, history_key)
             merge!(stored_durations, durations)
             setdiff!(stored_failures, passed)
             union!(stored_failures, failed)
@@ -654,14 +659,14 @@ end
 PendingHistory() = PendingHistory(Dict{String, Float64}(), Set{String}(), Set{String}())
 
 function record_test_history!(pending::Lockable{PendingHistory}, mod::Module, test::String, result, duration::Real;
-                              history_flush_every::Integer)
+                              history_flush_every::Integer, history_key = nothing)
     failed = !(result isa AbstractTestRecord) || anynonpass(result[])
     batch = @lock pending begin
         pending[].durations[test] = Float64(duration)
         push!(failed ? pending[].failed : pending[].passed, test)
         length(pending[].durations) >= history_flush_every ? take_pending_history!(pending[]) : nothing
     end
-    batch === nothing || update_test_history!(mod, batch...)
+    batch === nothing || update_test_history!(mod, batch...; history_key)
     return nothing
 end
 
@@ -671,15 +676,16 @@ function take_pending_history!(pending::PendingHistory)
     return batch
 end
 
-function flush_test_history!(pending::Lockable{PendingHistory}, mod::Module)
+function flush_test_history!(pending::Lockable{PendingHistory}, mod::Module; history_key = nothing)
     batch = @lock pending take_pending_history!(pending[])
-    isempty(batch[1]) || update_test_history!(mod, batch...)
+    isempty(batch[1]) || update_test_history!(mod, batch...; history_key)
     return nothing
 end
 
 # Replace the whole history, e.g. to seed it in tests.
-function save_test_history(mod::Module, history::Tuple{Dict{String, Float64}, Set{String}})
-    history_file = get_history_file(mod)
+function save_test_history(mod::Module, history::Tuple{Dict{String, Float64}, Set{String}};
+                           history_key = nothing)
+    history_file = get_history_file(mod, history_key)
     try
         with_history_lock(history_file) do
             write_test_history(history_file, history)
@@ -1010,6 +1016,7 @@ end
              recycle_on_failure::Bool = false,
              retries::Integer = 0,
              history_flush_every::Integer = 20,
+             history_key = nothing,
              )
     runtests(mod::Module, ARGS; ...)
 
@@ -1071,6 +1078,9 @@ Several keyword arguments are also supported:
   written when the run ends. Larger values mean fewer lock, write and rename operations,
   which matters on slow filesystems, while an interrupted run loses at most that many
   measurements.
+- `history_key`: Keep a separate history of test durations and failures for this
+  configuration, e.g. `"gpu"` or the name of a CI job, when the same test suite is run in
+  configurations whose timings differ. By default all runs of `mod` share one history.
 
 ## Command Line Options
 
@@ -1180,14 +1190,15 @@ function runtests(mod::Module, args::ParsedArgs;
                   recycle_on_failure::Bool = false,
                   retries::Integer = 0,
                   history_flush_every::Integer = 20,
+                  history_key::Union{Nothing, AbstractString} = nothing,
                   )
     #
     # set-up
     #
 
     # list tests, if requested
     if args.list !== nothing
-        historical_durations, historical_failures = load_test_history(mod)
+        historical_durations, historical_failures = load_test_history(mod, history_key)
         sorted_tests = sort(collect(keys(testsuite)))
         name_align = isempty(sorted_tests) ? 0 : maximum(textwidth, sorted_tests)
         duration_strs = Dict(
@@ -1219,7 +1230,7 @@ function runtests(mod::Module, args::ParsedArgs;
     # determine test order
     tests = collect(keys(testsuite))
     Random.shuffle!(tests)
-    historical_durations, historical_failures = load_test_history(mod)
+    historical_durations, historical_failures = load_test_history(mod, history_key)
     get_historical_duration(test) = TestHistoryEntry(get(historical_durations, test, Inf), test in historical_failures)
     sort!(tests, by = x -> get_historical_duration(x), rev = true)
 
@@ -1229,6 +1240,7 @@ function runtests(mod::Module, args::ParsedArgs;
         tests,
         historical_durations,
         historical_failures,
+        history_key,
         init_code,
         init_worker_code,
         test_worker,
@@ -1255,6 +1267,7 @@ function _runtests(mod::Module, args::ParsedArgs;
                    tests::Vector{String},
                    historical_durations::Dict{String, Float64} = Dict{String, Float64}(),
                    historical_failures::Set{String} = Set{String}(),
+                   history_key = nothing,
                    init_code = :(),
                    init_worker_code = :(),
                    test_worker = Returns(nothing),
@@ -1640,7 +1653,7 @@ function _runtests(mod::Module, args::ParsedArgs;
                               retry_mode && filter!(r -> r.test != test, results[])
                               push!(results[], (; test, result, output, test_t0, test_t1))
                           end
-                          record_test_history!(pending_history, mod, test, result, test_t1 - test_t0; history_flush_every)
+                          record_test_history!(pending_history, mod, test, result, test_t1 - test_t0; history_flush_every, history_key)
 
                           # act on the results
                           if result isa AbstractTestRecord
@@ -1906,7 +1919,7 @@ function _runtests(mod::Module, args::ParsedArgs;
             Test.TESTSET_PRINT_ENABLE[] = old_print_setting
         end
     end
-    flush_test_history!(pending_history, mod)
+    flush_test_history!(pending_history, mod; history_key)
 
     # display the results
     println(io_ctx.stdout)

test/history.jl

@@ -202,6 +202,78 @@ end
     end
 end
 
+@testset "history_key selects the history file" begin
+    mod = history_module("key")
+    @test ParallelTestRunner.get_history_file(mod, nothing) == history_file(mod)
+    keyed = ParallelTestRunner.get_history_file(mod, "gpu")
+    @test dirname(keyed) == dirname(history_file(mod))
+    @test basename(keyed) == "HistoryTest_key-gpu.jls"
+    sanitized = ParallelTestRunner.get_history_file(mod, "cuda/12.9 x")
+    @test dirname(sanitized) == dirname(history_file(mod))
+    @test basename(sanitized) == "HistoryTest_key-cuda_12.9_x.jls"
+    @test_throws ArgumentError ParallelTestRunner.get_history_file(mod, "")
+end
+
+@testset "keyed and unkeyed histories are independent" begin
+    mod = history_module("isolation")
+    keyed_file = ParallelTestRunner.get_history_file(mod, "gpu")
+    remove_history(mod)
+    rm(keyed_file; force=true)
+    try
+        io = IOBuffer()
+        runtests(mod, ["--jobs=1"]; testsuite=Dict("a" => :(@test true)), history_key="gpu", stdout=io, stderr=io)
+        runtests(mod, ["--jobs=1"]; testsuite=Dict("b" => :(@test true)), stdout=io, stderr=io)
+        @test collect(keys(first(ParallelTestRunner.load_test_history(mod, "gpu")))) == ["a"]
+        @test collect(keys(first(ParallelTestRunner.load_test_history(mod)))) == ["b"]
+    finally
+        remove_history(mod)
+        rm(keyed_file; force=true)
+    end
+end
+
+@testset "scheduling uses the keyed history" begin
+    mod = history_module("keyed_order")
+    keyed_file = ParallelTestRunner.get_history_file(mod, "gpu")
+    remove_history(mod)
+    rm(keyed_file; force=true)
+    try
+        # the two histories disagree on which test is slow; the keyed one must decide
+        ParallelTestRunner.save_test_history(mod, (Dict("slow" => 10.0, "fast" => 1.0), Set{String}()); history_key="gpu")
+        ParallelTestRunner.save_test_history(mod, (Dict("slow" => 1.0, "fast" => 10.0), Set{String}()))
+        testsuite = Dict("slow" => :(@test true), "fast" => :(@test true))
+        io = IOBuffer()
+        runtests(mod, ["--jobs=1", "--verbose"]; testsuite, history_key="gpu", stdout=io, stderr=io)
+        str = String(take!(io))
+        @test findfirst("slow", str) < findfirst("fast", str)
+    finally
+        remove_history(mod)
+        rm(keyed_file; force=true)
+    end
+end
+
+@testset "--list respects history_key" begin
+    mod = history_module("keyed_list")
+    keyed_file = ParallelTestRunner.get_history_file(mod, "gpu")
+    remove_history(mod)
+    rm(keyed_file; force=true)
+    function list_output(kwargs)
+        code = """
+            using ParallelTestRunner
+            testsuite = Dict("alpha" => :(@test true))
+            runtests(Module($(repr(nameof(mod)))), ["--list"]; testsuite, $kwargs)
+            """
+        return readlines(`$run_history_test_process --color=no -e $code`)
+    end
+    try
+        ParallelTestRunner.save_test_history(mod, (Dict("alpha" => 1.234), Set{String}()); history_key="gpu")
+        @test list_output("history_key=\"gpu\"") == ["Available tests:", " - alpha  (1.23s)"]
+        @test list_output("history_key=nothing") == ["Available tests:", " - alpha"]
+    finally
+        remove_history(mod)
+        rm(keyed_file; force=true)
+    end
+end
+
 # All workers must have been stopped once `runtests` returns.
 @testset "no workers running" begin
     children = _count_child_pids()