Race condition in history file fix?

I haven't tagged 2.9.1 because this seems like a worse position to be in than having passing tests run first every once in a while until this is determined to be a fluke or resolved. https://github.com/JuliaTesting/ParallelTestRunner.jl/actions/runs/35551060457/job/106185841081

Since the log files are ephemeral, the errors are all like this

  Error in testset retried test passes and runs alone (serial=String[], before):
  Error During Test at D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\test\retries.jl:18
    Got exception outside of a @test
    TaskFailedException
    Stacktrace:
      [1] #wait#596
        @ .\task.jl:363 [inlined]
      [2] wait
        @ .\task.jl:360 [inlined]
      [3] _runtests(mod::Module, args::ParallelTestRunner.ParsedArgs; testsuite::Dict{String, Expr}, tests::Vector{String}, historical_durations::Dict{String, Float64}, historical_failures::Set{String}, history_key::Nothing, init_code::Expr, init_worker_code::Expr, test_worker::Returns{Nothing}, RecordType::Type{ParallelTestRunner.TestRecord}, custom_args::@NamedTuple{}, exename::Nothing, exeflags::Nothing, env::Vector{Pair{String, String}}, serial::Vector{String}, serial_position::Symbol, stdout::IOBuffer, stderr::IOBuffer, max_worker_rss::Int64, memory_per_worker::Int64, recycle_on_failure::Bool, retries::Int64, history_flush_every::Int64)
        @ ParallelTestRunner D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\src\ParallelTestRunner.jl:1472
      [4] kwcall(::@NamedTuple{testsuite::Dict{String, Expr}, tests::Vector{String}, init_code::Expr, serial::Vector{String}, serial_position::Symbol, stdout::IOBuffer, stderr::IOBuffer, retries::Int64}, ::typeof(ParallelTestRunner._runtests), mod::Module, args::ParallelTestRunner.ParsedArgs)
        @ ParallelTestRunner D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\src\ParallelTestRunner.jl:937
      [5] macro expansion
        @ .\timing.jl:463 [inlined]
      [6] macro expansion
        @ D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\test\utils.jl:5 [inlined]
      [7] (::Main.var"##retries#142".var"#5#6"{Symbol, Vector{String}})(dir::String)
        @ Main.var"##retries#142" D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\test\retries.jl:41
      [8] mktempdir(fn::Main.var"##retries#142".var"#5#6"{Symbol, Vector{String}}, parent::String; prefix::String)
        @ Base.Filesystem .\file.jl:941
      [9] mktempdir(fn::Function, parent::String)
        @ Base.Filesystem .\file.jl:937
     [10] top-level scope
        @ D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\test\retries.jl:5
     [11] macro expansion
        @ C:\hostedtoolcache\windows\julia\1.13.0\x64\share\julia\stdlib\v1.13\Test\src\Test.jl:1962 [inlined]
     [12] macro expansion
        @ D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\test\retries.jl:18 [inlined]
     [13] macro expansion
        @ C:\hostedtoolcache\windows\julia\1.13.0\x64\share\julia\stdlib\v1.13\Test\src\Test.jl:2043 [inlined]
     [14] macro expansion
        @ D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\test\retries.jl:22 [inlined]
     [15] include(mapexpr::Function, mod::Module, _path::String)
        @ Base .\Base.jl:310
     [16] IncludeInto
        @ .\Base.jl:311 [inlined]
     [17] macro expansion
        @ D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\src\ParallelTestRunner.jl:367 [inlined]
     [18] macro expansion
        @ C:\hostedtoolcache\windows\julia\1.13.0\x64\share\julia\stdlib\v1.13\Test\src\Test.jl:1962 [inlined]
     [19] macro expansion
        @ D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\src\ParallelTestRunner.jl:367 [inlined]
     [20] macro expansion
        @ C:\hostedtoolcache\windows\julia\1.13.0\x64\share\julia\stdlib\v1.13\Test\src\Test.jl:1962 [inlined]
     [21] macro expansion
        @ D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\src\ParallelTestRunner.jl:366 [inlined]
     [22] macro expansion
        @ .\timing.jl:699 [inlined]
     [23] top-level scope
        @ D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\src\ParallelTestRunner.jl:365
     [24] eval(m::Module, e::Any)
        @ Core .\boot.jl:489
     [25] execute(::Type{ParallelTestRunner.TestRecord}, mod::Module, f::Expr, name::String, start_time::Float64, _custom_args::@NamedTuple{})
        @ ParallelTestRunner D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\src\ParallelTestRunner.jl:358
     [26] (::ParallelTestRunner.var"#inner#runtest##0"{Type{ParallelTestRunner.TestRecord}, Expr, String, Expr, Float64, @NamedTuple{}})()
        @ ParallelTestRunner D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\src\ParallelTestRunner.jl:395
     [27] runtest(RecordType::Type{ParallelTestRunner.TestRecord}, f::Expr, name::String, init_code::Expr, start_time::Float64, custom_args::@NamedTuple{})
        @ ParallelTestRunner D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\src\ParallelTestRunner.jl:400
     [28] (::var"#handle##0#handle##1"{Sockets.TCPSocket, UInt64, Bool, @Kwargs{}, Tuple{typeof(ParallelTestRunner.runtest), DataType, Expr, String, Expr, Float64, @NamedTuple{}}, typeof(invokelatest)})()
        @ Main C:\Users\runneradmin\.julia\packages\Malt\OteGQ\src\worker.jl:120
        nested task error: Worker process exited before we could connect. Stderr:
        nothing
        Stacktrace:
         [1] error(s::String)
           @ Base .\error.jl:44
         [2] Malt.Worker(; env::Vector{Pair{String, String}}, exename::String, exeflags::Vector{String}, monitor_stdout::Bool, monitor_stderr::Bool)
           @ Malt C:\Users\runneradmin\.julia\packages\Malt\OteGQ\src\Malt.jl:154
         [3] Worker
           @ C:\Users\runneradmin\.julia\packages\Malt\OteGQ\src\Malt.jl:115 [inlined]
         [4] ParallelTestRunner.PTRWorker(; exename::String, exeflags::Vector{String}, env::Vector{Pair{String, String}})
           @ ParallelTestRunner D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\src\ptrworker.jl:12
         [5] PTRWorker
           @ D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\src\ptrworker.jl:10 [inlined]
         [6] addworker(; env::Vector{Pair{String, String}}, init_worker_code::Expr, exename::Nothing, exeflags::Nothing, color::Bool)
           @ ParallelTestRunner D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\src\ptrworker.jl:101
         [7] addworker
           @ D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\src\ptrworker.jl:79 [inlined]
         [8] (::ParallelTestRunner.var"#70#71"{Bool, Vector{String}, Base.Semaphore, Nothing, Base.Threads.Atomic{Int64}, Dict{String, Expr}, Nothing, Expr, Expr, Returns{Nothing}, Type{ParallelTestRunner.TestRecord}, @NamedTuple{}, Nothing, Nothing, Vector{Pair{String, String}}, Int64, Bool, Int64, Module, ParallelTestRunner.ParsedArgs, Base.Threads.Atomic{Int64}, Channel{Tuple}, ParallelTestRunner.TestIOContext, ParallelTestRunner.var"#stop_work#52"{Base.RefValue{Bool}, Vector{Task}}, Base.RefValue{Bool}, Base.Threads.Atomic{Bool}, Base.Threads.Atomic{Float64}, Lockable{Dict{String, Float64}, ReentrantLock}, Lockable{ParallelTestRunner.TestHistory.PendingHistory, ReentrantLock}, Lockable{Vector{Any}, ReentrantLock}, Channel{Union{Nothing, ParallelTestRunner.PTRWorker}}})()
           @ ParallelTestRunner D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\src\ParallelTestRunner.jl:1298
        
        caused by: Timeout
        Stacktrace:
         [1] error(s::String)
           @ Base .\error.jl:44
         [2] Malt.Worker(; env::Vector{Pair{String, String}}, exename::String, exeflags::Vector{String}, monitor_stdout::Bool, monitor_stderr::Bool)
           @ Malt C:\Users\runneradmin\.julia\packages\Malt\OteGQ\src\Malt.jl:150
         [3] Worker
           @ C:\Users\runneradmin\.julia\packages\Malt\OteGQ\src\Malt.jl:115 [inlined]
         [4] ParallelTestRunner.PTRWorker(; exename::String, exeflags::Vector{String}, env::Vector{Pair{String, String}})
           @ ParallelTestRunner D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\src\ptrworker.jl:12
         [5] PTRWorker
           @ D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\src\ptrworker.jl:10 [inlined]
         [6] addworker(; env::Vector{Pair{String, String}}, init_worker_code::Expr, exename::Nothing, exeflags::Nothing, color::Bool)
           @ ParallelTestRunner D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\src\ptrworker.jl:101
         [7] addworker
           @ D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\src\ptrworker.jl:79 [inlined]
         [8] (::ParallelTestRunner.var"#70#71"{Bool, Vector{String}, Base.Semaphore, Nothing, Base.Threads.Atomic{Int64}, Dict{String, Expr}, Nothing, Expr, Expr, Returns{Nothing}, Type{ParallelTestRunner.TestRecord}, @NamedTuple{}, Nothing, Nothing, Vector{Pair{String, String}}, Int64, Bool, Int64, Module, ParallelTestRunner.ParsedArgs, Base.Threads.Atomic{Int64}, Channel{Tuple}, ParallelTestRunner.TestIOContext, ParallelTestRunner.var"#stop_work#52"{Base.RefValue{Bool}, Vector{Task}}, Base.RefValue{Bool}, Base.Threads.Atomic{Bool}, Base.Threads.Atomic{Float64}, Lockable{Dict{String, Float64}, ReentrantLock}, Lockable{ParallelTestRunner.TestHistory.PendingHistory, ReentrantLock}, Lockable{Vector{Any}, ReentrantLock}, Channel{Union{Nothing, ParallelTestRunner.PTRWorker}}})()
           @ ParallelTestRunner D:\a\ParallelTestRunner.jl\ParallelTestRunner.jl\src\ParallelTestRunner.jl:1298

I'm not entirely sure of what's going on, but my hunch is that Malt had troubles syncing up with the workers on this machine? I retried multiple times, after your retry, still haven't caught the same error again.

Do we just tag 2.9.1 and hope it was a fluke?

After 15 retries (for 15 jobs, total of 225 new jobs) still no sign of the same error again. Oceananigans CI has been running non-stop today with v2.9.0 and we haven't seen anything dodgy so far, and I'm having a hard time seeing how 0be4b81 could be problematic (it's modifying a Set, but under a lock).

One thing that the bot pointed out to me in the log of the failed job is https://github.com/JuliaTesting/ParallelTestRunner.jl/actions/runs/35551060457/job/106185841081#step:5:146

  Running 8 tests using 4 parallel jobs. To change the number of jobs, specify the `--jobs=N` argument to the tests, or set the `PTR_NUM_JOBS` environment variable.
  Available memory: 11.339 GiB; Max worker RSS: 3.711 GiB
                 │   Test   │   Init   │ Compile │ ──────────────── CPU ──────────────── │
  Test  (Worker) │ time (s) │ time (s) │   (%)   │ GC (s) │ GC % │ Alloc (MB) │ RSS (MB) │
  history    (3) │         started at 2026-09-21T01:35:49.138
  retries    (2) │         started at 2026-09-21T01:35:49.539
  serial     (4) │         started at 2026-09-21T01:35:49.678
  workers    (1) │         started at 2026-09-21T01:35:49.779
  retries    (2) │   181.78 │    14.52 │    failed at 2026-09-21T01:38:54.124
  output     (2) │         started at 2026-09-21T01:38:54.167
  history    (3) │   198.99 │    14.32 │    failed at 2026-09-21T01:39:10.600
  core       (3) │         started at 2026-09-21T01:39:10.600
  serial     (4) │   204.10 │    14.47 │    failed at 2026-09-21T01:39:15.865
  failures   (4) │         started at 2026-09-21T01:39:15.881
  output     (2) │    55.42 │     0.47 │    5.28 │   0.04 │  0.1 │     185.71 │   426.35 │
  unit       (2) │         started at 2026-09-21T01:39:49.900
  workers    (1) │   238.06 │    15.01 │    failed at 2026-09-21T01:39:50.043
  unit       (2) │     1.22 │     0.44 │   69.33 │   0.00 │  0.0 │      42.08 │   426.35 │
  failures   (4) │    42.89 │     0.45 │    2.85 │   0.00 │  0.0 │      45.31 │   510.09 │
  core       (3) │    50.87 │     0.40 │    9.22 │   0.04 │  0.1 │     211.15 │   450.02 │

The init time of the fresh workers was exceptionally long, I checked other Windows jobs and it's usually around 7-8 seconds (which is still long-ish compared to the other OSes, but that's what it is), so this points in the direction of "we just got an unlucky machine" (I swear I only gave the bot the raw log, didn't link to this issue!).

Alright I say we leave this open for a bit just in case and I’ve triggered the registrator

Navigation Menu

Repository navigation

Race condition in history file fix? #199

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Race condition in history file fix? #199

Description

Activity

giordano commented on Sep 21, 2026

christiangnrd commented on Sep 21, 2026

giordano commented on Sep 21, 2026

christiangnrd commented on Sep 21, 2026

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Issue actions