From 82b373368f0e5b8a5fb80c93aa09c1276beb230f Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Wed, 9 Sep 2026 19:14:38 +0000 Subject: [PATCH 1/2] fix: a cap of one read "at most 1 marks" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by a sweep over the twenty-three public names no probe round had touched, and the twelve usage patterns behind them. Everything else answered correctly; this is the only thing that did not, and it is a `@testset` description — user-facing text, which is what a reader sees when CI goes red. One is the interesting cap: it is what a package sets when it means "the next mark is a decision", so it is the description most likely to be read. The assertion needed a helper. `Collect` discarded the description it was handed, so the tree it builds could not be asked what the gate called itself; it keeps it now, and `gate_descriptions` reads it out. In a function rather than inline, for the same reason `gate_failed` is one — an inline `@testset` here would be counted as a behaviour of this file carrying no assertion of its own, which is exactly what the count is for. Checked against the plural restored: 1 failure with it, 0 without. 193 behaviours, 1251 assertions, green. Co-Authored-By: Claude Opus 5 --- ext/ExperimentalAPITestExt.jl | 2 +- test/spec/test_spec_integration.jl | 29 ++++++++++++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/ext/ExperimentalAPITestExt.jl b/ext/ExperimentalAPITestExt.jl index 5c45cf9..3cc2317 100644 --- a/ext/ExperimentalAPITestExt.jl +++ b/ext/ExperimentalAPITestExt.jl @@ -82,7 +82,7 @@ function ExperimentalAPI.test_surface( # The ratchet. `nothing` rather than `typemax`: a cap that is off must not read as a cap # that is enormous, because the second is a number somebody has to justify. if max_marks !== nothing - @testset "at most $max_marks marks" begin + @testset "at most $max_marks mark$(max_marks == 1 ? "" : "s")" begin @test length(experimental(m)) <= max_marks end end diff --git a/test/spec/test_spec_integration.jl b/test/spec/test_spec_integration.jl index 68ff74e..aa87d05 100644 --- a/test/spec/test_spec_integration.jl +++ b/test/spec/test_spec_integration.jl @@ -220,8 +220,9 @@ end # `@testset` — so the failing direction has to be run under a test set that records instead of # propagating. Two lines of `AbstractTestSet` is the whole cost of checking that. struct Collect <: Test.AbstractTestSet + description::String results::Vector{Any} - Collect(::AbstractString) = new(Any[]) + Collect(d::AbstractString) = new(String(d), Any[]) end Test.record(ts::Collect, res) = (push!(ts.results, res); res) # A nested `@testset` that does not name a type inherits the enclosing one, so every set inside @@ -239,6 +240,17 @@ Run `f` under a test set that records instead of propagating, and say whether an failed. This is how a gate is shown to fire without the failure it is supposed to produce reaching the suite that is checking for it. """ +# Every `@testset` description `f` produced. A gate's description is user-facing text — it is what +# a reader sees when CI goes red — so it is worth asserting on, not only the pass/fail. In a +# function for the same reason `gate_failed` is: a `@testset` written inline here would be counted +# as a behaviour of this file that carries no assertion of its own. +function gate_descriptions(f) + ts = @testset Collect "probe" begin + f() + end + return _descriptions(ts) +end + function gate_failed(f) ts = @testset Collect "probe" begin f() @@ -247,6 +259,17 @@ function gate_failed(f) end _flatten(ts::Collect) = reduce(vcat, (_flatten(r) for r in ts.results); init=Any[]) + +# Every `@testset` description in the tree. A gate's description is user-facing text — it is what +# a reader sees when CI goes red — so it is worth asserting on and not only the pass/fail. +function _descriptions(ts::Collect) + return vcat( + [ts.description], + reduce( + vcat, (_descriptions(r) for r in ts.results if r isa Collect); init=String[] + ), + ) +end function _flatten(ts::Test.DefaultTestSet) return reduce(vcat, (_flatten(r) for r in ts.results); init=Any[]) end @@ -278,6 +301,10 @@ end @test ExperimentalAPI.test_surface(Shown; max_marks=1) isa ExperimentalAPI.Audit @test !gate_failed(() -> ExperimentalAPI.test_surface(Shown; max_marks=1)) @test gate_failed(() -> ExperimentalAPI.test_surface(Shown; max_marks=0)) + # …and the cap of one names itself in the singular. One is the cap a package sets when it + # means "the next mark is a decision", so it is the description most likely to be read. + descs = gate_descriptions(() -> ExperimentalAPI.test_surface(Shown; max_marks=1)) + @test any(d -> occursin("at most 1 mark", d) && !occursin("at most 1 marks", d), descs) end @testset "a mark older than N releases is reported" begin From 248e35772874b4c11dac10b362c44cfbd211bced Mon Sep 17 00:00:00 2001 From: sotashimozono Date: Wed, 9 Sep 2026 20:00:01 +0000 Subject: [PATCH 2/2] fix: the new helper sat between a docstring and the function it documents MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review catch. `gate_descriptions` went in above `gate_failed`, between that function and its docstring — and the comment above it made a second separator. Measured rather than assumed, because the failure is worse than misattribution: docstring and definition adjacent docstring recorded under :f one blank line between them NO DOCSTRING RECORDED one comment line between them NO DOCSTRING RECORDED a comment and a blank line NO DOCSTRING RECORDED So `gate_failed` was not documenting the wrong thing — it had no docstring at all. The helper moves below `gate_failed`, and the comment goes: `_descriptions` two lines down already carries the reason descriptions are worth asserting on, and "in a function like its neighbour" is visible from the neighbour. Swept the repository for the same shape with the rule now known: one candidate, and it is the closing quote of a multi-line string constant rather than a docstring. `src/` is clean because `test_surface(ExperimentalAPI)` asserts every public name has a docstring, so a detachment there goes red — this one was in a test file, where nothing was looking. 193 behaviours, green. Co-Authored-By: Claude Opus 5 --- test/spec/test_spec_integration.jl | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/test/spec/test_spec_integration.jl b/test/spec/test_spec_integration.jl index aa87d05..9b97384 100644 --- a/test/spec/test_spec_integration.jl +++ b/test/spec/test_spec_integration.jl @@ -240,22 +240,18 @@ Run `f` under a test set that records instead of propagating, and say whether an failed. This is how a gate is shown to fire without the failure it is supposed to produce reaching the suite that is checking for it. """ -# Every `@testset` description `f` produced. A gate's description is user-facing text — it is what -# a reader sees when CI goes red — so it is worth asserting on, not only the pass/fail. In a -# function for the same reason `gate_failed` is: a `@testset` written inline here would be counted -# as a behaviour of this file that carries no assertion of its own. -function gate_descriptions(f) +function gate_failed(f) ts = @testset Collect "probe" begin f() end - return _descriptions(ts) + return any(r -> r isa Test.Fail || r isa Test.Error, _flatten(ts)) end -function gate_failed(f) +function gate_descriptions(f) ts = @testset Collect "probe" begin f() end - return any(r -> r isa Test.Fail || r isa Test.Error, _flatten(ts)) + return _descriptions(ts) end _flatten(ts::Collect) = reduce(vcat, (_flatten(r) for r in ts.results); init=Any[])