From 2da9f21a6be86e1b69c4354d75ab3b09928143a2 Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Wed, 18 Nov 2026 15:39:05 +0000 Subject: [PATCH 1/3] feat(rspec): report shared and library-defined examples at their spec location With TRUNK_REPORT_SPEC_LOCATION=true, an example defined outside its spec file (a shared example, or rswag's run_test!) is reported at the spec file RSpec reruns it from: the it_behaves_like or include_examples line, or else the nearest enclosing group in the spec. Code owners follow that file. It is opt-in because the file is part of a test's id. Co-Authored-By: Claude Opus 5.5 --- rspec-trunk-flaky-tests/README.md | 6 ++ .../lib/trunk_spec_helper.rb | 33 +++++++- rspec-trunk-flaky-tests/test/location_spec.rb | 79 +++++++++++++++++++ .../test/support/location_helpers.rb | 10 +++ 4 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 rspec-trunk-flaky-tests/test/location_spec.rb create mode 100644 rspec-trunk-flaky-tests/test/support/location_helpers.rb diff --git a/rspec-trunk-flaky-tests/README.md b/rspec-trunk-flaky-tests/README.md index fdaf0f8f..783ec4fb 100644 --- a/rspec-trunk-flaky-tests/README.md +++ b/rspec-trunk-flaky-tests/README.md @@ -126,6 +126,12 @@ When set to a file path, the gem writes a JSON summary of the run to that file a TRUNK_SUMMARY_OUTPUT_FILE=tmp/trunk-summary.json bundle exec rspec ``` +#### `TRUNK_REPORT_SPEC_LOCATION` + +When set to `true`, an example defined outside its spec file, such as a shared example or rswag's `run_test!`, is reported at the spec file RSpec reruns it from, and its code owners come from that file. The line is the `it_behaves_like` or `include_examples` call, or else the enclosing group in the spec. Other examples are unchanged. + +The file is part of a test's id, so turning this on gives most of those examples new ids in Trunk, with no history, and quarantines set on their old ids no longer apply to them. + #### `TRUNK_LOCAL_UPLOAD_DIR` (Experimental) > **⚠️ Experimental Feature**: This feature is experimental. Please reach out to support@trunk.io before attempting to use it. diff --git a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb index d14ff5ab..602a687c 100644 --- a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb +++ b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb @@ -32,6 +32,8 @@ # TRUNK_QUARANTINED_TESTS_DISK_CACHE_TTL_SECS - Time to cache quarantined tests on disk (in seconds) # TRUNK_QUARANTINE_QUERY_FAILURE_EXIT - Set to 'true' to abort the RSpec run when quarantine # lookup fails (remaining examples are skipped) +# TRUNK_REPORT_SPEC_LOCATION - Set to 'true' to report an example defined outside its spec file +# (a shared example, or one a library defines) at the spec file RSpec reruns it from # DISABLE_RSPEC_TRUNK_FLAKY_TESTS - Set to 'true' to completely disable Trunk # require 'rspec/core' @@ -145,6 +147,10 @@ def disable_quarantining? ENV['TRUNK_DISABLE_QUARANTINING'] == 'true' end + def report_spec_location? + ENV['TRUNK_REPORT_SPEC_LOCATION'] == 'true' + end + def command_line "#{$PROGRAM_NAME} #{ARGV.join(' ')}" end @@ -173,10 +179,33 @@ def strip_ansi_codes(text) # The escaped file path and the dotted classname derived from it. def file_and_classname(example) - file = escape(example.metadata[:file_path]) + file = escape(location(example).first) [file, file.sub(%r{\.[^/.]+\Z}, '').gsub('/', '.').gsub(/\A\.+|\.+\Z/, '')] end + # The file and line the example is reported at. With TRUNK_REPORT_SPEC_LOCATION, an example + # defined outside its spec file is reported at the line in the spec that brought it in. + def location(example) + metadata = example.metadata + file = metadata[:rerun_file_path] + return [metadata[:file_path], metadata[:line_number]] if !report_spec_location? || metadata[:file_path] == file + + [file, inclusion_line(metadata, file) || group_line(example, file) || metadata[:line_number]] + end + + def inclusion_line(metadata, file) + Array(metadata[:shared_group_inclusion_backtrace]).reverse_each do |frame| + path, line = frame.inclusion_location.sub(/:in .+\z/m, '').match(/\A(.+):(\d+)\z/)&.captures + return line.to_i if path && RSpec::Core::Metadata.relative_path(path) == file + end + nil + end + + def group_line(example, file) + group = example.example_group.parent_groups.find { |parent| parent.metadata[:file_path] == file } + group&.metadata&.fetch(:line_number) + end + def parent_name(example) name = description(example.example_group.metadata) name.empty? ? 'rspec' : name @@ -598,7 +627,7 @@ def add_test_case(example) # TODO: should we use concatenated string or alias when auto-generated description? name = Trunk.full_description(example) file, classname = Trunk.file_and_classname(example) - line = example.metadata[:line_number] + line = Trunk.location(example).last started_at = example.execution_result.started_at.to_f finished_at = example.execution_result.finished_at.to_f diff --git a/rspec-trunk-flaky-tests/test/location_spec.rb b/rspec-trunk-flaky-tests/test/location_spec.rb new file mode 100644 index 00000000..dc329e09 --- /dev/null +++ b/rspec-trunk-flaky-tests/test/location_spec.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +require 'rspec_trunk_flaky_tests' +require_relative '../spec/spec_helper' +require_relative 'support/location_helpers' +require 'rspec/core/sandbox' + +# trunk-ignore(rubocop/Metrics/BlockLength) +RSpec.describe 'reported test location' do + spec_file = './test/location_spec.rb' + helpers_file = './test/support/location_helpers.rb' + lines = {} + + # trunk-ignore(rubocop/Metrics/AbcSize,rubocop/Metrics/MethodLength) + define_method(:locations) do + examples = nil + RSpec::Core::Sandbox.sandboxed do |_config| + group = RSpec.describe('API') do + extend LocationHelpers + shared_examples('shared', &LocationHelpers::SHARED_EXAMPLES) + lines[:own] = __LINE__ + 1 + it('is its own') { expect(1).to eq(1) } + lines[:it_behaves_like] = __LINE__ + 1 + it_behaves_like 'shared' + context 'included' do + lines[:include_examples] = __LINE__ + 1 + include_examples 'shared' + end + lines[:response] = __LINE__ + 1 + context('response 200') { run_test! } + end + group.run(RSpec::Core::NullReporter) + examples = group.descendants.flat_map(&:examples) + end + examples.to_h { |example| [example.full_description, RSpec::Trunk.location(example)] } + end + + around do |example| + previous = ENV.fetch('TRUNK_REPORT_SPEC_LOCATION', nil) + ENV['TRUNK_REPORT_SPEC_LOCATION'] = report_spec_location + example.run + ensure + ENV['TRUNK_REPORT_SPEC_LOCATION'] = previous + end + + context 'with TRUNK_REPORT_SPEC_LOCATION' do + let(:report_spec_location) { 'true' } + + it 'reports each example at the spec line that brought it in' do + expect(locations).to eq( + 'API is its own' => [spec_file, lines[:own]], + 'API behaves like shared is shared' => [spec_file, lines[:it_behaves_like]], + 'API included is shared' => [spec_file, lines[:include_examples]], + 'API response 200 returns a 200 response' => [spec_file, lines[:response]] + ) + end + + it 'derives the classname from the spec file' do + example = nil + RSpec::Core::Sandbox.sandboxed do |_config| + group = RSpec.describe('API') do + extend LocationHelpers + run_test! + end + example = group.examples.first + end + expect(RSpec::Trunk.file_and_classname(example)).to eq([spec_file, 'test.location_spec']) + end + end + + context 'without it' do + let(:report_spec_location) { nil } + + it 'reports each example where it is defined' do + expect(locations.values.map(&:first)).to eq([spec_file, helpers_file, helpers_file, helpers_file]) + expect(locations['API is its own']).to eq([spec_file, lines[:own]]) + end + end +end diff --git a/rspec-trunk-flaky-tests/test/support/location_helpers.rb b/rspec-trunk-flaky-tests/test/support/location_helpers.rb new file mode 100644 index 00000000..3df5c704 --- /dev/null +++ b/rspec-trunk-flaky-tests/test/support/location_helpers.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +# Defines examples from a file other than the spec, as shared examples and rswag's run_test! do. +module LocationHelpers + SHARED_EXAMPLES = proc { it('is shared') { expect(1).to eq(1) } } + + def run_test! + it('returns a 200 response') { expect(1).to eq(1) } + end +end From a9e50a45f84ac5f6d628e4ba8392973addd2159b Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Wed, 18 Nov 2026 15:53:37 +0000 Subject: [PATCH 2/3] test(rspec): cover run_test! with the real rswag-specs gem The location spec now defines its rswag example with rswag-specs 2.17.0 instead of a stand-in. The lockfile is resolved under Ruby 3.1, which the unit-test job runs, so it takes Rails 7.2 components. The upload harness Gemfile gets rswag-specs too, since the main-branch smoke run executes the same suite there. A shared-group inclusion frame with no location no longer raises. Co-Authored-By: Claude Opus 5.5 --- .github/actions/test_ruby_gem_uploads/Gemfile | 1 + rspec-trunk-flaky-tests/Gemfile | 1 + rspec-trunk-flaky-tests/Gemfile.lock | 126 ++++++++++++++++++ .../lib/trunk_spec_helper.rb | 2 +- rspec-trunk-flaky-tests/test/location_spec.rb | 23 ++-- .../test/support/location_helpers.rb | 6 +- 6 files changed, 145 insertions(+), 14 deletions(-) diff --git a/.github/actions/test_ruby_gem_uploads/Gemfile b/.github/actions/test_ruby_gem_uploads/Gemfile index 16522d0e..1d51be08 100644 --- a/.github/actions/test_ruby_gem_uploads/Gemfile +++ b/.github/actions/test_ruby_gem_uploads/Gemfile @@ -8,4 +8,5 @@ gem 'knapsack_pro' gem 'rake' gem 'rake-compiler' gem 'logger' +gem 'rswag-specs' gem 'rspec_trunk_flaky_tests', :path => '../../../rspec-trunk-flaky-tests' diff --git a/rspec-trunk-flaky-tests/Gemfile b/rspec-trunk-flaky-tests/Gemfile index 0fc9d524..737d3494 100644 --- a/rspec-trunk-flaky-tests/Gemfile +++ b/rspec-trunk-flaky-tests/Gemfile @@ -6,3 +6,4 @@ gemspec gem 'rake' gem 'rake-compiler', '1.3.1' +gem 'rswag-specs', '2.17.0' diff --git a/rspec-trunk-flaky-tests/Gemfile.lock b/rspec-trunk-flaky-tests/Gemfile.lock index 2d033305..d3ca402c 100644 --- a/rspec-trunk-flaky-tests/Gemfile.lock +++ b/rspec-trunk-flaky-tests/Gemfile.lock @@ -8,13 +8,125 @@ PATH GEM remote: https://rubygems.org/ specs: + actionpack (7.2.4) + actionview (= 7.2.4) + activesupport (= 7.2.4) + cgi + nokogiri (>= 1.8.5) + racc + rack (>= 2.2.4, < 3.3) + rack-session (>= 1.0.1) + rack-test (>= 0.6.3) + rails-dom-testing (~> 2.2) + rails-html-sanitizer (~> 1.6) + useragent (~> 0.16) + actionview (7.2.4) + activesupport (= 7.2.4) + builder (~> 3.1) + cgi + erubi (~> 1.11) + rails-dom-testing (~> 2.2) + rails-html-sanitizer (~> 1.6) + activesupport (7.2.4) + base64 + benchmark (>= 0.3) + bigdecimal + concurrent-ruby (~> 1.0, >= 1.3.1) + connection_pool (>= 2.2.5, < 3) + drb + i18n (>= 1.6, < 2) + logger (>= 1.4.2) + minitest (>= 5.1, < 6) + securerandom (>= 0.3) + tzinfo (~> 2.0, >= 2.0.5) + addressable (2.9.2) + public_suffix (>= 2.0.2, < 8.0) + base64 (0.3.0) + benchmark (0.5.0) + bigdecimal (4.1.3) + builder (3.3.0) + cgi (0.5.2) + concurrent-ruby (1.3.8) + connection_pool (2.5.5) + crass (1.0.8) + date (3.6.0) diff-lcs (1.5.1) + drb (2.2.3) + erb (4.0.4.1) + cgi (>= 0.3.3) + erubi (1.13.1) + i18n (1.15.2) + concurrent-ruby (~> 1.0) + io-console (0.9.5) + irb (1.19.0) + pp (>= 0.6.0) + prism (>= 1.3.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + json-schema (6.2.1) + addressable (~> 2.8) + bigdecimal (>= 3.1, < 5) + logger (1.7.0) + loofah (2.25.2) + crass (~> 1.0.2) + nokogiri (>= 1.12.0) + mini_portile2 (2.8.9) + minitest (5.27.0) + nokogiri (1.18.10) + mini_portile2 (~> 2.8.2) + racc (~> 1.4) + nokogiri (1.18.10-arm64-darwin) + racc (~> 1.4) + nokogiri (1.18.10-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.18.10-x86_64-linux-gnu) + racc (~> 1.4) + pp (0.6.4) + prettyprint + prettyprint (0.2.1) + prism (2.1.0) + psych (5.5.0) + date + stringio + public_suffix (6.0.2) + racc (1.8.1) + rack (3.2.8) + rack-session (2.1.2) + base64 (>= 0.1.0) + rack (>= 3.0.0) + rack-test (2.3.0) + rack (>= 1.3) + rackup (2.3.1) + rack (>= 3) + rails-dom-testing (2.3.0) + activesupport (>= 5.0.0) + minitest + nokogiri (>= 1.6) + rails-html-sanitizer (1.7.1) + loofah (~> 2.25, >= 2.25.2) + nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) + railties (7.2.4) + actionpack (= 7.2.4) + activesupport (= 7.2.4) + cgi + irb (~> 1.13) + rackup (>= 1.0.0) + rake (>= 12.2) + thor (~> 1.0, >= 1.2.2) + tsort (>= 0.2) + zeitwerk (~> 2.6) rake (13.2.1) rake-compiler (1.3.1) rake rake-compiler-dock (1.12.0) rb_sys (0.9.130) rake-compiler-dock (= 1.12.0) + rdoc (7.2.0) + erb + psych (>= 4.0.0) + tsort + reline (0.7.0) + io-console (~> 0.5) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) @@ -28,6 +140,19 @@ GEM diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) rspec-support (3.13.2) + rswag-specs (2.17.0) + activesupport (>= 5.2, < 8.2) + json-schema (>= 2.2, < 7.0) + railties (>= 5.2, < 8.2) + rspec-core (>= 2.14) + securerandom (0.4.1) + stringio (3.2.0) + thor (1.5.0) + tsort (0.2.0) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + useragent (0.16.11) + zeitwerk (2.6.18) PLATFORMS arm64-darwin @@ -41,6 +166,7 @@ DEPENDENCIES rake-compiler (= 1.3.1) rspec rspec_trunk_flaky_tests! + rswag-specs (= 2.17.0) BUNDLED WITH 2.5.23 diff --git a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb index 602a687c..c119d643 100644 --- a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb +++ b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb @@ -195,7 +195,7 @@ def location(example) def inclusion_line(metadata, file) Array(metadata[:shared_group_inclusion_backtrace]).reverse_each do |frame| - path, line = frame.inclusion_location.sub(/:in .+\z/m, '').match(/\A(.+):(\d+)\z/)&.captures + path, line = frame.inclusion_location.to_s.sub(/:in .+\z/m, '').match(/\A(.+):(\d+)\z/)&.captures return line.to_i if path && RSpec::Core::Metadata.relative_path(path) == file end nil diff --git a/rspec-trunk-flaky-tests/test/location_spec.rb b/rspec-trunk-flaky-tests/test/location_spec.rb index dc329e09..447f3bcd 100644 --- a/rspec-trunk-flaky-tests/test/location_spec.rb +++ b/rspec-trunk-flaky-tests/test/location_spec.rb @@ -3,12 +3,15 @@ require 'rspec_trunk_flaky_tests' require_relative '../spec/spec_helper' require_relative 'support/location_helpers' +require 'rails/version' +require 'rswag/specs' require 'rspec/core/sandbox' # trunk-ignore(rubocop/Metrics/BlockLength) RSpec.describe 'reported test location' do spec_file = './test/location_spec.rb' helpers_file = './test/support/location_helpers.rb' + rswag_file = Rswag::Specs::ExampleGroupHelpers.instance_method(:run_test!).source_location.first lines = {} # trunk-ignore(rubocop/Metrics/AbcSize,rubocop/Metrics/MethodLength) @@ -16,7 +19,7 @@ examples = nil RSpec::Core::Sandbox.sandboxed do |_config| group = RSpec.describe('API') do - extend LocationHelpers + extend Rswag::Specs::ExampleGroupHelpers shared_examples('shared', &LocationHelpers::SHARED_EXAMPLES) lines[:own] = __LINE__ + 1 it('is its own') { expect(1).to eq(1) } @@ -26,8 +29,12 @@ lines[:include_examples] = __LINE__ + 1 include_examples 'shared' end - lines[:response] = __LINE__ + 1 - context('response 200') { run_test! } + path('/users') do + get('lists users') do + lines[:response] = __LINE__ + 1 + response('200', 'ok') { run_test! } + end + end end group.run(RSpec::Core::NullReporter) examples = group.descendants.flat_map(&:examples) @@ -51,7 +58,7 @@ 'API is its own' => [spec_file, lines[:own]], 'API behaves like shared is shared' => [spec_file, lines[:it_behaves_like]], 'API included is shared' => [spec_file, lines[:include_examples]], - 'API response 200 returns a 200 response' => [spec_file, lines[:response]] + 'API /users get ok returns a 200 response' => [spec_file, lines[:response]] ) end @@ -59,10 +66,10 @@ example = nil RSpec::Core::Sandbox.sandboxed do |_config| group = RSpec.describe('API') do - extend LocationHelpers - run_test! + extend Rswag::Specs::ExampleGroupHelpers + path('/users') { get('lists users') { response('200', 'ok') { run_test! } } } end - example = group.examples.first + example = group.descendants.flat_map(&:examples).first end expect(RSpec::Trunk.file_and_classname(example)).to eq([spec_file, 'test.location_spec']) end @@ -72,7 +79,7 @@ let(:report_spec_location) { nil } it 'reports each example where it is defined' do - expect(locations.values.map(&:first)).to eq([spec_file, helpers_file, helpers_file, helpers_file]) + expect(locations.values.map(&:first)).to eq([spec_file, helpers_file, helpers_file, rswag_file]) expect(locations['API is its own']).to eq([spec_file, lines[:own]]) end end diff --git a/rspec-trunk-flaky-tests/test/support/location_helpers.rb b/rspec-trunk-flaky-tests/test/support/location_helpers.rb index 3df5c704..28a26262 100644 --- a/rspec-trunk-flaky-tests/test/support/location_helpers.rb +++ b/rspec-trunk-flaky-tests/test/support/location_helpers.rb @@ -1,10 +1,6 @@ # frozen_string_literal: true -# Defines examples from a file other than the spec, as shared examples and rswag's run_test! do. +# Shared examples defined in a file other than the spec that includes them. module LocationHelpers SHARED_EXAMPLES = proc { it('is shared') { expect(1).to eq(1) } } - - def run_test! - it('returns a 200 response') { expect(1).to eq(1) } - end end From 7f08fb3b768be51b0387bcb462e02276fefee459 Mon Sep 17 00:00:00 2001 From: Tyler Jang Date: Thu, 19 Nov 2026 15:48:52 +0000 Subject: [PATCH 3/3] ci(rspec): lock the upload harness to the built gem's platform rswag-specs brings in nokogiri, which ships native builds for many platforms, and Bundler puts every one of them in the harness lockfile. The built gem exists for one platform only, so `bundle install --local` could not resolve it. The harness now locks to the gem's own platform before adding it to the Gemfile. Co-Authored-By: Claude Opus 5.5 --- .github/actions/test_ruby_gem_uploads/action.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/test_ruby_gem_uploads/action.yaml b/.github/actions/test_ruby_gem_uploads/action.yaml index 0317a46f..c6efb210 100644 --- a/.github/actions/test_ruby_gem_uploads/action.yaml +++ b/.github/actions/test_ruby_gem_uploads/action.yaml @@ -108,6 +108,7 @@ runs: # Read the version from gem metadata (robust to the platform suffix). GEM_VERSION=$(ruby -e "require 'rubygems/package'; puts Gem::Package.new(ARGV[0]).spec.version" "${GEM_FILE}") echo "Extracted version: ${GEM_VERSION}" + GEM_PLATFORM=$(ruby -e "require 'rubygems/package'; puts Gem::Package.new(ARGV[0]).spec.platform" "${GEM_FILE}") gem install "${GEM_FILE}" --local # Unpack the specific gem file directly, not from installed gems @@ -118,9 +119,13 @@ runs: sed -i.bak "/gem 'rspec_trunk_flaky_tests'.*/d" Gemfile && rm -f Gemfile.bak bundle install + # The built gem exists for one platform, but a dependency with native builds (nokogiri) + # puts every platform it ships for in the lockfile, so lock to the gem's platform alone. + bundle lock --add-platform "${GEM_PLATFORM}" + OTHER_PLATFORMS=$(ruby -rbundler -e 'puts Bundler::LockfileParser.new(File.read("Gemfile.lock")).platforms.map(&:to_s) - [ARGV[0]]' "${GEM_PLATFORM}") + [ -z "${OTHER_PLATFORMS}" ] || bundle lock --remove-platform ${OTHER_PLATFORMS} echo "gem 'rspec_trunk_flaky_tests', '${GEM_VERSION}', :path => './rspec_trunk_flaky_tests'" >> Gemfile - bundle lock --remove-platform ruby || true bundle install --gemfile Gemfile --local - name: Run regular tests