diff --git a/.github/actions/test_ruby_gem_uploads/Gemfile b/.github/actions/test_ruby_gem_uploads/Gemfile index 16522d0e..1d51be08 100644 --- a/.github/actions/test_ruby_gem_uploads/Gemfile +++ b/.github/actions/test_ruby_gem_uploads/Gemfile @@ -8,4 +8,5 @@ gem 'knapsack_pro' gem 'rake' gem 'rake-compiler' gem 'logger' +gem 'rswag-specs' gem 'rspec_trunk_flaky_tests', :path => '../../../rspec-trunk-flaky-tests' diff --git a/.github/actions/test_ruby_gem_uploads/action.yaml b/.github/actions/test_ruby_gem_uploads/action.yaml index 0317a46f..c6efb210 100644 --- a/.github/actions/test_ruby_gem_uploads/action.yaml +++ b/.github/actions/test_ruby_gem_uploads/action.yaml @@ -108,6 +108,7 @@ runs: # Read the version from gem metadata (robust to the platform suffix). GEM_VERSION=$(ruby -e "require 'rubygems/package'; puts Gem::Package.new(ARGV[0]).spec.version" "${GEM_FILE}") echo "Extracted version: ${GEM_VERSION}" + GEM_PLATFORM=$(ruby -e "require 'rubygems/package'; puts Gem::Package.new(ARGV[0]).spec.platform" "${GEM_FILE}") gem install "${GEM_FILE}" --local # Unpack the specific gem file directly, not from installed gems @@ -118,9 +119,13 @@ runs: sed -i.bak "/gem 'rspec_trunk_flaky_tests'.*/d" Gemfile && rm -f Gemfile.bak bundle install + # The built gem exists for one platform, but a dependency with native builds (nokogiri) + # puts every platform it ships for in the lockfile, so lock to the gem's platform alone. + bundle lock --add-platform "${GEM_PLATFORM}" + OTHER_PLATFORMS=$(ruby -rbundler -e 'puts Bundler::LockfileParser.new(File.read("Gemfile.lock")).platforms.map(&:to_s) - [ARGV[0]]' "${GEM_PLATFORM}") + [ -z "${OTHER_PLATFORMS}" ] || bundle lock --remove-platform ${OTHER_PLATFORMS} echo "gem 'rspec_trunk_flaky_tests', '${GEM_VERSION}', :path => './rspec_trunk_flaky_tests'" >> Gemfile - bundle lock --remove-platform ruby || true bundle install --gemfile Gemfile --local - name: Run regular tests diff --git a/rspec-trunk-flaky-tests/Gemfile b/rspec-trunk-flaky-tests/Gemfile index 0fc9d524..737d3494 100644 --- a/rspec-trunk-flaky-tests/Gemfile +++ b/rspec-trunk-flaky-tests/Gemfile @@ -6,3 +6,4 @@ gemspec gem 'rake' gem 'rake-compiler', '1.3.1' +gem 'rswag-specs', '2.17.0' diff --git a/rspec-trunk-flaky-tests/Gemfile.lock b/rspec-trunk-flaky-tests/Gemfile.lock index 2d033305..d3ca402c 100644 --- a/rspec-trunk-flaky-tests/Gemfile.lock +++ b/rspec-trunk-flaky-tests/Gemfile.lock @@ -8,13 +8,125 @@ PATH GEM remote: https://rubygems.org/ specs: + actionpack (7.2.4) + actionview (= 7.2.4) + activesupport (= 7.2.4) + cgi + nokogiri (>= 1.8.5) + racc + rack (>= 2.2.4, < 3.3) + rack-session (>= 1.0.1) + rack-test (>= 0.6.3) + rails-dom-testing (~> 2.2) + rails-html-sanitizer (~> 1.6) + useragent (~> 0.16) + actionview (7.2.4) + activesupport (= 7.2.4) + builder (~> 3.1) + cgi + erubi (~> 1.11) + rails-dom-testing (~> 2.2) + rails-html-sanitizer (~> 1.6) + activesupport (7.2.4) + base64 + benchmark (>= 0.3) + bigdecimal + concurrent-ruby (~> 1.0, >= 1.3.1) + connection_pool (>= 2.2.5, < 3) + drb + i18n (>= 1.6, < 2) + logger (>= 1.4.2) + minitest (>= 5.1, < 6) + securerandom (>= 0.3) + tzinfo (~> 2.0, >= 2.0.5) + addressable (2.9.2) + public_suffix (>= 2.0.2, < 8.0) + base64 (0.3.0) + benchmark (0.5.0) + bigdecimal (4.1.3) + builder (3.3.0) + cgi (0.5.2) + concurrent-ruby (1.3.8) + connection_pool (2.5.5) + crass (1.0.8) + date (3.6.0) diff-lcs (1.5.1) + drb (2.2.3) + erb (4.0.4.1) + cgi (>= 0.3.3) + erubi (1.13.1) + i18n (1.15.2) + concurrent-ruby (~> 1.0) + io-console (0.9.5) + irb (1.19.0) + pp (>= 0.6.0) + prism (>= 1.3.0) + rdoc (>= 4.0.0) + reline (>= 0.4.2) + json-schema (6.2.1) + addressable (~> 2.8) + bigdecimal (>= 3.1, < 5) + logger (1.7.0) + loofah (2.25.2) + crass (~> 1.0.2) + nokogiri (>= 1.12.0) + mini_portile2 (2.8.9) + minitest (5.27.0) + nokogiri (1.18.10) + mini_portile2 (~> 2.8.2) + racc (~> 1.4) + nokogiri (1.18.10-arm64-darwin) + racc (~> 1.4) + nokogiri (1.18.10-x86_64-darwin) + racc (~> 1.4) + nokogiri (1.18.10-x86_64-linux-gnu) + racc (~> 1.4) + pp (0.6.4) + prettyprint + prettyprint (0.2.1) + prism (2.1.0) + psych (5.5.0) + date + stringio + public_suffix (6.0.2) + racc (1.8.1) + rack (3.2.8) + rack-session (2.1.2) + base64 (>= 0.1.0) + rack (>= 3.0.0) + rack-test (2.3.0) + rack (>= 1.3) + rackup (2.3.1) + rack (>= 3) + rails-dom-testing (2.3.0) + activesupport (>= 5.0.0) + minitest + nokogiri (>= 1.6) + rails-html-sanitizer (1.7.1) + loofah (~> 2.25, >= 2.25.2) + nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0) + railties (7.2.4) + actionpack (= 7.2.4) + activesupport (= 7.2.4) + cgi + irb (~> 1.13) + rackup (>= 1.0.0) + rake (>= 12.2) + thor (~> 1.0, >= 1.2.2) + tsort (>= 0.2) + zeitwerk (~> 2.6) rake (13.2.1) rake-compiler (1.3.1) rake rake-compiler-dock (1.12.0) rb_sys (0.9.130) rake-compiler-dock (= 1.12.0) + rdoc (7.2.0) + erb + psych (>= 4.0.0) + tsort + reline (0.7.0) + io-console (~> 0.5) rspec (3.13.0) rspec-core (~> 3.13.0) rspec-expectations (~> 3.13.0) @@ -28,6 +140,19 @@ GEM diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.13.0) rspec-support (3.13.2) + rswag-specs (2.17.0) + activesupport (>= 5.2, < 8.2) + json-schema (>= 2.2, < 7.0) + railties (>= 5.2, < 8.2) + rspec-core (>= 2.14) + securerandom (0.4.1) + stringio (3.2.0) + thor (1.5.0) + tsort (0.2.0) + tzinfo (2.0.6) + concurrent-ruby (~> 1.0) + useragent (0.16.11) + zeitwerk (2.6.18) PLATFORMS arm64-darwin @@ -41,6 +166,7 @@ DEPENDENCIES rake-compiler (= 1.3.1) rspec rspec_trunk_flaky_tests! + rswag-specs (= 2.17.0) BUNDLED WITH 2.5.23 diff --git a/rspec-trunk-flaky-tests/README.md b/rspec-trunk-flaky-tests/README.md index fdaf0f8f..783ec4fb 100644 --- a/rspec-trunk-flaky-tests/README.md +++ b/rspec-trunk-flaky-tests/README.md @@ -126,6 +126,12 @@ When set to a file path, the gem writes a JSON summary of the run to that file a TRUNK_SUMMARY_OUTPUT_FILE=tmp/trunk-summary.json bundle exec rspec ``` +#### `TRUNK_REPORT_SPEC_LOCATION` + +When set to `true`, an example defined outside its spec file, such as a shared example or rswag's `run_test!`, is reported at the spec file RSpec reruns it from, and its code owners come from that file. The line is the `it_behaves_like` or `include_examples` call, or else the enclosing group in the spec. Other examples are unchanged. + +The file is part of a test's id, so turning this on gives most of those examples new ids in Trunk, with no history, and quarantines set on their old ids no longer apply to them. + #### `TRUNK_LOCAL_UPLOAD_DIR` (Experimental) > **⚠️ Experimental Feature**: This feature is experimental. Please reach out to support@trunk.io before attempting to use it. diff --git a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb index d14ff5ab..c119d643 100644 --- a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb +++ b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb @@ -32,6 +32,8 @@ # TRUNK_QUARANTINED_TESTS_DISK_CACHE_TTL_SECS - Time to cache quarantined tests on disk (in seconds) # TRUNK_QUARANTINE_QUERY_FAILURE_EXIT - Set to 'true' to abort the RSpec run when quarantine # lookup fails (remaining examples are skipped) +# TRUNK_REPORT_SPEC_LOCATION - Set to 'true' to report an example defined outside its spec file +# (a shared example, or one a library defines) at the spec file RSpec reruns it from # DISABLE_RSPEC_TRUNK_FLAKY_TESTS - Set to 'true' to completely disable Trunk # require 'rspec/core' @@ -145,6 +147,10 @@ def disable_quarantining? ENV['TRUNK_DISABLE_QUARANTINING'] == 'true' end + def report_spec_location? + ENV['TRUNK_REPORT_SPEC_LOCATION'] == 'true' + end + def command_line "#{$PROGRAM_NAME} #{ARGV.join(' ')}" end @@ -173,10 +179,33 @@ def strip_ansi_codes(text) # The escaped file path and the dotted classname derived from it. def file_and_classname(example) - file = escape(example.metadata[:file_path]) + file = escape(location(example).first) [file, file.sub(%r{\.[^/.]+\Z}, '').gsub('/', '.').gsub(/\A\.+|\.+\Z/, '')] end + # The file and line the example is reported at. With TRUNK_REPORT_SPEC_LOCATION, an example + # defined outside its spec file is reported at the line in the spec that brought it in. + def location(example) + metadata = example.metadata + file = metadata[:rerun_file_path] + return [metadata[:file_path], metadata[:line_number]] if !report_spec_location? || metadata[:file_path] == file + + [file, inclusion_line(metadata, file) || group_line(example, file) || metadata[:line_number]] + end + + def inclusion_line(metadata, file) + Array(metadata[:shared_group_inclusion_backtrace]).reverse_each do |frame| + path, line = frame.inclusion_location.to_s.sub(/:in .+\z/m, '').match(/\A(.+):(\d+)\z/)&.captures + return line.to_i if path && RSpec::Core::Metadata.relative_path(path) == file + end + nil + end + + def group_line(example, file) + group = example.example_group.parent_groups.find { |parent| parent.metadata[:file_path] == file } + group&.metadata&.fetch(:line_number) + end + def parent_name(example) name = description(example.example_group.metadata) name.empty? ? 'rspec' : name @@ -598,7 +627,7 @@ def add_test_case(example) # TODO: should we use concatenated string or alias when auto-generated description? name = Trunk.full_description(example) file, classname = Trunk.file_and_classname(example) - line = example.metadata[:line_number] + line = Trunk.location(example).last started_at = example.execution_result.started_at.to_f finished_at = example.execution_result.finished_at.to_f diff --git a/rspec-trunk-flaky-tests/test/location_spec.rb b/rspec-trunk-flaky-tests/test/location_spec.rb new file mode 100644 index 00000000..447f3bcd --- /dev/null +++ b/rspec-trunk-flaky-tests/test/location_spec.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +require 'rspec_trunk_flaky_tests' +require_relative '../spec/spec_helper' +require_relative 'support/location_helpers' +require 'rails/version' +require 'rswag/specs' +require 'rspec/core/sandbox' + +# trunk-ignore(rubocop/Metrics/BlockLength) +RSpec.describe 'reported test location' do + spec_file = './test/location_spec.rb' + helpers_file = './test/support/location_helpers.rb' + rswag_file = Rswag::Specs::ExampleGroupHelpers.instance_method(:run_test!).source_location.first + lines = {} + + # trunk-ignore(rubocop/Metrics/AbcSize,rubocop/Metrics/MethodLength) + define_method(:locations) do + examples = nil + RSpec::Core::Sandbox.sandboxed do |_config| + group = RSpec.describe('API') do + extend Rswag::Specs::ExampleGroupHelpers + shared_examples('shared', &LocationHelpers::SHARED_EXAMPLES) + lines[:own] = __LINE__ + 1 + it('is its own') { expect(1).to eq(1) } + lines[:it_behaves_like] = __LINE__ + 1 + it_behaves_like 'shared' + context 'included' do + lines[:include_examples] = __LINE__ + 1 + include_examples 'shared' + end + path('/users') do + get('lists users') do + lines[:response] = __LINE__ + 1 + response('200', 'ok') { run_test! } + end + end + end + group.run(RSpec::Core::NullReporter) + examples = group.descendants.flat_map(&:examples) + end + examples.to_h { |example| [example.full_description, RSpec::Trunk.location(example)] } + end + + around do |example| + previous = ENV.fetch('TRUNK_REPORT_SPEC_LOCATION', nil) + ENV['TRUNK_REPORT_SPEC_LOCATION'] = report_spec_location + example.run + ensure + ENV['TRUNK_REPORT_SPEC_LOCATION'] = previous + end + + context 'with TRUNK_REPORT_SPEC_LOCATION' do + let(:report_spec_location) { 'true' } + + it 'reports each example at the spec line that brought it in' do + expect(locations).to eq( + 'API is its own' => [spec_file, lines[:own]], + 'API behaves like shared is shared' => [spec_file, lines[:it_behaves_like]], + 'API included is shared' => [spec_file, lines[:include_examples]], + 'API /users get ok returns a 200 response' => [spec_file, lines[:response]] + ) + end + + it 'derives the classname from the spec file' do + example = nil + RSpec::Core::Sandbox.sandboxed do |_config| + group = RSpec.describe('API') do + extend Rswag::Specs::ExampleGroupHelpers + path('/users') { get('lists users') { response('200', 'ok') { run_test! } } } + end + example = group.descendants.flat_map(&:examples).first + end + expect(RSpec::Trunk.file_and_classname(example)).to eq([spec_file, 'test.location_spec']) + end + end + + context 'without it' do + let(:report_spec_location) { nil } + + it 'reports each example where it is defined' do + expect(locations.values.map(&:first)).to eq([spec_file, helpers_file, helpers_file, rswag_file]) + expect(locations['API is its own']).to eq([spec_file, lines[:own]]) + end + end +end diff --git a/rspec-trunk-flaky-tests/test/support/location_helpers.rb b/rspec-trunk-flaky-tests/test/support/location_helpers.rb new file mode 100644 index 00000000..28a26262 --- /dev/null +++ b/rspec-trunk-flaky-tests/test/support/location_helpers.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +# Shared examples defined in a file other than the spec that includes them. +module LocationHelpers + SHARED_EXAMPLES = proc { it('is shared') { expect(1).to eq(1) } } +end