Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/actions/test_ruby_gem_uploads/Gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,5 @@ gem 'knapsack_pro'
gem 'rake'
gem 'rake-compiler'
gem 'logger'
gem 'rswag-specs'
gem 'rspec_trunk_flaky_tests', :path => '../../../rspec-trunk-flaky-tests'
7 changes: 6 additions & 1 deletion .github/actions/test_ruby_gem_uploads/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ runs:
# Read the version from gem metadata (robust to the platform suffix).
GEM_VERSION=$(ruby -e "require 'rubygems/package'; puts Gem::Package.new(ARGV[0]).spec.version" "${GEM_FILE}")
echo "Extracted version: ${GEM_VERSION}"
GEM_PLATFORM=$(ruby -e "require 'rubygems/package'; puts Gem::Package.new(ARGV[0]).spec.platform" "${GEM_FILE}")

gem install "${GEM_FILE}" --local
# Unpack the specific gem file directly, not from installed gems
Expand All @@ -118,9 +119,13 @@ runs:

sed -i.bak "/gem 'rspec_trunk_flaky_tests'.*/d" Gemfile && rm -f Gemfile.bak
bundle install
# The built gem exists for one platform, but a dependency with native builds (nokogiri)
# puts every platform it ships for in the lockfile, so lock to the gem's platform alone.
bundle lock --add-platform "${GEM_PLATFORM}"
OTHER_PLATFORMS=$(ruby -rbundler -e 'puts Bundler::LockfileParser.new(File.read("Gemfile.lock")).platforms.map(&:to_s) - [ARGV[0]]' "${GEM_PLATFORM}")
[ -z "${OTHER_PLATFORMS}" ] || bundle lock --remove-platform ${OTHER_PLATFORMS}
echo "gem 'rspec_trunk_flaky_tests', '${GEM_VERSION}', :path => './rspec_trunk_flaky_tests'" >> Gemfile

bundle lock --remove-platform ruby || true
bundle install --gemfile Gemfile --local

- name: Run regular tests
Expand Down
1 change: 1 addition & 0 deletions rspec-trunk-flaky-tests/Gemfile
Original file line number Diff line number Diff line change
Expand Up @@ -6,3 +6,4 @@ gemspec

gem 'rake'
gem 'rake-compiler', '1.3.1'
gem 'rswag-specs', '2.17.0'
126 changes: 126 additions & 0 deletions rspec-trunk-flaky-tests/Gemfile.lock
Original file line number Diff line number Diff line change
Expand Up @@ -8,13 +8,125 @@ PATH
GEM
remote: https://rubygems.org/
specs:
actionpack (7.2.4)
actionview (= 7.2.4)
activesupport (= 7.2.4)
cgi
nokogiri (>= 1.8.5)
racc
rack (>= 2.2.4, < 3.3)
rack-session (>= 1.0.1)
rack-test (>= 0.6.3)
rails-dom-testing (~> 2.2)
rails-html-sanitizer (~> 1.6)
useragent (~> 0.16)
actionview (7.2.4)
activesupport (= 7.2.4)
builder (~> 3.1)
cgi
erubi (~> 1.11)
rails-dom-testing (~> 2.2)
rails-html-sanitizer (~> 1.6)
activesupport (7.2.4)
base64
benchmark (>= 0.3)
bigdecimal
concurrent-ruby (~> 1.0, >= 1.3.1)
connection_pool (>= 2.2.5, < 3)
drb
i18n (>= 1.6, < 2)
logger (>= 1.4.2)
minitest (>= 5.1, < 6)
securerandom (>= 0.3)
tzinfo (~> 2.0, >= 2.0.5)
addressable (2.9.2)
public_suffix (>= 2.0.2, < 8.0)
base64 (0.3.0)
benchmark (0.5.0)
bigdecimal (4.1.3)
builder (3.3.0)
cgi (0.5.2)
concurrent-ruby (1.3.8)
connection_pool (2.5.5)
crass (1.0.8)
date (3.6.0)
diff-lcs (1.5.1)
drb (2.2.3)
erb (4.0.4.1)
cgi (>= 0.3.3)
erubi (1.13.1)
i18n (1.15.2)
concurrent-ruby (~> 1.0)
io-console (0.9.5)
irb (1.19.0)
pp (>= 0.6.0)
prism (>= 1.3.0)
rdoc (>= 4.0.0)
reline (>= 0.4.2)
json-schema (6.2.1)
addressable (~> 2.8)
bigdecimal (>= 3.1, < 5)
logger (1.7.0)
loofah (2.25.2)
crass (~> 1.0.2)
nokogiri (>= 1.12.0)
mini_portile2 (2.8.9)
minitest (5.27.0)
nokogiri (1.18.10)
mini_portile2 (~> 2.8.2)
racc (~> 1.4)
nokogiri (1.18.10-arm64-darwin)
racc (~> 1.4)
nokogiri (1.18.10-x86_64-darwin)
racc (~> 1.4)
nokogiri (1.18.10-x86_64-linux-gnu)
racc (~> 1.4)
pp (0.6.4)
prettyprint
prettyprint (0.2.1)
prism (2.1.0)
psych (5.5.0)
date
stringio
public_suffix (6.0.2)
racc (1.8.1)
rack (3.2.8)
rack-session (2.1.2)
base64 (>= 0.1.0)
rack (>= 3.0.0)
rack-test (2.3.0)
rack (>= 1.3)
rackup (2.3.1)
rack (>= 3)
rails-dom-testing (2.3.0)
activesupport (>= 5.0.0)
minitest
nokogiri (>= 1.6)
rails-html-sanitizer (1.7.1)
loofah (~> 2.25, >= 2.25.2)
nokogiri (>= 1.15.7, != 1.16.7, != 1.16.6, != 1.16.5, != 1.16.4, != 1.16.3, != 1.16.2, != 1.16.1, != 1.16.0.rc1, != 1.16.0)
railties (7.2.4)
actionpack (= 7.2.4)
activesupport (= 7.2.4)
cgi
irb (~> 1.13)
rackup (>= 1.0.0)
rake (>= 12.2)
thor (~> 1.0, >= 1.2.2)
tsort (>= 0.2)
zeitwerk (~> 2.6)
rake (13.2.1)
rake-compiler (1.3.1)
rake
rake-compiler-dock (1.12.0)
rb_sys (0.9.130)
rake-compiler-dock (= 1.12.0)
rdoc (7.2.0)
erb
psych (>= 4.0.0)
tsort
reline (0.7.0)
io-console (~> 0.5)
rspec (3.13.0)
rspec-core (~> 3.13.0)
rspec-expectations (~> 3.13.0)
Expand All @@ -28,6 +140,19 @@ GEM
diff-lcs (>= 1.2.0, < 2.0)
rspec-support (~> 3.13.0)
rspec-support (3.13.2)
rswag-specs (2.17.0)
activesupport (>= 5.2, < 8.2)
json-schema (>= 2.2, < 7.0)
railties (>= 5.2, < 8.2)
rspec-core (>= 2.14)
securerandom (0.4.1)
stringio (3.2.0)
thor (1.5.0)
tsort (0.2.0)
tzinfo (2.0.6)
concurrent-ruby (~> 1.0)
useragent (0.16.11)
zeitwerk (2.6.18)

PLATFORMS
arm64-darwin
Expand All @@ -41,6 +166,7 @@ DEPENDENCIES
rake-compiler (= 1.3.1)
rspec
rspec_trunk_flaky_tests!
rswag-specs (= 2.17.0)

BUNDLED WITH
2.5.23
6 changes: 6 additions & 0 deletions rspec-trunk-flaky-tests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,12 @@ When set to a file path, the gem writes a JSON summary of the run to that file a
TRUNK_SUMMARY_OUTPUT_FILE=tmp/trunk-summary.json bundle exec rspec
```

#### `TRUNK_REPORT_SPEC_LOCATION`

When set to `true`, an example defined outside its spec file, such as a shared example or rswag's `run_test!`, is reported at the spec file RSpec reruns it from, and its code owners come from that file. The line is the `it_behaves_like` or `include_examples` call, or else the enclosing group in the spec. Other examples are unchanged.

The file is part of a test's id, so turning this on gives most of those examples new ids in Trunk, with no history, and quarantines set on their old ids no longer apply to them.

#### `TRUNK_LOCAL_UPLOAD_DIR` (Experimental)

> **⚠️ Experimental Feature**: This feature is experimental. Please reach out to support@trunk.io before attempting to use it.
Expand Down
33 changes: 31 additions & 2 deletions rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
# TRUNK_QUARANTINED_TESTS_DISK_CACHE_TTL_SECS - Time to cache quarantined tests on disk (in seconds)
# TRUNK_QUARANTINE_QUERY_FAILURE_EXIT - Set to 'true' to abort the RSpec run when quarantine
# lookup fails (remaining examples are skipped)
# TRUNK_REPORT_SPEC_LOCATION - Set to 'true' to report an example defined outside its spec file
# (a shared example, or one a library defines) at the spec file RSpec reruns it from
# DISABLE_RSPEC_TRUNK_FLAKY_TESTS - Set to 'true' to completely disable Trunk
#
require 'rspec/core'
Expand Down Expand Up @@ -145,6 +147,10 @@ def disable_quarantining?
ENV['TRUNK_DISABLE_QUARANTINING'] == 'true'
end

def report_spec_location?
ENV['TRUNK_REPORT_SPEC_LOCATION'] == 'true'
end

def command_line
"#{$PROGRAM_NAME} #{ARGV.join(' ')}"
end
Expand Down Expand Up @@ -173,10 +179,33 @@ def strip_ansi_codes(text)

# The escaped file path and the dotted classname derived from it.
def file_and_classname(example)
file = escape(example.metadata[:file_path])
file = escape(location(example).first)
[file, file.sub(%r{\.[^/.]+\Z}, '').gsub('/', '.').gsub(/\A\.+|\.+\Z/, '')]
end

# The file and line the example is reported at. With TRUNK_REPORT_SPEC_LOCATION, an example
# defined outside its spec file is reported at the line in the spec that brought it in.
def location(example)
metadata = example.metadata
file = metadata[:rerun_file_path]
return [metadata[:file_path], metadata[:line_number]] if !report_spec_location? || metadata[:file_path] == file

[file, inclusion_line(metadata, file) || group_line(example, file) || metadata[:line_number]]
end

def inclusion_line(metadata, file)
Array(metadata[:shared_group_inclusion_backtrace]).reverse_each do |frame|
path, line = frame.inclusion_location.to_s.sub(/:in .+\z/m, '').match(/\A(.+):(\d+)\z/)&.captures
return line.to_i if path && RSpec::Core::Metadata.relative_path(path) == file
end
nil
end

def group_line(example, file)
group = example.example_group.parent_groups.find { |parent| parent.metadata[:file_path] == file }
group&.metadata&.fetch(:line_number)
end

def parent_name(example)
name = description(example.example_group.metadata)
name.empty? ? 'rspec' : name
Expand Down Expand Up @@ -598,7 +627,7 @@ def add_test_case(example)
# TODO: should we use concatenated string or alias when auto-generated description?
name = Trunk.full_description(example)
file, classname = Trunk.file_and_classname(example)
line = example.metadata[:line_number]
line = Trunk.location(example).last
started_at = example.execution_result.started_at.to_f
finished_at = example.execution_result.finished_at.to_f

Expand Down
86 changes: 86 additions & 0 deletions rspec-trunk-flaky-tests/test/location_spec.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
# frozen_string_literal: true

require 'rspec_trunk_flaky_tests'
require_relative '../spec/spec_helper'
require_relative 'support/location_helpers'
require 'rails/version'
require 'rswag/specs'
require 'rspec/core/sandbox'

# trunk-ignore(rubocop/Metrics/BlockLength)
RSpec.describe 'reported test location' do
spec_file = './test/location_spec.rb'
helpers_file = './test/support/location_helpers.rb'
rswag_file = Rswag::Specs::ExampleGroupHelpers.instance_method(:run_test!).source_location.first
lines = {}

# trunk-ignore(rubocop/Metrics/AbcSize,rubocop/Metrics/MethodLength)
define_method(:locations) do
examples = nil
RSpec::Core::Sandbox.sandboxed do |_config|
group = RSpec.describe('API') do
extend Rswag::Specs::ExampleGroupHelpers
shared_examples('shared', &LocationHelpers::SHARED_EXAMPLES)
lines[:own] = __LINE__ + 1
it('is its own') { expect(1).to eq(1) }
lines[:it_behaves_like] = __LINE__ + 1
it_behaves_like 'shared'
context 'included' do
lines[:include_examples] = __LINE__ + 1
include_examples 'shared'
end
path('/users') do
get('lists users') do
lines[:response] = __LINE__ + 1
response('200', 'ok') { run_test! }
end
end
end
group.run(RSpec::Core::NullReporter)
examples = group.descendants.flat_map(&:examples)
end
examples.to_h { |example| [example.full_description, RSpec::Trunk.location(example)] }
end

around do |example|
previous = ENV.fetch('TRUNK_REPORT_SPEC_LOCATION', nil)
ENV['TRUNK_REPORT_SPEC_LOCATION'] = report_spec_location
example.run
ensure
ENV['TRUNK_REPORT_SPEC_LOCATION'] = previous
end

context 'with TRUNK_REPORT_SPEC_LOCATION' do
let(:report_spec_location) { 'true' }

it 'reports each example at the spec line that brought it in' do
expect(locations).to eq(
'API is its own' => [spec_file, lines[:own]],
'API behaves like shared is shared' => [spec_file, lines[:it_behaves_like]],
'API included is shared' => [spec_file, lines[:include_examples]],
'API /users get ok returns a 200 response' => [spec_file, lines[:response]]
)
end

it 'derives the classname from the spec file' do
example = nil
RSpec::Core::Sandbox.sandboxed do |_config|
group = RSpec.describe('API') do
extend Rswag::Specs::ExampleGroupHelpers
path('/users') { get('lists users') { response('200', 'ok') { run_test! } } }
end
example = group.descendants.flat_map(&:examples).first
end
expect(RSpec::Trunk.file_and_classname(example)).to eq([spec_file, 'test.location_spec'])
end
end

context 'without it' do
let(:report_spec_location) { nil }

it 'reports each example where it is defined' do
expect(locations.values.map(&:first)).to eq([spec_file, helpers_file, helpers_file, rswag_file])
expect(locations['API is its own']).to eq([spec_file, lines[:own]])
end
end
end
6 changes: 6 additions & 0 deletions rspec-trunk-flaky-tests/test/support/location_helpers.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
# frozen_string_literal: true

# Shared examples defined in a file other than the spec that includes them.
module LocationHelpers
SHARED_EXAMPLES = proc { it('is shared') { expect(1).to eq(1) } }
end
Loading