Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .github/workflows/pull_request.yml
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,14 @@ jobs:
with:
lfs: "true"

# pyo3 links whatever `python3` the runner has, and the macOS image's 3.15 ships no
# libpython to link against; pin the version setup_build_pyo3 uses.
- name: Set up Python (macOS)
if: ${{ contains(matrix.platform.os-name, 'darwin') }}
uses: actions/setup-python@v5
with:
python-version: 3.13

- name: Setup Rust & Cargo
uses: ./.github/actions/setup_rust_cargo

Expand Down
2 changes: 1 addition & 1 deletion rspec-trunk-flaky-tests/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ require 'trunk_spec_helper'

### Environment Variables

For a complete list of environment variables that the gem accepts, see [`lib/trunk_spec_helper.rb`](lib/trunk_spec_helper.rb). The gem uses the same environment variables as the Trunk Analytics CLI for configuration overrides. `TRUNK_ORG_URL_SLUG` and `TRUNK_API_TOKEN` must be set to activate the plugin, and `TRUNK_TEST_COLLECTION_ID` too: with the first two set and no test collection, the plugin prints a warning and stays off. `TRUNK_API_TOKEN` must be a v2 API key with the Upload test results permission, or a Trunk JWT that outlasts the run (one lasts at most 15 minutes); with an organization token, the plugin prints a warning and stays off. See [Credentials](../README.md#credentials).
For a complete list of environment variables that the gem accepts, see [`lib/trunk_spec_helper.rb`](lib/trunk_spec_helper.rb). The gem uses the same environment variables as the Trunk Analytics CLI for configuration overrides. `TRUNK_ORG_URL_SLUG` and a non-empty `TRUNK_API_TOKEN` activate the plugin; without either it stays off. Once active it needs `TRUNK_TEST_COLLECTION_ID` too, and `TRUNK_API_TOKEN` must be a v2 API key with the Upload test results permission, or a Trunk JWT that outlasts the run (one lasts at most 15 minutes). With no test collection, or with an organization token, loading the plugin raises `RSpec::Trunk::ConfigurationError`, so RSpec runs no examples and exits non-zero; set `DISABLE_RSPEC_TRUNK_FLAKY_TESTS=true` to run without Trunk. See [Credentials](../README.md#credentials).

#### `TRUNK_QUARANTINE_QUERY_FAILURE_EXIT`

Expand Down
36 changes: 19 additions & 17 deletions rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ module Trunk
MODULE_TO_S = Module.instance_method(:to_s)
MEMORY_ADDRESS = /:0x\h+>/

# Raised when Trunk is configured to upload but can't: see .check_configuration!.
class ConfigurationError < StandardError; end

# Knapsack example detector instantiates all test cases in order to determine how to shard them
# These instantiations should not generate test bundles, so we
# disable the gem when running under knapsack_pro:rspec_test_example_detector
Expand All @@ -82,6 +85,7 @@ class << self
def setup
return if current_run || disabled?

check_configuration!
run = @current_run = Run.new
RSpec::Core::Example.prepend(ExampleExtension)
RSpec::Core::ExampleGroup.singleton_class.prepend(ExampleGroupExtension)
Expand All @@ -105,30 +109,28 @@ def install(config, run)
config.reporter.register_listener AnalyticsListener.new(run), :example_finished, :close
end

# An empty token counts as none: it is what a fork pull request's missing secret turns into.
def disabled?
knapsack_detector_mode? || ENV['DISABLE_RSPEC_TRUNK_FLAKY_TESTS'] == 'true' ||
ENV['TRUNK_ORG_URL_SLUG'].nil? || ENV['TRUNK_API_TOKEN'].nil? || missing_test_collection? ||
refused_token?
ENV['TRUNK_ORG_URL_SLUG'].nil? || ENV.fetch('TRUNK_API_TOKEN', '').strip.empty?
end

# 1.0.0 takes a v2 API key or a Trunk JWT, never the organization token. Warns for the same
# reason as missing_test_collection?.
def refused_token?
# A run with a token and a slug means to upload, so one that can't is an error rather than a
# warning: a warning leaves the build green while uploads and quarantining stop, which is how
# a dependency bot's upgrade to 1.0.0 would merge unnoticed. Raised while spec_helper loads,
# so RSpec runs no examples and exits non-zero, as the CLI's `test` command does.
def check_configuration!
if ENV.fetch('TRUNK_TEST_COLLECTION_ID', '').strip.empty?
configuration_error!('this run names no test collection. Set TRUNK_TEST_COLLECTION_ID to upload its ' \
"test results; the ID is on the collection's page in Trunk.")
end
refusal = RSpec::Trunk.token_refusal(ENV.fetch('TRUNK_API_TOKEN'))
return false unless refusal

warn("Trunk Flaky Tests is disabled: #{refusal}")
true
configuration_error!(refusal) if refusal
end

# Warns rather than staying silent like the checks before it: a run with a token and a slug
# means to upload, and the native extension's own warnings never reach the console.
def missing_test_collection?
return false unless ENV.fetch('TRUNK_TEST_COLLECTION_ID', '').strip.empty?

warn('Trunk Flaky Tests is disabled: this run names no test collection. Set ' \
"TRUNK_TEST_COLLECTION_ID to upload its test results; the ID is on the collection's page in Trunk.")
true
def configuration_error!(reason)
raise ConfigurationError, "Trunk Flaky Tests: #{reason} To run without Trunk, set " \
'DISABLE_RSPEC_TRUNK_FLAKY_TESTS=true.'
end

def knapsack_detector_mode?
Expand Down
47 changes: 38 additions & 9 deletions rspec-trunk-flaky-tests/test/test_collection_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

require 'open3'
require 'rbconfig'
require 'tmpdir'
require_relative '../spec/spec_helper'

# trunk-ignore(rubocop/Metrics/BlockLength)
Expand All @@ -16,32 +17,60 @@

let(:lib_dir) { File.dirname($LOAD_PATH.resolve_feature_path('trunk_spec_helper').last) }

def load_gem(test_collection_id)
env = {
def env(test_collection_id)
{
'TRUNK_ORG_URL_SLUG' => 'test-collection-spec',
'TRUNK_API_TOKEN' => 'trunk_v2_test-collection-spec',
'TRUNK_TEST_COLLECTION_ID' => test_collection_id,
'DISABLE_RSPEC_TRUNK_FLAKY_TESTS' => nil
}
Open3.capture3(env, RbConfig.ruby, '-I', lib_dir, '-e', probe)
end

def load_gem(test_collection_id, extra_env = {})
Open3.capture3(env(test_collection_id).merge(extra_env), RbConfig.ruby, '-I', lib_dir, '-e', probe)
end

[nil, '', ' '].each do |test_collection_id|
it "disables Trunk and says why when it is #{test_collection_id.inspect}" do
it "refuses to load, and says why, when it is #{test_collection_id.inspect}" do
stdout, stderr, status = load_gem(test_collection_id)

expect(status).to be_success, stderr
expect(stdout.lines.last.strip).to eq('disabled')
expect(stderr).to include('Trunk Flaky Tests is disabled: this run names no test collection. ' \
'Set TRUNK_TEST_COLLECTION_ID')
expect(status).not_to be_success
expect(stdout).to be_empty
expect(stderr).to include('RSpec::Trunk::ConfigurationError')
expect(stderr).to include('Trunk Flaky Tests: this run names no test collection. Set TRUNK_TEST_COLLECTION_ID')
expect(stderr).to include('To run without Trunk, set DISABLE_RSPEC_TRUNK_FLAKY_TESTS=true.')
end
end

it 'fails an rspec run before any example runs' do
Dir.mktmpdir do |dir|
spec_file = File.join(dir, 'probe_spec.rb')
File.write(spec_file, <<~'RUBY')
require 'trunk_spec_helper'
RSpec.describe('probe') { it('runs') { puts 'EXAMPLE RAN' } }
RUBY
rspec = Gem.bin_path('rspec-core', 'rspec')
stdout, stderr, status = Open3.capture3(env(''), RbConfig.ruby, '-I', lib_dir, rspec, spec_file)

expect(status).not_to be_success
expect(stdout).not_to include('EXAMPLE RAN')
expect(stdout + stderr).to include('this run names no test collection')
end
end

it 'stays off quietly when Trunk is turned off' do
stdout, stderr, status = load_gem('', 'DISABLE_RSPEC_TRUNK_FLAKY_TESTS' => 'true')

expect(status).to be_success, stderr
expect(stdout.lines.last.strip).to eq('disabled')
expect(stderr).not_to include('Trunk Flaky Tests')
end

it 'enables Trunk when it is set' do
stdout, stderr, status = load_gem('aB3xY9kQ')

expect(status).to be_success, stderr
expect(stdout.lines.last.strip).to eq('enabled')
expect(stderr).not_to include('Trunk Flaky Tests is disabled')
expect(stderr).not_to include('Trunk Flaky Tests')
end
end
25 changes: 19 additions & 6 deletions rspec-trunk-flaky-tests/test/token_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,29 @@ def load_gem(token)
Open3.capture3(env, RbConfig.ruby, '-I', lib_dir, '-e', probe)
end

['0123456789abcdef0123456789abcdef', 'a.b', ''].each do |token|
it "disables Trunk and says why for an organization token like #{token.inspect}" do
['0123456789abcdef0123456789abcdef', 'a.b'].each do |token|
it "refuses to load, and says why, for an organization token like #{token.inspect}" do
stdout, stderr, status = load_gem(token)

expect(status).to be_success, stderr
expect(stdout.lines.last.strip).to eq('disabled')
expect(stderr).to include('Trunk Flaky Tests is disabled: This Trunk token is not a v2 API key.')
expect(status).not_to be_success
expect(stdout).to be_empty
expect(stderr).to include('RSpec::Trunk::ConfigurationError')
expect(stderr).to include('Trunk Flaky Tests: This Trunk token is not a v2 API key.')
expect(stderr).to include('https://app.trunk.io/settings/developer/api-tokens')
expect(stderr).to include('Upload test results')
expect(stderr).to include('until December 1, 2027, pin analytics-cli 0.17.x')
expect(stderr).to include('To run without Trunk, set DISABLE_RSPEC_TRUNK_FLAKY_TESTS=true.')
end
end

# A fork pull request's missing secret arrives as an empty string.
[nil, '', ' '].each do |token|
it "stays off quietly with no token, like #{token.inspect}" do
stdout, stderr, status = load_gem(token)

expect(status).to be_success, stderr
expect(stdout.lines.last.strip).to eq('disabled')
expect(stderr).not_to include('Trunk Flaky Tests')
end
end

Expand All @@ -45,7 +58,7 @@ def load_gem(token)

expect(status).to be_success, stderr
expect(stdout.lines.last.strip).to eq('enabled')
expect(stderr).not_to include('Trunk Flaky Tests is disabled')
expect(stderr).not_to include('Trunk Flaky Tests')
end
end
end
Loading