diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 5e34152d..d33966d8 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -192,6 +192,14 @@ jobs: with: lfs: "true" + # pyo3 links whatever `python3` the runner has, and the macOS image's 3.15 ships no + # libpython to link against; pin the version setup_build_pyo3 uses. + - name: Set up Python (macOS) + if: ${{ contains(matrix.platform.os-name, 'darwin') }} + uses: actions/setup-python@v5 + with: + python-version: 3.13 + - name: Setup Rust & Cargo uses: ./.github/actions/setup_rust_cargo diff --git a/rspec-trunk-flaky-tests/README.md b/rspec-trunk-flaky-tests/README.md index 302bfb86..54d8621a 100644 --- a/rspec-trunk-flaky-tests/README.md +++ b/rspec-trunk-flaky-tests/README.md @@ -106,7 +106,7 @@ require 'trunk_spec_helper' ### Environment Variables -For a complete list of environment variables that the gem accepts, see [`lib/trunk_spec_helper.rb`](lib/trunk_spec_helper.rb). The gem uses the same environment variables as the Trunk Analytics CLI for configuration overrides. `TRUNK_ORG_URL_SLUG` and `TRUNK_API_TOKEN` must be set to activate the plugin, and `TRUNK_TEST_COLLECTION_ID` too: with the first two set and no test collection, the plugin prints a warning and stays off. `TRUNK_API_TOKEN` must be a v2 API key with the Upload test results permission, or a Trunk JWT that outlasts the run (one lasts at most 15 minutes); with an organization token, the plugin prints a warning and stays off. See [Credentials](../README.md#credentials). +For a complete list of environment variables that the gem accepts, see [`lib/trunk_spec_helper.rb`](lib/trunk_spec_helper.rb). The gem uses the same environment variables as the Trunk Analytics CLI for configuration overrides. `TRUNK_ORG_URL_SLUG` and a non-empty `TRUNK_API_TOKEN` activate the plugin; without either it stays off. Once active it needs `TRUNK_TEST_COLLECTION_ID` too, and `TRUNK_API_TOKEN` must be a v2 API key with the Upload test results permission, or a Trunk JWT that outlasts the run (one lasts at most 15 minutes). With no test collection, or with an organization token, loading the plugin raises `RSpec::Trunk::ConfigurationError`, so RSpec runs no examples and exits non-zero; set `DISABLE_RSPEC_TRUNK_FLAKY_TESTS=true` to run without Trunk. See [Credentials](../README.md#credentials). #### `TRUNK_QUARANTINE_QUERY_FAILURE_EXIT` diff --git a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb index 297102ad..8dd52818 100644 --- a/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb +++ b/rspec-trunk-flaky-tests/lib/trunk_spec_helper.rb @@ -56,6 +56,9 @@ module Trunk MODULE_TO_S = Module.instance_method(:to_s) MEMORY_ADDRESS = /:0x\h+>/ + # Raised when Trunk is configured to upload but can't: see .check_configuration!. + class ConfigurationError < StandardError; end + # Knapsack example detector instantiates all test cases in order to determine how to shard them # These instantiations should not generate test bundles, so we # disable the gem when running under knapsack_pro:rspec_test_example_detector @@ -82,6 +85,7 @@ class << self def setup return if current_run || disabled? + check_configuration! run = @current_run = Run.new RSpec::Core::Example.prepend(ExampleExtension) RSpec::Core::ExampleGroup.singleton_class.prepend(ExampleGroupExtension) @@ -105,30 +109,28 @@ def install(config, run) config.reporter.register_listener AnalyticsListener.new(run), :example_finished, :close end + # An empty token counts as none: it is what a fork pull request's missing secret turns into. def disabled? knapsack_detector_mode? || ENV['DISABLE_RSPEC_TRUNK_FLAKY_TESTS'] == 'true' || - ENV['TRUNK_ORG_URL_SLUG'].nil? || ENV['TRUNK_API_TOKEN'].nil? || missing_test_collection? || - refused_token? + ENV['TRUNK_ORG_URL_SLUG'].nil? || ENV.fetch('TRUNK_API_TOKEN', '').strip.empty? end - # 1.0.0 takes a v2 API key or a Trunk JWT, never the organization token. Warns for the same - # reason as missing_test_collection?. - def refused_token? + # A run with a token and a slug means to upload, so one that can't is an error rather than a + # warning: a warning leaves the build green while uploads and quarantining stop, which is how + # a dependency bot's upgrade to 1.0.0 would merge unnoticed. Raised while spec_helper loads, + # so RSpec runs no examples and exits non-zero, as the CLI's `test` command does. + def check_configuration! + if ENV.fetch('TRUNK_TEST_COLLECTION_ID', '').strip.empty? + configuration_error!('this run names no test collection. Set TRUNK_TEST_COLLECTION_ID to upload its ' \ + "test results; the ID is on the collection's page in Trunk.") + end refusal = RSpec::Trunk.token_refusal(ENV.fetch('TRUNK_API_TOKEN')) - return false unless refusal - - warn("Trunk Flaky Tests is disabled: #{refusal}") - true + configuration_error!(refusal) if refusal end - # Warns rather than staying silent like the checks before it: a run with a token and a slug - # means to upload, and the native extension's own warnings never reach the console. - def missing_test_collection? - return false unless ENV.fetch('TRUNK_TEST_COLLECTION_ID', '').strip.empty? - - warn('Trunk Flaky Tests is disabled: this run names no test collection. Set ' \ - "TRUNK_TEST_COLLECTION_ID to upload its test results; the ID is on the collection's page in Trunk.") - true + def configuration_error!(reason) + raise ConfigurationError, "Trunk Flaky Tests: #{reason} To run without Trunk, set " \ + 'DISABLE_RSPEC_TRUNK_FLAKY_TESTS=true.' end def knapsack_detector_mode? diff --git a/rspec-trunk-flaky-tests/test/test_collection_spec.rb b/rspec-trunk-flaky-tests/test/test_collection_spec.rb index 13693c41..1d40b8ea 100644 --- a/rspec-trunk-flaky-tests/test/test_collection_spec.rb +++ b/rspec-trunk-flaky-tests/test/test_collection_spec.rb @@ -2,6 +2,7 @@ require 'open3' require 'rbconfig' +require 'tmpdir' require_relative '../spec/spec_helper' # trunk-ignore(rubocop/Metrics/BlockLength) @@ -16,32 +17,60 @@ let(:lib_dir) { File.dirname($LOAD_PATH.resolve_feature_path('trunk_spec_helper').last) } - def load_gem(test_collection_id) - env = { + def env(test_collection_id) + { 'TRUNK_ORG_URL_SLUG' => 'test-collection-spec', 'TRUNK_API_TOKEN' => 'trunk_v2_test-collection-spec', 'TRUNK_TEST_COLLECTION_ID' => test_collection_id, 'DISABLE_RSPEC_TRUNK_FLAKY_TESTS' => nil } - Open3.capture3(env, RbConfig.ruby, '-I', lib_dir, '-e', probe) + end + + def load_gem(test_collection_id, extra_env = {}) + Open3.capture3(env(test_collection_id).merge(extra_env), RbConfig.ruby, '-I', lib_dir, '-e', probe) end [nil, '', ' '].each do |test_collection_id| - it "disables Trunk and says why when it is #{test_collection_id.inspect}" do + it "refuses to load, and says why, when it is #{test_collection_id.inspect}" do stdout, stderr, status = load_gem(test_collection_id) - expect(status).to be_success, stderr - expect(stdout.lines.last.strip).to eq('disabled') - expect(stderr).to include('Trunk Flaky Tests is disabled: this run names no test collection. ' \ - 'Set TRUNK_TEST_COLLECTION_ID') + expect(status).not_to be_success + expect(stdout).to be_empty + expect(stderr).to include('RSpec::Trunk::ConfigurationError') + expect(stderr).to include('Trunk Flaky Tests: this run names no test collection. Set TRUNK_TEST_COLLECTION_ID') + expect(stderr).to include('To run without Trunk, set DISABLE_RSPEC_TRUNK_FLAKY_TESTS=true.') + end + end + + it 'fails an rspec run before any example runs' do + Dir.mktmpdir do |dir| + spec_file = File.join(dir, 'probe_spec.rb') + File.write(spec_file, <<~'RUBY') + require 'trunk_spec_helper' + RSpec.describe('probe') { it('runs') { puts 'EXAMPLE RAN' } } + RUBY + rspec = Gem.bin_path('rspec-core', 'rspec') + stdout, stderr, status = Open3.capture3(env(''), RbConfig.ruby, '-I', lib_dir, rspec, spec_file) + + expect(status).not_to be_success + expect(stdout).not_to include('EXAMPLE RAN') + expect(stdout + stderr).to include('this run names no test collection') end end + it 'stays off quietly when Trunk is turned off' do + stdout, stderr, status = load_gem('', 'DISABLE_RSPEC_TRUNK_FLAKY_TESTS' => 'true') + + expect(status).to be_success, stderr + expect(stdout.lines.last.strip).to eq('disabled') + expect(stderr).not_to include('Trunk Flaky Tests') + end + it 'enables Trunk when it is set' do stdout, stderr, status = load_gem('aB3xY9kQ') expect(status).to be_success, stderr expect(stdout.lines.last.strip).to eq('enabled') - expect(stderr).not_to include('Trunk Flaky Tests is disabled') + expect(stderr).not_to include('Trunk Flaky Tests') end end diff --git a/rspec-trunk-flaky-tests/test/token_spec.rb b/rspec-trunk-flaky-tests/test/token_spec.rb index 283445f4..17ac3de1 100644 --- a/rspec-trunk-flaky-tests/test/token_spec.rb +++ b/rspec-trunk-flaky-tests/test/token_spec.rb @@ -26,16 +26,29 @@ def load_gem(token) Open3.capture3(env, RbConfig.ruby, '-I', lib_dir, '-e', probe) end - ['0123456789abcdef0123456789abcdef', 'a.b', ''].each do |token| - it "disables Trunk and says why for an organization token like #{token.inspect}" do + ['0123456789abcdef0123456789abcdef', 'a.b'].each do |token| + it "refuses to load, and says why, for an organization token like #{token.inspect}" do stdout, stderr, status = load_gem(token) - expect(status).to be_success, stderr - expect(stdout.lines.last.strip).to eq('disabled') - expect(stderr).to include('Trunk Flaky Tests is disabled: This Trunk token is not a v2 API key.') + expect(status).not_to be_success + expect(stdout).to be_empty + expect(stderr).to include('RSpec::Trunk::ConfigurationError') + expect(stderr).to include('Trunk Flaky Tests: This Trunk token is not a v2 API key.') expect(stderr).to include('https://app.trunk.io/settings/developer/api-tokens') expect(stderr).to include('Upload test results') expect(stderr).to include('until December 1, 2027, pin analytics-cli 0.17.x') + expect(stderr).to include('To run without Trunk, set DISABLE_RSPEC_TRUNK_FLAKY_TESTS=true.') + end + end + + # A fork pull request's missing secret arrives as an empty string. + [nil, '', ' '].each do |token| + it "stays off quietly with no token, like #{token.inspect}" do + stdout, stderr, status = load_gem(token) + + expect(status).to be_success, stderr + expect(stdout.lines.last.strip).to eq('disabled') + expect(stderr).not_to include('Trunk Flaky Tests') end end @@ -45,7 +58,7 @@ def load_gem(token) expect(status).to be_success, stderr expect(stdout.lines.last.strip).to eq('enabled') - expect(stderr).not_to include('Trunk Flaky Tests is disabled') + expect(stderr).not_to include('Trunk Flaky Tests') end end end