From 910136c2a3ef6be8fdc976cb0c6eadc3e4862d78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 07:46:01 +0200 Subject: [PATCH 01/19] test: Add the evmone test command One command runs both fixture formats, deciding which each test case is when it runs it: "_info.fixture-format" names it, and a case without one is recognised by the shape only that format has. A format the tool does not run is skipped, as is a file holding no test at all, which is what EEST's shared pre-allocation is. Anything else it cannot recognise is a fault in the fixture. The two test tools are untouched and keep working. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/integration/CMakeLists.txt | 141 ++++++++++++++---- .../blockchaintest/eip7778_block_gas.json | 5 +- .../testcmd/one_unsupported_case.json | 116 ++++++++++++++ .../integration/testcmd_broken/truncated.json | 3 + .../testcmd_skipped/all_unsupported.json | 9 ++ test/integration/testcmd_skipped/empty.json | 1 + .../testcmd_skipped/partial_state_test.json | 20 +++ .../testcmd_skipped/pre_alloc.json | 27 ++++ test/utils/test_collector.cpp | 54 +++++++ test/utils/test_collector.hpp | 6 + test/utils/test_driver.cpp | 119 +++++++++++++++ test/utils/test_driver.hpp | 39 +++++ tools/evmone/main.cpp | 58 +++++++ 13 files changed, 565 insertions(+), 33 deletions(-) create mode 100644 test/integration/testcmd/one_unsupported_case.json create mode 100644 test/integration/testcmd_broken/truncated.json create mode 100644 test/integration/testcmd_skipped/all_unsupported.json create mode 100644 test/integration/testcmd_skipped/empty.json create mode 100644 test/integration/testcmd_skipped/partial_state_test.json create mode 100644 test/integration/testcmd_skipped/pre_alloc.json diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index 4abbe17ece..87aa3c3210 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -82,39 +82,116 @@ DUP1,4 {\"pc\":6,\"op\":3,\"gas\":\"0xf4234\",\"gasCost\":\"0x3\",\"memSize\":0,\"stack\":\\[\"0x0\",\"0x4\"\\],\"depth\":1,\"refund\":0,\"opName\":\"SUB\"} ") -endif() + # One command runs both fixture formats, deciding per test case which one it is. + add_test(NAME ${PREFIX}/test_both_formats COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/statetest/tests1/SuiteA/test1.json + ${CMAKE_CURRENT_SOURCE_DIR}/blockchaintest/eip7778_block_gas.json) + set_tests_properties( + ${PREFIX}/test_both_formats PROPERTIES PASS_REGULAR_EXPRESSION "= 2 passed in") + + # JSON which is not a test at all, as fixture directories hold beside their fixtures, is + # skipped rather than guessed at. + add_test(NAME ${PREFIX}/test_not_a_fixture COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/statetest/tests1/SuiteA/index.json) + set_tests_properties( + ${PREFIX}/test_not_a_fixture PROPERTIES PASS_REGULAR_EXPRESSION + "index\\.json::[^\n]* - not a test.*0 passed, 2 skipped in") + + # Over a directory a test is a whole file, so a case the loader refuses must not take the + # rest of the file with it: the file passes on the case beside it, the refused one first. + add_test(NAME ${PREFIX}/test_directory_with_unsupported_case COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd) + set_tests_properties( + ${PREFIX}/test_directory_with_unsupported_case PROPERTIES + PASS_REGULAR_EXPRESSION "= 1 passed in") + + # A PASS_REGULAR_EXPRESSION makes CTest ignore the exit code, so these two assert on nothing + # else: a sound fixture directory succeeds, and a fault in one fails. + add_test(NAME ${PREFIX}/test_exit_code_success COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd) + + add_test(NAME ${PREFIX}/test_exit_code_failure COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault) + set_tests_properties(${PREFIX}/test_exit_code_failure PROPERTIES WILL_FAIL TRUE) + + # WILL_FAIL above accepts any non-zero exit, and a fault downgraded to a skip exits non-zero + # too, so the counts are what distinguish the two. + add_test(NAME ${PREFIX}/test_fault_is_not_a_skip COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault) + set_tests_properties( + ${PREFIX}/test_fault_is_not_a_skip PROPERTIES PASS_REGULAR_EXPRESSION "1 failed, 0 passed") -# A file holding a case which is not a fixture at all, beside one which runs. -add_test(NAME ${PREFIX}/fixture_fault COMMAND evmone-statetest - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault) -set_tests_properties(${PREFIX}/fixture_fault PROPERTIES WILL_FAIL TRUE) - -# WILL_FAIL above accepts any non-zero exit, and a fault downgraded to a skip exits non-zero too, -# so the counts are what distinguish the two. -add_test(NAME ${PREFIX}/fixture_fault_is_not_a_skip COMMAND evmone-statetest - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault) -set_tests_properties( - ${PREFIX}/fixture_fault_is_not_a_skip PROPERTIES PASS_REGULAR_EXPRESSION "1 failed, 0 passed") - -# Selecting only the case which is not a fixture must still fault. -add_test(NAME ${PREFIX}/fixture_fault_survives_filter COMMAND evmone-statetest - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault -k b_not_a_fixture) -set_tests_properties( - ${PREFIX}/fixture_fault_survives_filter PROPERTIES - PASS_REGULAR_EXPRESSION "1 failed, 0 passed") - -# A case whose load throws takes the rest of the file with it: the whole file is loaded before -# any of it runs, so the case after it is never reached and the failure is named after the file -# rather than the case it came from. FAILED pins that as a failure: a PASS_REGULAR_EXPRESSION -# makes CTest ignore the exit code, and a fault downgraded to a skip names the file too. -add_test(NAME ${PREFIX}/case_after_exception COMMAND evmone-statetest - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_cases) -set_tests_properties( - ${PREFIX}/case_after_exception PROPERTIES - PASS_REGULAR_EXPRESSION - "collected 1 test.*FAILED[^\n]*case_after_exception\\.json[^\n]*exception" - FAIL_REGULAR_EXPRESSION "b_wrong_state_root" -) + # Whether a file is a fixture file is a property of the file, not of what -k selected from + # it: selecting only the case which is not a fixture must still fault. + add_test(NAME ${PREFIX}/test_fault_survives_filter COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault -k b_not_a_fixture) + set_tests_properties( + ${PREFIX}/test_fault_survives_filter PROPERTIES + PASS_REGULAR_EXPRESSION "1 failed, 0 passed") + + # A case whose load throws must not abandon the cases after it, so both faults are reported. + add_test(NAME ${PREFIX}/test_case_after_exception COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_cases) + set_tests_properties( + ${PREFIX}/test_case_after_exception PROPERTIES + # "collected 1 test" keeps this on the directory form, where the whole file is one test + # and the per-case catch is what keeps the second fault reachable. + PASS_REGULAR_EXPRESSION + "collected 1 test.*a_load_error:\n exception.*b_wrong_state_root:.*state root") + + # Collected from a directory a file is one test, so a file with nothing to run is skipped + # whole, named by the first reason which explains it. All three reasons live here. + add_test(NAME ${PREFIX}/test_directory_skips_whole_files COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_skipped) + set_tests_properties( + ${PREFIX}/test_directory_skips_whole_files PROPERTIES + PASS_REGULAR_EXPRESSION + "all_unsupported\\.json - unsupported fixture format[^\n]*\nSKIPPED[^\n]*empty\\.json - no test cases.*pre_alloc\\.json - not a test.*0 passed, 4 skipped in") + + # Nothing to run is not nothing to report: the run fails rather than passing empty. + add_test(NAME ${PREFIX}/test_directory_skips_whole_files_exit_code COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_skipped) + set_tests_properties( + ${PREFIX}/test_directory_skips_whole_files_exit_code PROPERTIES WILL_FAIL TRUE) + + # Naming the file instead of the directory makes each case its own test, and a fault in one + # is still a fault: the case beside it runs. + add_test(NAME ${PREFIX}/test_file_fault COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault/unrecognised_case.json) + set_tests_properties( + ${PREFIX}/test_file_fault PROPERTIES PASS_REGULAR_EXPRESSION + "FAILED[^\n]*unrecognised_case\\.json::b_not_a_fixture.*1 failed, 1 passed in") + + # -k selects among the cases of a named file too. + add_test(NAME ${PREFIX}/test_file_filter COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault/unrecognised_case.json -k a_runs) + set_tests_properties( + ${PREFIX}/test_file_filter PROPERTIES PASS_REGULAR_EXPRESSION "= 1 passed in") + + # Naming a file whose cases this tool does not run skips them one by one, where collecting + # the same file from a directory would skip it whole. + add_test(NAME ${PREFIX}/test_file_declined_cases COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd/one_unsupported_case.json) + set_tests_properties( + ${PREFIX}/test_file_declined_cases PROPERTIES PASS_REGULAR_EXPRESSION + "SKIPPED[^\n]*c_declared_format_wins - unsupported fixture format.*= 1 passed, 2 skipped in") + + # --trace reaches the runner through the app-level flag, and turns the progress row into a + # line naming each test so the trace which follows it can be told apart. + add_test(NAME ${PREFIX}/test_trace COMMAND evmone-cli test --trace + ${CMAKE_CURRENT_SOURCE_DIR}/statetest/tests1/SuiteA/test1.json) + set_tests_properties( + ${PREFIX}/test_trace PROPERTIES PASS_REGULAR_EXPRESSION + "test1\\.json::test1\n\\{\"pc\":0,[^\n]*\"opName\":\"PUSH1\"\\}.*= 1 passed in") + + # A named file which does not parse names itself, because no case in it can be. + add_test(NAME ${PREFIX}/test_file_does_not_parse COMMAND evmone-cli test + ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_broken/truncated.json) + set_tests_properties( + ${PREFIX}/test_file_does_not_parse PROPERTIES PASS_REGULAR_EXPRESSION + "FAILED[^\n]*truncated\\.json - exception[^\n]*parse error.*1 failed, 0 passed in") + +endif() add_subdirectory(blockchaintest) add_subdirectory(export) diff --git a/test/integration/blockchaintest/eip7778_block_gas.json b/test/integration/blockchaintest/eip7778_block_gas.json index 326c57ef0b..f7bf7042c1 100644 --- a/test/integration/blockchaintest/eip7778_block_gas.json +++ b/test/integration/blockchaintest/eip7778_block_gas.json @@ -130,6 +130,9 @@ }, "genesisRLP": "0x", "sealEngine": "NoProof", - "_info": {} + "_info": { + "fixture-format": "blockchain_test", + "comment": "The format every EEST blockchain fixture declares." + } } } diff --git a/test/integration/testcmd/one_unsupported_case.json b/test/integration/testcmd/one_unsupported_case.json new file mode 100644 index 0000000000..20b93aafb7 --- /dev/null +++ b/test/integration/testcmd/one_unsupported_case.json @@ -0,0 +1,116 @@ +{ + "a_unsupported_unsupported_rlp": { + "_info": { + "comment": "Declines to load: expectException without rlp_decoded. Nothing of this case runs, so the case beside it is what the file reports." + }, + "network": "Cancun", + "genesisBlockHeader": { + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "coinbase": "0x0000000000000000000000000000000000000000", + "stateRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "number": "0x00", + "gasLimit": "0x01000000", + "gasUsed": "0x00", + "timestamp": "0x00", + "extraData": "0x00", + "baseFeePerGas": "0x10", + "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "blobGasUsed": "0x00", + "excessBlobGas": "0x00", + "hash": "0x0000000000000000000000000000000000000000000000000000000000000001" + }, + "pre": { + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "code": "0x", + "nonce": "0x00", + "balance": "0x02540be400" + } + }, + "blocks": [ + { + "expectException": "TransactionException.INVALID_SIGNATURE_VRS", + "rlp": "0x" + } + ], + "lastblockhash": "0x0000000000000000000000000000000000000000000000000000000000000001", + "postState": { + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "code": "0x", + "nonce": "0x00", + "balance": "0x02540be400" + } + } + }, + "b_runs_state_test": { + "_info": { + "comment": "A state test beside blockchain ones, which is the whole point of deciding the format per case." + }, + "env": { + "currentBaseFee": "0x0a", + "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentDifficulty": "0x020000", + "currentGasLimit": "0xff112233445566", + "currentNumber": "0x01", + "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000", + "currentTimestamp": "0x03e8" + }, + "post": { + "London": [ + { + "hash": "0xe8010ce590f401c9d61fef8ab05bea9bcec24281b795e5868809bc4e515aa530", + "indexes": { + "data": 0, + "gas": 0, + "value": 0 + }, + "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" + } + ] + }, + "pre": { + "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { + "balance": "0x0de0b6b3a7640000", + "code": "0x600160010160005500", + "nonce": "0x00", + "storage": {} + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "balance": "0x00", + "code": "0x", + "nonce": "0x01", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x0de0b6b3a7640000", + "code": "0x", + "nonce": "0x00", + "storage": {} + } + }, + "transaction": { + "data": [ + "0x" + ], + "gasLimit": [ + "0x061a80" + ], + "gasPrice": "0x0a", + "nonce": "0x00", + "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", + "value": [ + "0x0186a0" + ] + } + }, + "c_declared_format_wins": { + "_info": { + "fixture-format": "blockchain_test_engine", + "comment": "Carries blocks, so the shape alone would run it; the declared format is what decides, and this tool does not run that one." + }, + "blocks": [] + } +} diff --git a/test/integration/testcmd_broken/truncated.json b/test/integration/testcmd_broken/truncated.json new file mode 100644 index 0000000000..0c72185851 --- /dev/null +++ b/test/integration/testcmd_broken/truncated.json @@ -0,0 +1,3 @@ +{ + "a_case": { + "_info": {"comment": "The file ends here, mid-object, so it does not parse at all."} diff --git a/test/integration/testcmd_skipped/all_unsupported.json b/test/integration/testcmd_skipped/all_unsupported.json new file mode 100644 index 0000000000..400df5dbae --- /dev/null +++ b/test/integration/testcmd_skipped/all_unsupported.json @@ -0,0 +1,9 @@ +{ + "a_only_case_declines": { + "_info": { + "fixture-format": "blockchain_test_engine", + "comment": "The only case, in a format this tool does not run, so the file has nothing left to run and is skipped rather than passed." + }, + "blocks": [] + } +} diff --git a/test/integration/testcmd_skipped/empty.json b/test/integration/testcmd_skipped/empty.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/test/integration/testcmd_skipped/empty.json @@ -0,0 +1 @@ +{} diff --git a/test/integration/testcmd_skipped/partial_state_test.json b/test/integration/testcmd_skipped/partial_state_test.json new file mode 100644 index 0000000000..64d800acd4 --- /dev/null +++ b/test/integration/testcmd_skipped/partial_state_test.json @@ -0,0 +1,20 @@ +{ + "a_transaction_without_post": { + "comment": "Carries one of the two fields a state test is recognised by, so the shape does not name it and nothing else does either.", + "transaction": { + "data": [ + "0x" + ], + "gasLimit": [ + "0x061a80" + ], + "gasPrice": "0x0a", + "nonce": "0x00", + "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", + "value": [ + "0x0186a0" + ] + } + } +} diff --git a/test/integration/testcmd_skipped/pre_alloc.json b/test/integration/testcmd_skipped/pre_alloc.json new file mode 100644 index 0000000000..355c895850 --- /dev/null +++ b/test/integration/testcmd_skipped/pre_alloc.json @@ -0,0 +1,27 @@ +{ + "testIds": [ + "tests/ported_static/vmIOandFlowOperations/test_mload.py::test_mload[fork_Osaka-blockchain_test_engine_x]" + ], + "environment": { + "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentGasLimit": "0xff112233445566", + "currentNumber": "0x01" + }, + "network": "Osaka", + "chainId": 1, + "pre": { + "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { + "balance": "0x0de0b6b3a7640000", + "code": "0x600160010160005500", + "nonce": "0x00", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x0de0b6b3a7640000", + "code": "0x", + "nonce": "0x00", + "storage": {} + } + }, + "comment": "EEST keeps the pre-allocation its fixtures share beside them, 35715 files of it in one release, and none of it is a test. Several entries, so the reason the file is skipped for is recorded once rather than per entry." +} diff --git a/test/utils/test_collector.cpp b/test/utils/test_collector.cpp index ab30751b91..55f26b1d7e 100644 --- a/test/utils/test_collector.cpp +++ b/test/utils/test_collector.cpp @@ -4,6 +4,8 @@ #include "test_collector.hpp" #include +#include +#include #include namespace evmone::test @@ -54,4 +56,56 @@ void ignore_test_files(std::vector& files, std::span i ignored, [&relative](const fs::path& prefix) { return is_under(relative, prefix); }); }); } + +bool collect_tests( + std::vector& cases, const fs::path& root, const TestSettings& settings, evmc::VM& vm) +{ + if (is_directory(root)) + { + auto files = collect_test_files(root); + ignore_test_files(files, settings.ignored); + cases.reserve(cases.size() + files.size()); + for (const auto& file : files) + { + // Loaded when the test runs: loading a whole tree up front costs far more. + cases.push_back( + {file.path.string(), [path = file.path, &settings, &vm](TestReport& report) { + run_fixture_file(path, settings, vm, report); + }}); + } + return true; + } + + // Naming a file loads it now, to name the fixtures in it. One which cannot be loaded + // becomes a single test reporting why. + json::json file; + try + { + std::ifstream f{root}; + file = json::json::parse(f); + } + catch (const std::exception& ex) + { + // Also reported here: --collect-only never runs the test. + std::cerr << root.string() << ": " << ex.what() << '\n'; + cases.push_back({root.string(), + [error = std::current_exception()](auto&) { std::rethrow_exception(error); }}); + return false; + } + + // Whether the file is a fixture file at all decides what an unrecognised fixture in it means, + // exactly as it does for a whole file collected from a directory. + const auto file_holds_fixtures = is_fixture_file(file); + + for (const auto& [name, fixture] : file.items()) + { + if (!settings.selects(name)) + continue; + cases.push_back({root.string() + "::" + name, + [name, fixture, file_holds_fixtures, &settings, &vm](TestReport& report) { + run_fixture(name, fixture, file_holds_fixtures, settings, vm, report); + }}); + } + return true; +} } // namespace evmone::test diff --git a/test/utils/test_collector.hpp b/test/utils/test_collector.hpp index f116d6639e..596b802fd3 100644 --- a/test/utils/test_collector.hpp +++ b/test/utils/test_collector.hpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 #pragma once +#include #include #include #include @@ -32,4 +33,9 @@ struct TestFile /// "bc4895-withdrawals". void ignore_test_files( std::vector& files, std::span ignored); + +/// Adds to @p cases every test under @p root: one per file for a directory, one per fixture in +/// the file when the file itself is named. Returns whether every test was collected. +bool collect_tests(std::vector& cases, const std::filesystem::path& root, + const TestSettings& settings, evmc::VM& vm); } // namespace evmone::test diff --git a/test/utils/test_driver.cpp b/test/utils/test_driver.cpp index 5ad47d8382..0827f86b6d 100644 --- a/test/utils/test_driver.cpp +++ b/test/utils/test_driver.cpp @@ -3,13 +3,18 @@ // SPDX-License-Identifier: Apache-2.0 #include "test_driver.hpp" +#include +#include #include +#include #include #include #include namespace evmone::test { +namespace fs = std::filesystem; + namespace { /// The report is laid out like pytest's. @@ -65,6 +70,49 @@ class Progress m_column = 0; } }; + +/// The fixture formats this tool runs. EEST names the format in each fixture's "_info", and a +/// heuristic covers the hand-written and pre-EEST files which have no "_info" at all. +enum class Format +{ + state_test, + blockchain_test, +}; + +/// What this tool makes of one fixture. +struct Classification +{ + /// How to run it, when this tool runs it. + std::optional format; + /// A fixture, whether or not this tool runs it. + bool is_fixture = false; + /// Why it is not run, when it is not. + std::string reason; +}; + +Classification classify(const json::json& fixture) +{ + if (const auto info = fixture.find("_info"); info != fixture.end()) + { + if (const auto format = info->find("fixture-format"); format != info->end()) + { + if (*format == "state_test") + return {.format = Format::state_test, .is_fixture = true}; + if (*format == "blockchain_test") + return {.format = Format::blockchain_test, .is_fixture = true}; + return {.is_fixture = true, .reason = "unsupported fixture format: " + format->dump()}; + } + } + // Nothing declares the format: a hand-written or pre-EEST file, or an "_info" without one. + // Each shape is named by what only it carries; anything else is not a test at all, as EEST's + // shared pre-allocation kept beside the fixtures is not. + if (fixture.contains("blocks")) + return {.format = Format::blockchain_test, .is_fixture = true}; + if (fixture.contains("transaction") && fixture.contains("post")) + return {.format = Format::state_test, .is_fixture = true}; + return {.reason = "not a test"}; +} + } // namespace int run_tests(std::span cases, std::ostream& out, const RunOptions& options) @@ -185,4 +233,75 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio // case of its own still counts as passed, which this does not change. return passed == 0 ? NOTHING_VERIFIED : SUCCESS; } + +bool is_fixture_file(const json::json& contents) +{ + return std::ranges::any_of( + contents.items(), [](const auto& item) { return classify(item.value()).is_fixture; }); +} + +void run_fixture_file( + const fs::path& path, const TestSettings& settings, evmc::VM& vm, TestReport& report) +{ + std::ifstream f{path}; + // Named, because items() only borrows: iterating a temporary dangles. + const auto contents = json::json::parse(f); + const auto file_holds_fixtures = is_fixture_file(contents); + + std::optional declined; // The reason for the first fixture this tool declined. + bool any_ran = false; + for (const auto& [name, fixture] : contents.items()) + { + if (!settings.selects(name)) + continue; + try + { + run_fixture(name, fixture, file_holds_fixtures, settings, vm, report); + } + catch (const UnsupportedTestFeature& ex) + { + // This tool's own limit, whether the fixture declared a format it does not run or + // the file holds no test for its shape to be read against. + if (!declined) + declined = ex.what(); + continue; + } + catch (const std::exception& ex) + { + // The fixture is a test and it went wrong, which is this file's verdict but not the + // end of it: the fixtures after it are still worth running. + report.fail(concat("exception: ", ex.what())); + } + any_ran = true; + } + + // A file in which this tool ran nothing it was asked for is skipped, not passed. + // TODO: A file whose cases -k all deselected still passes, as it did before this command + // existed, so a filter which matches nothing reports a tree of passing tests. Skip it + // instead, and an empty selection reaches NOTHING_VERIFIED on its own. + if (!any_ran) + { + if (declined) + throw UnsupportedTestFeature{*declined}; + if (!file_holds_fixtures) + throw UnsupportedTestFeature{"no test cases"}; + } +} + +void run_fixture(const std::string& name, const json::json& fixture, bool file_holds_fixtures, + const TestSettings& settings, evmc::VM& vm, TestReport& report) +{ + report.start_case(name); // Names whatever the load itself reports. + const auto [format, is_fixture, reason] = classify(fixture); + if (format == Format::state_test) + run_state_test(make_state_test(name, fixture), vm, settings.trace_summary, report); + else if (format == Format::blockchain_test) + run_blockchain_test(make_blockchain_test(name, fixture), vm, report); + // Not recognising a fixture at all is a fault in the file, once the rest of it shows the + // file to be a fixture file. Anything else is this tool's own limit. + else if (file_holds_fixtures && !is_fixture) + report.fail(reason); + else + throw UnsupportedTestFeature{reason}; +} } // namespace evmone::test diff --git a/test/utils/test_driver.hpp b/test/utils/test_driver.hpp index 2694b48680..8431b7e270 100644 --- a/test/utils/test_driver.hpp +++ b/test/utils/test_driver.hpp @@ -3,10 +3,16 @@ // SPDX-License-Identifier: Apache-2.0 #pragma once +#include +#include #include +#include +#include namespace evmone::test { +namespace json = nlohmann; + /// Nothing failed and something passed. constexpr int SUCCESS = 0; @@ -43,4 +49,37 @@ struct RunOptions /// got. [[nodiscard]] int run_tests( std::span cases, std::ostream& out, const RunOptions& options = {}); + +/// What the tests are run with. +struct TestSettings +{ + /// Run only the fixtures whose name contains this. + std::optional name_filter; + + /// Paths, relative to a test directory, not to collect tests from. + std::vector ignored; + + /// Report each test's execution summary on the trace stream. + bool trace_summary = false; + + /// Whether the name filter, if any, keeps the fixture called @p name. + [[nodiscard]] bool selects(const std::string& name) const noexcept + { + return !name_filter.has_value() || name.find(*name_filter) != std::string::npos; + } +}; + +/// Whether the file holds fixtures at all. A property of the file, never of what the name +/// filter selected out of it, so both ways of naming a test agree. +[[nodiscard]] bool is_fixture_file(const json::json& contents); + +/// Runs every selected fixture of one fixture file, which together are one test. Throws +/// UnsupportedTestFeature for a file this tool has nothing to run in. +void run_fixture_file(const std::filesystem::path& path, const TestSettings& settings, evmc::VM& vm, + TestReport& report); + +/// Runs one fixture of a file named on its own. Whether the file holds fixtures at all decides +/// what an unrecognised one in it means, so @p file_holds_fixtures is asked of the whole file. +void run_fixture(const std::string& name, const json::json& fixture, bool file_holds_fixtures, + const TestSettings& settings, evmc::VM& vm, TestReport& report); } // namespace evmone::test diff --git a/tools/evmone/main.cpp b/tools/evmone/main.cpp index e5e6c24ece..9bd76d374a 100644 --- a/tools/evmone/main.cpp +++ b/tools/evmone/main.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -151,6 +152,57 @@ int exec_t8n_cmd(evmc::VM& vm, const T8nOptions& opts) evmone::tooling::t8n(vm, args); return 0; } + +/// The options of the "test" command. +struct TestOptions +{ + std::vector paths; + evmone::test::TestSettings settings; + evmone::test::RunOptions run; +}; + +const CLI::App& setup_test_cmd(CLI::App& app, TestOptions& opts) +{ + auto& cmd = *app.add_subcommand("test", "Run Ethereum tests")->fallthrough(); + cmd.add_option("path", opts.paths, + "Test file or directory. Every .json file under a directory, except index.json, is a " + "test; naming a file makes each test case in it a test of its own.") + ->required() + ->check(CLI::ExistingPath); + cmd.add_option( + "-k", opts.settings.name_filter, "Run only the test cases whose name contains this."); + cmd.add_option("--ignore", opts.settings.ignored, + "Path, relative to a test directory, not to collect tests from. May be given more than " + "once. Whole path components are matched, so --ignore bc4895 keeps bc4895-withdrawals.") + // Without this the option is variadic and swallows the positional paths after it. + ->allow_extra_args(false); + cmd.add_flag("--collect-only", opts.run.collect_only, + "List each collected test, one per line, and exit."); + cmd.add_flag("--trace-summary", opts.settings.trace_summary, + "Report each state test's execution summary, as --trace also does. Blockchain tests " + "have no summary to report."); + return cmd; +} + +int exec_test_cmd(evmc::VM& vm, TestOptions opts, bool trace) +{ + // main() has switched the tracer on already. Its line per instruction is worth + // unsynchronising the streams for, and would run into the progress row, as a summary would. + if (trace) + std::ios::sync_with_stdio(false); + opts.settings.trace_summary |= trace; + opts.run.progress = !opts.settings.trace_summary; + + std::vector cases; + bool all_collected = true; + for (const auto& p : opts.paths) + all_collected &= collect_tests(cases, p, opts.settings, vm); + + const auto exit_code = evmone::test::run_tests(cases, std::cout, opts.run); + // A file which could not be loaded fails the listing too, not only a run of it. + return all_collected ? exit_code : evmone::test::TESTS_FAILED; +} + } // namespace int main(int argc, const char* const* argv) noexcept @@ -198,6 +250,9 @@ int main(int argc, const char* const* argv) noexcept T8nOptions t8n_opts; const auto& t8n_cmd = setup_t8n_cmd(app, t8n_opts); + TestOptions test_opts; + const auto& test_cmd = setup_test_cmd(app, test_opts); + try { app.parse(argc, argv); @@ -218,6 +273,9 @@ int main(int argc, const char* const* argv) noexcept if (t8n_cmd) return exec_t8n_cmd(vm, t8n_opts); + if (test_cmd) + return exec_test_cmd(vm, test_opts, trace); + return 0; } catch (const CLI::ParseError& e) From 8b12a8f0257f51e6c27c4ca8947f182cdf357e27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 09:08:52 +0200 Subject: [PATCH 02/19] test: Settle that a file is not a test before running any of it An entry the tool did not recognise was a fault or a skip depending on whether the rest of its file holds fixtures, a bool threaded from the collector through a lambda into run_fixture to feed one branch. Loading the file settles it instead: a file with no fixture in it is skipped whole and nothing in it reaches execution, so an unrecognised entry in a fixture file is simply a fault. Named on the command line, such a file is one skipped test rather than one per entry. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/integration/CMakeLists.txt | 8 +++---- test/utils/test_collector.cpp | 27 +++++++++++---------- test/utils/test_driver.cpp | 42 +++++++++++++++------------------ test/utils/test_driver.hpp | 14 +++++------ 4 files changed, 44 insertions(+), 47 deletions(-) diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index 87aa3c3210..8ef2915b83 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -90,12 +90,12 @@ DUP1,4 ${PREFIX}/test_both_formats PROPERTIES PASS_REGULAR_EXPRESSION "= 2 passed in") # JSON which is not a test at all, as fixture directories hold beside their fixtures, is - # skipped rather than guessed at. + # skipped whole rather than guessed at, even named on its own. add_test(NAME ${PREFIX}/test_not_a_fixture COMMAND evmone-cli test ${CMAKE_CURRENT_SOURCE_DIR}/statetest/tests1/SuiteA/index.json) set_tests_properties( ${PREFIX}/test_not_a_fixture PROPERTIES PASS_REGULAR_EXPRESSION - "index\\.json::[^\n]* - not a test.*0 passed, 2 skipped in") + "SKIPPED[^\n]*index\\.json - not a test.*0 passed, 1 skipped in") # Over a directory a test is a whole file, so a case the loader refuses must not take the # rest of the file with it: the file passes on the case beside it, the refused one first. @@ -140,13 +140,13 @@ DUP1,4 "collected 1 test.*a_load_error:\n exception.*b_wrong_state_root:.*state root") # Collected from a directory a file is one test, so a file with nothing to run is skipped - # whole, named by the first reason which explains it. All three reasons live here. + # whole, named by the first reason which explains it. Both reasons live here. add_test(NAME ${PREFIX}/test_directory_skips_whole_files COMMAND evmone-cli test ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_skipped) set_tests_properties( ${PREFIX}/test_directory_skips_whole_files PROPERTIES PASS_REGULAR_EXPRESSION - "all_unsupported\\.json - unsupported fixture format[^\n]*\nSKIPPED[^\n]*empty\\.json - no test cases.*pre_alloc\\.json - not a test.*0 passed, 4 skipped in") + "all_unsupported\\.json - unsupported fixture format[^\n]*\nSKIPPED[^\n]*empty\\.json - not a test.*pre_alloc\\.json - not a test.*0 passed, 4 skipped in") # Nothing to run is not nothing to report: the run fails rather than passing empty. add_test(NAME ${PREFIX}/test_directory_skips_whole_files_exit_code COMMAND evmone-cli test diff --git a/test/utils/test_collector.cpp b/test/utils/test_collector.cpp index 55f26b1d7e..53d5df257a 100644 --- a/test/utils/test_collector.cpp +++ b/test/utils/test_collector.cpp @@ -4,7 +4,6 @@ #include "test_collector.hpp" #include -#include #include #include @@ -76,13 +75,19 @@ bool collect_tests( return true; } - // Naming a file loads it now, to name the fixtures in it. One which cannot be loaded - // becomes a single test reporting why. + // Naming a file loads it now, to name the fixtures in it. One which cannot be loaded, or + // which is not a test, becomes a single test reporting why. json::json file; try { - std::ifstream f{root}; - file = json::json::parse(f); + file = load_fixture_file(root); + } + catch (const UnsupportedTestFeature&) + { + // A skip, as when collected from a directory, not a broken collection. + cases.push_back({root.string(), + [error = std::current_exception()](auto&) { std::rethrow_exception(error); }}); + return true; } catch (const std::exception& ex) { @@ -93,18 +98,14 @@ bool collect_tests( return false; } - // Whether the file is a fixture file at all decides what an unrecognised fixture in it means, - // exactly as it does for a whole file collected from a directory. - const auto file_holds_fixtures = is_fixture_file(file); - for (const auto& [name, fixture] : file.items()) { if (!settings.selects(name)) continue; - cases.push_back({root.string() + "::" + name, - [name, fixture, file_holds_fixtures, &settings, &vm](TestReport& report) { - run_fixture(name, fixture, file_holds_fixtures, settings, vm, report); - }}); + cases.push_back( + {root.string() + "::" + name, [name, fixture, &settings, &vm](TestReport& report) { + run_fixture(name, fixture, settings, vm, report); + }}); } return true; } diff --git a/test/utils/test_driver.cpp b/test/utils/test_driver.cpp index 0827f86b6d..e8ab9de929 100644 --- a/test/utils/test_driver.cpp +++ b/test/utils/test_driver.cpp @@ -234,19 +234,23 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio return passed == 0 ? NOTHING_VERIFIED : SUCCESS; } -bool is_fixture_file(const json::json& contents) +json::json load_fixture_file(const fs::path& path) { - return std::ranges::any_of( - contents.items(), [](const auto& item) { return classify(item.value()).is_fixture; }); + std::ifstream f{path}; + auto contents = json::json::parse(f); + // Not one fixture in it: EEST keeps its shared pre-allocation and an index of the fixtures + // beside them, and neither is a test. + if (std::ranges::none_of( + contents.items(), [](const auto& item) { return classify(item.value()).is_fixture; })) + throw UnsupportedTestFeature{"not a test"}; + return contents; } void run_fixture_file( const fs::path& path, const TestSettings& settings, evmc::VM& vm, TestReport& report) { - std::ifstream f{path}; // Named, because items() only borrows: iterating a temporary dangles. - const auto contents = json::json::parse(f); - const auto file_holds_fixtures = is_fixture_file(contents); + const auto contents = load_fixture_file(path); std::optional declined; // The reason for the first fixture this tool declined. bool any_ran = false; @@ -256,12 +260,11 @@ void run_fixture_file( continue; try { - run_fixture(name, fixture, file_holds_fixtures, settings, vm, report); + run_fixture(name, fixture, settings, vm, report); } catch (const UnsupportedTestFeature& ex) { - // This tool's own limit, whether the fixture declared a format it does not run or - // the file holds no test for its shape to be read against. + // This tool's own limit: a format it does not run, or a fixture its loader refuses. if (!declined) declined = ex.what(); continue; @@ -279,17 +282,12 @@ void run_fixture_file( // TODO: A file whose cases -k all deselected still passes, as it did before this command // existed, so a filter which matches nothing reports a tree of passing tests. Skip it // instead, and an empty selection reaches NOTHING_VERIFIED on its own. - if (!any_ran) - { - if (declined) - throw UnsupportedTestFeature{*declined}; - if (!file_holds_fixtures) - throw UnsupportedTestFeature{"no test cases"}; - } + if (!any_ran && declined) + throw UnsupportedTestFeature{*declined}; } -void run_fixture(const std::string& name, const json::json& fixture, bool file_holds_fixtures, - const TestSettings& settings, evmc::VM& vm, TestReport& report) +void run_fixture(const std::string& name, const json::json& fixture, const TestSettings& settings, + evmc::VM& vm, TestReport& report) { report.start_case(name); // Names whatever the load itself reports. const auto [format, is_fixture, reason] = classify(fixture); @@ -297,11 +295,9 @@ void run_fixture(const std::string& name, const json::json& fixture, bool file_h run_state_test(make_state_test(name, fixture), vm, settings.trace_summary, report); else if (format == Format::blockchain_test) run_blockchain_test(make_blockchain_test(name, fixture), vm, report); - // Not recognising a fixture at all is a fault in the file, once the rest of it shows the - // file to be a fixture file. Anything else is this tool's own limit. - else if (file_holds_fixtures && !is_fixture) - report.fail(reason); + else if (is_fixture) + throw UnsupportedTestFeature{reason}; // A format this tool does not run. else - throw UnsupportedTestFeature{reason}; + report.fail(reason); // The rest of the file holds fixtures, so this one is broken. } } // namespace evmone::test diff --git a/test/utils/test_driver.hpp b/test/utils/test_driver.hpp index 8431b7e270..27cd9fce65 100644 --- a/test/utils/test_driver.hpp +++ b/test/utils/test_driver.hpp @@ -69,17 +69,17 @@ struct TestSettings } }; -/// Whether the file holds fixtures at all. A property of the file, never of what the name -/// filter selected out of it, so both ways of naming a test agree. -[[nodiscard]] bool is_fixture_file(const json::json& contents); +/// Parses the fixture file at @p path. Throws UnsupportedTestFeature for a file with no fixture +/// in it, which is nothing to run: fixture directories hold other JSON beside the fixtures. +[[nodiscard]] json::json load_fixture_file(const std::filesystem::path& path); /// Runs every selected fixture of one fixture file, which together are one test. Throws /// UnsupportedTestFeature for a file this tool has nothing to run in. void run_fixture_file(const std::filesystem::path& path, const TestSettings& settings, evmc::VM& vm, TestReport& report); -/// Runs one fixture of a file named on its own. Whether the file holds fixtures at all decides -/// what an unrecognised one in it means, so @p file_holds_fixtures is asked of the whole file. -void run_fixture(const std::string& name, const json::json& fixture, bool file_holds_fixtures, - const TestSettings& settings, evmc::VM& vm, TestReport& report); +/// Runs one fixture of a fixture file. One this tool does not recognise is a fault in the file; +/// one in a format it does not run is skipped. +void run_fixture(const std::string& name, const json::json& fixture, const TestSettings& settings, + evmc::VM& vm, TestReport& report); } // namespace evmone::test From b01df2a73c744acb955b1498265b5e756f2fd3a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 14:21:44 +0200 Subject: [PATCH 03/19] test: Pin the filter, the trace summary and the first decline Three of the command's ctests were named for behaviour they could not observe: deleting the -k filter from a directory run, hardcoding the trace summary off, or taking the last decline reason rather than the first all left the integration suite green. One run with -k and --trace-summary over a directory holding a failing case covers the first two, and a second declining case in all_unsupported names which reason wins. The two fault tests differed only in the filter, so they fold into one. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/integration/CMakeLists.txt | 93 +++++++++---------- .../testcmd/one_unsupported_case.json | 6 +- .../testcmd_cases/case_after_exception.json | 2 +- .../testcmd_fault/unrecognised_case.json | 2 +- .../testcmd_skipped/all_unsupported.json | 10 +- .../testcmd_skipped/partial_state_test.json | 2 +- test/utils/test_driver.cpp | 4 +- 7 files changed, 61 insertions(+), 58 deletions(-) diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index 8ef2915b83..c384300a21 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -82,113 +82,110 @@ DUP1,4 {\"pc\":6,\"op\":3,\"gas\":\"0xf4234\",\"gasCost\":\"0x3\",\"memSize\":0,\"stack\":\\[\"0x0\",\"0x4\"\\],\"depth\":1,\"refund\":0,\"opName\":\"SUB\"} ") + set(DATA ${CMAKE_CURRENT_SOURCE_DIR}) + # One command runs both fixture formats, deciding per test case which one it is. add_test(NAME ${PREFIX}/test_both_formats COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/statetest/tests1/SuiteA/test1.json - ${CMAKE_CURRENT_SOURCE_DIR}/blockchaintest/eip7778_block_gas.json) + ${DATA}/statetest/tests1/SuiteA/test1.json ${DATA}/blockchaintest/eip7778_block_gas.json) set_tests_properties( ${PREFIX}/test_both_formats PROPERTIES PASS_REGULAR_EXPRESSION "= 2 passed in") # JSON which is not a test at all, as fixture directories hold beside their fixtures, is # skipped whole rather than guessed at, even named on its own. - add_test(NAME ${PREFIX}/test_not_a_fixture COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/statetest/tests1/SuiteA/index.json) + add_test(NAME ${PREFIX}/test_not_a_test COMMAND evmone-cli test + ${DATA}/statetest/tests1/SuiteA/index.json) set_tests_properties( - ${PREFIX}/test_not_a_fixture PROPERTIES PASS_REGULAR_EXPRESSION + ${PREFIX}/test_not_a_test PROPERTIES PASS_REGULAR_EXPRESSION "SKIPPED[^\n]*index\\.json - not a test.*0 passed, 1 skipped in") # Over a directory a test is a whole file, so a case the loader refuses must not take the # rest of the file with it: the file passes on the case beside it, the refused one first. - add_test(NAME ${PREFIX}/test_directory_with_unsupported_case COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd) + add_test(NAME ${PREFIX}/test_dir_declined COMMAND evmone-cli test ${DATA}/testcmd) set_tests_properties( - ${PREFIX}/test_directory_with_unsupported_case PROPERTIES - PASS_REGULAR_EXPRESSION "= 1 passed in") + ${PREFIX}/test_dir_declined PROPERTIES PASS_REGULAR_EXPRESSION "= 1 passed in") # A PASS_REGULAR_EXPRESSION makes CTest ignore the exit code, so these two assert on nothing # else: a sound fixture directory succeeds, and a fault in one fails. - add_test(NAME ${PREFIX}/test_exit_code_success COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd) + add_test(NAME ${PREFIX}/test_exit_ok COMMAND evmone-cli test ${DATA}/testcmd) - add_test(NAME ${PREFIX}/test_exit_code_failure COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault) - set_tests_properties(${PREFIX}/test_exit_code_failure PROPERTIES WILL_FAIL TRUE) + add_test(NAME ${PREFIX}/test_exit_fail COMMAND evmone-cli test ${DATA}/testcmd_fault) + set_tests_properties(${PREFIX}/test_exit_fail PROPERTIES WILL_FAIL TRUE) # WILL_FAIL above accepts any non-zero exit, and a fault downgraded to a skip exits non-zero - # too, so the counts are what distinguish the two. - add_test(NAME ${PREFIX}/test_fault_is_not_a_skip COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault) + # too, so the counts are what distinguish the two. Whether a file is a fixture file is a + # property of the file, not of what -k selected: the case which is not a test still faults. + add_test(NAME ${PREFIX}/test_dir_fault COMMAND evmone-cli test + ${DATA}/testcmd_fault -k b_not_a_test) set_tests_properties( - ${PREFIX}/test_fault_is_not_a_skip PROPERTIES PASS_REGULAR_EXPRESSION "1 failed, 0 passed") + ${PREFIX}/test_dir_fault PROPERTIES PASS_REGULAR_EXPRESSION "1 failed, 0 passed") - # Whether a file is a fixture file is a property of the file, not of what -k selected from - # it: selecting only the case which is not a fixture must still fault. - add_test(NAME ${PREFIX}/test_fault_survives_filter COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault -k b_not_a_fixture) + # Over a directory -k selects within the file's test, so the summary is what proves a case + # ran: a count alone would hold just as well if the filter had dropped every case. The + # summary appears only with --trace-summary, which nothing else here asks for. + add_test(NAME ${PREFIX}/test_dir_filter COMMAND evmone-cli test + ${DATA}/statetest/filter -k passing_test_case --trace-summary) set_tests_properties( - ${PREFIX}/test_fault_survives_filter PROPERTIES - PASS_REGULAR_EXPRESSION "1 failed, 0 passed") + ${PREFIX}/test_dir_filter PROPERTIES + PASS_REGULAR_EXPRESSION "\"pass\":true" + FAIL_REGULAR_EXPRESSION "failing_test_case") # A case whose load throws must not abandon the cases after it, so both faults are reported. - add_test(NAME ${PREFIX}/test_case_after_exception COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_cases) + add_test(NAME ${PREFIX}/test_after_exception COMMAND evmone-cli test ${DATA}/testcmd_cases) set_tests_properties( - ${PREFIX}/test_case_after_exception PROPERTIES + ${PREFIX}/test_after_exception PROPERTIES # "collected 1 test" keeps this on the directory form, where the whole file is one test # and the per-case catch is what keeps the second fault reachable. PASS_REGULAR_EXPRESSION - "collected 1 test.*a_load_error:\n exception.*b_wrong_state_root:.*state root") + "collected 1 test.*a_load_error:\n exception.*b_bad_root:.*state root") # Collected from a directory a file is one test, so a file with nothing to run is skipped - # whole, named by the first reason which explains it. Both reasons live here. - add_test(NAME ${PREFIX}/test_directory_skips_whole_files COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_skipped) + # whole, named by the first reason which explains it: all_unsupported declines twice, and + # the format named is the first one's. + add_test(NAME ${PREFIX}/test_dir_skipped COMMAND evmone-cli test ${DATA}/testcmd_skipped) set_tests_properties( - ${PREFIX}/test_directory_skips_whole_files PROPERTIES + ${PREFIX}/test_dir_skipped PROPERTIES PASS_REGULAR_EXPRESSION - "all_unsupported\\.json - unsupported fixture format[^\n]*\nSKIPPED[^\n]*empty\\.json - not a test.*pre_alloc\\.json - not a test.*0 passed, 4 skipped in") + "all_unsupported\\.json - unsupported fixture format: \"blockchain_test_engine\"\nSKIPPED[^\n]*empty\\.json - not a test.*pre_alloc\\.json - not a test.*0 passed, 4 skipped in") # Nothing to run is not nothing to report: the run fails rather than passing empty. - add_test(NAME ${PREFIX}/test_directory_skips_whole_files_exit_code COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_skipped) - set_tests_properties( - ${PREFIX}/test_directory_skips_whole_files_exit_code PROPERTIES WILL_FAIL TRUE) + add_test(NAME ${PREFIX}/test_dir_skipped_exit COMMAND evmone-cli test ${DATA}/testcmd_skipped) + set_tests_properties(${PREFIX}/test_dir_skipped_exit PROPERTIES WILL_FAIL TRUE) # Naming the file instead of the directory makes each case its own test, and a fault in one # is still a fault: the case beside it runs. add_test(NAME ${PREFIX}/test_file_fault COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault/unrecognised_case.json) + ${DATA}/testcmd_fault/unrecognised_case.json) set_tests_properties( ${PREFIX}/test_file_fault PROPERTIES PASS_REGULAR_EXPRESSION - "FAILED[^\n]*unrecognised_case\\.json::b_not_a_fixture.*1 failed, 1 passed in") + "FAILED[^\n]*unrecognised_case\\.json::b_not_a_test.*1 failed, 1 passed in") # -k selects among the cases of a named file too. add_test(NAME ${PREFIX}/test_file_filter COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault/unrecognised_case.json -k a_runs) + ${DATA}/testcmd_fault/unrecognised_case.json -k a_runs) set_tests_properties( ${PREFIX}/test_file_filter PROPERTIES PASS_REGULAR_EXPRESSION "= 1 passed in") # Naming a file whose cases this tool does not run skips them one by one, where collecting # the same file from a directory would skip it whole. - add_test(NAME ${PREFIX}/test_file_declined_cases COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd/one_unsupported_case.json) + add_test(NAME ${PREFIX}/test_file_declined COMMAND evmone-cli test + ${DATA}/testcmd/one_unsupported_case.json) set_tests_properties( - ${PREFIX}/test_file_declined_cases PROPERTIES PASS_REGULAR_EXPRESSION - "SKIPPED[^\n]*c_declared_format_wins - unsupported fixture format.*= 1 passed, 2 skipped in") + ${PREFIX}/test_file_declined PROPERTIES PASS_REGULAR_EXPRESSION + "SKIPPED[^\n]*c_engine - unsupported fixture format.*= 1 passed, 2 skipped in") # --trace reaches the runner through the app-level flag, and turns the progress row into a # line naming each test so the trace which follows it can be told apart. add_test(NAME ${PREFIX}/test_trace COMMAND evmone-cli test --trace - ${CMAKE_CURRENT_SOURCE_DIR}/statetest/tests1/SuiteA/test1.json) + ${DATA}/statetest/tests1/SuiteA/test1.json) set_tests_properties( ${PREFIX}/test_trace PROPERTIES PASS_REGULAR_EXPRESSION "test1\\.json::test1\n\\{\"pc\":0,[^\n]*\"opName\":\"PUSH1\"\\}.*= 1 passed in") # A named file which does not parse names itself, because no case in it can be. - add_test(NAME ${PREFIX}/test_file_does_not_parse COMMAND evmone-cli test - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_broken/truncated.json) + add_test(NAME ${PREFIX}/test_file_unparsed COMMAND evmone-cli test + ${DATA}/testcmd_broken/truncated.json) set_tests_properties( - ${PREFIX}/test_file_does_not_parse PROPERTIES PASS_REGULAR_EXPRESSION + ${PREFIX}/test_file_unparsed PROPERTIES PASS_REGULAR_EXPRESSION "FAILED[^\n]*truncated\\.json - exception[^\n]*parse error.*1 failed, 0 passed in") endif() diff --git a/test/integration/testcmd/one_unsupported_case.json b/test/integration/testcmd/one_unsupported_case.json index 20b93aafb7..35eea677ad 100644 --- a/test/integration/testcmd/one_unsupported_case.json +++ b/test/integration/testcmd/one_unsupported_case.json @@ -1,5 +1,5 @@ { - "a_unsupported_unsupported_rlp": { + "a_bad_rlp": { "_info": { "comment": "Declines to load: expectException without rlp_decoded. Nothing of this case runs, so the case beside it is what the file reports." }, @@ -44,7 +44,7 @@ } } }, - "b_runs_state_test": { + "b_state": { "_info": { "comment": "A state test beside blockchain ones, which is the whole point of deciding the format per case." }, @@ -106,7 +106,7 @@ ] } }, - "c_declared_format_wins": { + "c_engine": { "_info": { "fixture-format": "blockchain_test_engine", "comment": "Carries blocks, so the shape alone would run it; the declared format is what decides, and this tool does not run that one." diff --git a/test/integration/testcmd_cases/case_after_exception.json b/test/integration/testcmd_cases/case_after_exception.json index 0b8f8e37f7..daac56ce62 100644 --- a/test/integration/testcmd_cases/case_after_exception.json +++ b/test/integration/testcmd_cases/case_after_exception.json @@ -5,7 +5,7 @@ "comment": "A state test with no pre state, so loading it throws." } }, - "b_wrong_state_root": { + "b_bad_root": { "_info": { "comment": "Runs and fails on the state root. Only reported if the case before it did not abandon the file." }, diff --git a/test/integration/testcmd_fault/unrecognised_case.json b/test/integration/testcmd_fault/unrecognised_case.json index e389b69953..2e3a02ea41 100644 --- a/test/integration/testcmd_fault/unrecognised_case.json +++ b/test/integration/testcmd_fault/unrecognised_case.json @@ -59,7 +59,7 @@ ] } }, - "b_not_a_fixture": { + "b_not_a_test": { "_info": { "comment": "Not a test: no fixture fields at all. Beside a case which runs, so this is unmistakably a fixture file with one broken case in it." } diff --git a/test/integration/testcmd_skipped/all_unsupported.json b/test/integration/testcmd_skipped/all_unsupported.json index 400df5dbae..2a9fb9915b 100644 --- a/test/integration/testcmd_skipped/all_unsupported.json +++ b/test/integration/testcmd_skipped/all_unsupported.json @@ -1,8 +1,14 @@ { - "a_only_case_declines": { + "a_engine": { "_info": { "fixture-format": "blockchain_test_engine", - "comment": "The only case, in a format this tool does not run, so the file has nothing left to run and is skipped rather than passed." + "comment": "The file is named by this reason, the first collected, not the one below." + }, + "blocks": [] + }, + "b_engine_x": { + "_info": { + "fixture-format": "blockchain_test_engine_x" }, "blocks": [] } diff --git a/test/integration/testcmd_skipped/partial_state_test.json b/test/integration/testcmd_skipped/partial_state_test.json index 64d800acd4..77e71ca03c 100644 --- a/test/integration/testcmd_skipped/partial_state_test.json +++ b/test/integration/testcmd_skipped/partial_state_test.json @@ -1,5 +1,5 @@ { - "a_transaction_without_post": { + "a_no_post": { "comment": "Carries one of the two fields a state test is recognised by, so the shape does not name it and nothing else does either.", "transaction": { "data": [ diff --git a/test/utils/test_driver.cpp b/test/utils/test_driver.cpp index e8ab9de929..ced3eeeeed 100644 --- a/test/utils/test_driver.cpp +++ b/test/utils/test_driver.cpp @@ -280,8 +280,8 @@ void run_fixture_file( // A file in which this tool ran nothing it was asked for is skipped, not passed. // TODO: A file whose cases -k all deselected still passes, as it did before this command - // existed, so a filter which matches nothing reports a tree of passing tests. Skip it - // instead, and an empty selection reaches NOTHING_VERIFIED on its own. + // existed, so a filter which matches nothing turns a failing tree green. Skip it instead, + // and an empty selection reaches NOTHING_VERIFIED on its own. if (!any_ran && declined) throw UnsupportedTestFeature{*declined}; } From ec2fab4b73e4445870999c45731333f96ba8f8f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 14:53:39 +0200 Subject: [PATCH 04/19] test: Name a fixture by its input, not by what it expects A fixture was taken for a blockchain test on a "blocks" key alone, so any JSON sidecar carrying one was run as a test and failed the whole run with "key 'genesisBlockHeader' not found", while the symmetric miss on the state shape was a quiet skip. Each shape now needs the state it starts from and what is applied to it: "pre" with "blocks", or "pre" with "transaction". Neither reads an expectation, so a fixture whose expectations are missing is still run rather than passing as no test at all, and "pre" is in every fixture of every format. classify() returns the format alone, the two ways of not running one among its values, so run_fixture switches over it exhaustively and no return site can label a fixture wrongly. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/integration/CMakeLists.txt | 8 ++- .../testcmd_skipped/partial_shapes.json | 24 +++++++ .../testcmd_skipped/partial_state_test.json | 20 ------ test/utils/test_driver.cpp | 63 +++++++++---------- 4 files changed, 60 insertions(+), 55 deletions(-) create mode 100644 test/integration/testcmd_skipped/partial_shapes.json delete mode 100644 test/integration/testcmd_skipped/partial_state_test.json diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index c384300a21..b79f2f0b66 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -84,11 +84,13 @@ DUP1,4 set(DATA ${CMAKE_CURRENT_SOURCE_DIR}) - # One command runs both fixture formats, deciding per test case which one it is. + # One command runs both fixture formats, deciding per test case which one it is: the state + # test and the last one by shape, the blockchain test by the format its "_info" declares. add_test(NAME ${PREFIX}/test_both_formats COMMAND evmone-cli test - ${DATA}/statetest/tests1/SuiteA/test1.json ${DATA}/blockchaintest/eip7778_block_gas.json) + ${DATA}/statetest/tests1/SuiteA/test1.json ${DATA}/blockchaintest/eip7778_block_gas.json + ${DATA}/blockchaintest/unrecovered_sender_blob_gas.json) set_tests_properties( - ${PREFIX}/test_both_formats PROPERTIES PASS_REGULAR_EXPRESSION "= 2 passed in") + ${PREFIX}/test_both_formats PROPERTIES PASS_REGULAR_EXPRESSION "= 3 passed in") # JSON which is not a test at all, as fixture directories hold beside their fixtures, is # skipped whole rather than guessed at, even named on its own. diff --git a/test/integration/testcmd_skipped/partial_shapes.json b/test/integration/testcmd_skipped/partial_shapes.json new file mode 100644 index 0000000000..5d85ea8afa --- /dev/null +++ b/test/integration/testcmd_skipped/partial_shapes.json @@ -0,0 +1,24 @@ +{ + "a_blocks_no_pre": { + "comment": "Carries one of the two keys a blockchain test is named by, so the shape does not name it and nothing else does either.", + "blocks": [] + }, + "b_tx_no_pre": { + "comment": "The same for a state test: a transaction to apply, but no state to apply it to.", + "transaction": { + "data": [ + "0x" + ], + "gasLimit": [ + "0x061a80" + ], + "gasPrice": "0x0a", + "nonce": "0x00", + "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", + "value": [ + "0x0186a0" + ] + } + } +} diff --git a/test/integration/testcmd_skipped/partial_state_test.json b/test/integration/testcmd_skipped/partial_state_test.json deleted file mode 100644 index 77e71ca03c..0000000000 --- a/test/integration/testcmd_skipped/partial_state_test.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "a_no_post": { - "comment": "Carries one of the two fields a state test is recognised by, so the shape does not name it and nothing else does either.", - "transaction": { - "data": [ - "0x" - ], - "gasLimit": [ - "0x061a80" - ], - "gasPrice": "0x0a", - "nonce": "0x00", - "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", - "value": [ - "0x0186a0" - ] - } - } -} diff --git a/test/utils/test_driver.cpp b/test/utils/test_driver.cpp index ced3eeeeed..979231ab7e 100644 --- a/test/utils/test_driver.cpp +++ b/test/utils/test_driver.cpp @@ -71,46 +71,38 @@ class Progress } }; -/// The fixture formats this tool runs. EEST names the format in each fixture's "_info", and a +/// What this tool makes of one fixture. EEST names the format in each fixture's "_info", and a /// heuristic covers the hand-written and pre-EEST files which have no "_info" at all. enum class Format { state_test, blockchain_test, + unsupported, ///< A fixture, in a format this tool does not run. + not_a_test, ///< Not a fixture at all. }; -/// What this tool makes of one fixture. -struct Classification -{ - /// How to run it, when this tool runs it. - std::optional format; - /// A fixture, whether or not this tool runs it. - bool is_fixture = false; - /// Why it is not run, when it is not. - std::string reason; -}; - -Classification classify(const json::json& fixture) +Format classify(const json::json& fixture) { if (const auto info = fixture.find("_info"); info != fixture.end()) { if (const auto format = info->find("fixture-format"); format != info->end()) { if (*format == "state_test") - return {.format = Format::state_test, .is_fixture = true}; + return Format::state_test; if (*format == "blockchain_test") - return {.format = Format::blockchain_test, .is_fixture = true}; - return {.is_fixture = true, .reason = "unsupported fixture format: " + format->dump()}; + return Format::blockchain_test; + return Format::unsupported; } } // Nothing declares the format: a hand-written or pre-EEST file, or an "_info" without one. - // Each shape is named by what only it carries; anything else is not a test at all, as EEST's - // shared pre-allocation kept beside the fixtures is not. - if (fixture.contains("blocks")) - return {.format = Format::blockchain_test, .is_fixture = true}; - if (fixture.contains("transaction") && fixture.contains("post")) - return {.format = Format::state_test, .is_fixture = true}; - return {.reason = "not a test"}; + // Each shape is named by the state it starts from and what is applied to it, never by what + // it expects, so a fixture whose expectations are missing is still a test and is run. + // Anything else is not a test at all, as EEST's shared pre-allocation is not. + if (fixture.contains("pre") && fixture.contains("blocks")) + return Format::blockchain_test; + if (fixture.contains("pre") && fixture.contains("transaction")) + return Format::state_test; + return Format::not_a_test; } } // namespace @@ -240,8 +232,8 @@ json::json load_fixture_file(const fs::path& path) auto contents = json::json::parse(f); // Not one fixture in it: EEST keeps its shared pre-allocation and an index of the fixtures // beside them, and neither is a test. - if (std::ranges::none_of( - contents.items(), [](const auto& item) { return classify(item.value()).is_fixture; })) + if (std::ranges::none_of(contents.items(), + [](const auto& i) { return classify(i.value()) != Format::not_a_test; })) throw UnsupportedTestFeature{"not a test"}; return contents; } @@ -290,14 +282,21 @@ void run_fixture(const std::string& name, const json::json& fixture, const TestS evmc::VM& vm, TestReport& report) { report.start_case(name); // Names whatever the load itself reports. - const auto [format, is_fixture, reason] = classify(fixture); - if (format == Format::state_test) + switch (classify(fixture)) + { + case Format::state_test: run_state_test(make_state_test(name, fixture), vm, settings.trace_summary, report); - else if (format == Format::blockchain_test) + break; + case Format::blockchain_test: run_blockchain_test(make_blockchain_test(name, fixture), vm, report); - else if (is_fixture) - throw UnsupportedTestFeature{reason}; // A format this tool does not run. - else - report.fail(reason); // The rest of the file holds fixtures, so this one is broken. + break; + case Format::unsupported: + throw UnsupportedTestFeature{ + "unsupported fixture format: " + fixture.at("_info").at("fixture-format").dump()}; + case Format::not_a_test: + // The rest of the file holds fixtures, so this one is broken. + report.fail("not a test"); + break; + } } } // namespace evmone::test From d398d3bec2bec09b27dffd8705bc7d7beb83b354 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 16:24:31 +0200 Subject: [PATCH 05/19] test: Refuse a file which is not an object, and one which will not open items() walks an array by index, so a top-level JSON array was collected as fixtures named "0", "1", ... and run. A document which is not an object holds no fixture and is not a test. An ifstream which never opened reads as EOF, so every I/O failure was reported as a JSON syntax error in a file which has none, a 0-byte file and an unreadable one giving the same message. Check the stream. The help and the collector both still described a named file as one test per case in it, which a file holding no fixture at all has not been since it started being skipped whole. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/integration/CMakeLists.txt | 2 +- .../integration/testcmd_skipped/not_an_object.json | 7 +++++++ test/utils/test_collector.hpp | 5 +++-- test/utils/test_driver.cpp | 14 ++++++++++---- tools/evmone/main.cpp | 5 +++-- 5 files changed, 24 insertions(+), 9 deletions(-) create mode 100644 test/integration/testcmd_skipped/not_an_object.json diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index b79f2f0b66..5ba1bd7922 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -147,7 +147,7 @@ DUP1,4 set_tests_properties( ${PREFIX}/test_dir_skipped PROPERTIES PASS_REGULAR_EXPRESSION - "all_unsupported\\.json - unsupported fixture format: \"blockchain_test_engine\"\nSKIPPED[^\n]*empty\\.json - not a test.*pre_alloc\\.json - not a test.*0 passed, 4 skipped in") + "all_unsupported\\.json - unsupported fixture format: \"blockchain_test_engine\"\nSKIPPED[^\n]*empty\\.json - not a test.*pre_alloc\\.json - not a test.*0 passed, 5 skipped in") # Nothing to run is not nothing to report: the run fails rather than passing empty. add_test(NAME ${PREFIX}/test_dir_skipped_exit COMMAND evmone-cli test ${DATA}/testcmd_skipped) diff --git a/test/integration/testcmd_skipped/not_an_object.json b/test/integration/testcmd_skipped/not_an_object.json new file mode 100644 index 0000000000..723839b091 --- /dev/null +++ b/test/integration/testcmd_skipped/not_an_object.json @@ -0,0 +1,7 @@ +[ + { + "comment": "A top-level array, so it holds no named fixture. Its one element would be taken for a blockchain test if the array were walked by index.", + "pre": {}, + "blocks": [] + } +] diff --git a/test/utils/test_collector.hpp b/test/utils/test_collector.hpp index 596b802fd3..2c193d57e3 100644 --- a/test/utils/test_collector.hpp +++ b/test/utils/test_collector.hpp @@ -34,8 +34,9 @@ struct TestFile void ignore_test_files( std::vector& files, std::span ignored); -/// Adds to @p cases every test under @p root: one per file for a directory, one per fixture in -/// the file when the file itself is named. Returns whether every test was collected. +/// Adds to @p cases every test under @p root: one per file for a directory, one per fixture for +/// a named file, or one for a named file holding no fixture. Returns whether every test was +/// collected. bool collect_tests(std::vector& cases, const std::filesystem::path& root, const TestSettings& settings, evmc::VM& vm); } // namespace evmone::test diff --git a/test/utils/test_driver.cpp b/test/utils/test_driver.cpp index 979231ab7e..ecca6e1d2e 100644 --- a/test/utils/test_driver.cpp +++ b/test/utils/test_driver.cpp @@ -229,11 +229,17 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio json::json load_fixture_file(const fs::path& path) { std::ifstream f{path}; - auto contents = json::json::parse(f); + // A stream which never opened reads as EOF, which parses as a syntax error in a file which + // has none. + if (!f) + throw std::runtime_error{"cannot open the file"}; + const auto contents = json::json::parse(f); // Not one fixture in it: EEST keeps its shared pre-allocation and an index of the fixtures - // beside them, and neither is a test. - if (std::ranges::none_of(contents.items(), - [](const auto& i) { return classify(i.value()) != Format::not_a_test; })) + // beside them, and neither is a test. Nor is a document which is not an object at all: + // items() would walk an array by index, naming its elements "0", "1", ... + if (!contents.is_object() || std::ranges::none_of(contents.items(), [](const auto& i) { + return classify(i.value()) != Format::not_a_test; + })) throw UnsupportedTestFeature{"not a test"}; return contents; } diff --git a/tools/evmone/main.cpp b/tools/evmone/main.cpp index 9bd76d374a..3cd6cf0778 100644 --- a/tools/evmone/main.cpp +++ b/tools/evmone/main.cpp @@ -165,8 +165,9 @@ const CLI::App& setup_test_cmd(CLI::App& app, TestOptions& opts) { auto& cmd = *app.add_subcommand("test", "Run Ethereum tests")->fallthrough(); cmd.add_option("path", opts.paths, - "Test file or directory. Every .json file under a directory, except index.json, is a " - "test; naming a file makes each test case in it a test of its own.") + "Test file or directory. Under a directory every .json file except index.json is " + "one test. A named file gives one test per fixture, or one for the file when it holds " + "no fixture.") ->required() ->check(CLI::ExistingPath); cmd.add_option( From 413cea4f96fade9970b2a43a2fcfbee00afa29ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 16:29:40 +0200 Subject: [PATCH 06/19] test: Say what the pre-allocation fixture is for Its comment described being skipped one entry at a time, which stopped being how a file with no fixture in it is read, and pinned a file count from one release into a checked-in fixture. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/integration/testcmd_skipped/pre_alloc.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/testcmd_skipped/pre_alloc.json b/test/integration/testcmd_skipped/pre_alloc.json index 355c895850..015cd24e1d 100644 --- a/test/integration/testcmd_skipped/pre_alloc.json +++ b/test/integration/testcmd_skipped/pre_alloc.json @@ -23,5 +23,5 @@ "storage": {} } }, - "comment": "EEST keeps the pre-allocation its fixtures share beside them, 35715 files of it in one release, and none of it is a test. Several entries, so the reason the file is skipped for is recorded once rather than per entry." + "comment": "EEST keeps the pre-allocation its fixtures share beside them, and none of it is a test. Several entries, none of which names a format or carries a shape." } From 309534940537641122d59fcc2edbaef711e40c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 16:30:20 +0200 Subject: [PATCH 07/19] test: Report a fixture's exception as every other failure is The per-fixture catch built one string where the two-argument fail() splits the check from its detail, and only the detail is indented line by line, so a what() of several lines broke the report's columns. It also caught nothing but std::exception, where the run loop around it catches everything. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/utils/test_driver.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/utils/test_driver.cpp b/test/utils/test_driver.cpp index ecca6e1d2e..9599760339 100644 --- a/test/utils/test_driver.cpp +++ b/test/utils/test_driver.cpp @@ -270,8 +270,13 @@ void run_fixture_file( catch (const std::exception& ex) { // The fixture is a test and it went wrong, which is this file's verdict but not the - // end of it: the fixtures after it are still worth running. - report.fail(concat("exception: ", ex.what())); + // end of it: the fixtures after it are still worth running. Reported as run_tests + // reports one, so a what() of several lines keeps the indent of the rest. + report.fail("exception", ex.what()); + } + catch (...) + { + report.fail("exception", "not derived from std::exception"); } any_ran = true; } From c9252a9125e60efb75b8ebf9010ac10c185d303a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 16:31:40 +0200 Subject: [PATCH 08/19] test: Keep the histogram off the progress row --trace turned the progress row into a line per test so its output could be told apart, but --histogram wrote its table into the row mid-line. Both write per test, so both decide the same way. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/integration/CMakeLists.txt | 11 +++++++++-- tools/evmone/main.cpp | 9 +++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index 5ba1bd7922..bd562cd457 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -175,8 +175,15 @@ DUP1,4 ${PREFIX}/test_file_declined PROPERTIES PASS_REGULAR_EXPRESSION "SKIPPED[^\n]*c_engine - unsupported fixture format.*= 1 passed, 2 skipped in") - # --trace reaches the runner through the app-level flag, and turns the progress row into a - # line naming each test so the trace which follows it can be told apart. + # --trace and --histogram reach the runner through the app-level flags, and turn the progress + # row into a line naming each test so what each writes can be told apart. + add_test(NAME ${PREFIX}/test_histogram COMMAND evmone-cli test --histogram + ${DATA}/statetest/tests1/SuiteA/test1.json) + set_tests_properties( + ${PREFIX}/test_histogram PROPERTIES PASS_REGULAR_EXPRESSION + "test1\\.json::test1 +--- # HISTOGRAM.*= 1 passed in") + add_test(NAME ${PREFIX}/test_trace COMMAND evmone-cli test --trace ${DATA}/statetest/tests1/SuiteA/test1.json) set_tests_properties( diff --git a/tools/evmone/main.cpp b/tools/evmone/main.cpp index 3cd6cf0778..2dfafc0fe9 100644 --- a/tools/evmone/main.cpp +++ b/tools/evmone/main.cpp @@ -185,14 +185,15 @@ const CLI::App& setup_test_cmd(CLI::App& app, TestOptions& opts) return cmd; } -int exec_test_cmd(evmc::VM& vm, TestOptions opts, bool trace) +int exec_test_cmd(evmc::VM& vm, TestOptions opts, bool trace, bool histogram) { // main() has switched the tracer on already. Its line per instruction is worth - // unsynchronising the streams for, and would run into the progress row, as a summary would. + // unsynchronising the streams for, and anything it writes per test would run into the + // progress row, as a summary would. if (trace) std::ios::sync_with_stdio(false); opts.settings.trace_summary |= trace; - opts.run.progress = !opts.settings.trace_summary; + opts.run.progress = !(opts.settings.trace_summary || histogram); std::vector cases; bool all_collected = true; @@ -275,7 +276,7 @@ int main(int argc, const char* const* argv) noexcept return exec_t8n_cmd(vm, t8n_opts); if (test_cmd) - return exec_test_cmd(vm, test_opts, trace); + return exec_test_cmd(vm, test_opts, trace, histogram); return 0; } From a1f11e14f43435d4e490ffc2dbde37d553f5bcce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 16:32:10 +0200 Subject: [PATCH 09/19] test: Say that a collected test outlives the call which made it The tests hold the settings and the VM by reference, which only the caller can honour, and dropping the return value turned a tree which could not be collected from a failure into a silent success. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/utils/test_collector.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/utils/test_collector.hpp b/test/utils/test_collector.hpp index 2c193d57e3..72abe1c132 100644 --- a/test/utils/test_collector.hpp +++ b/test/utils/test_collector.hpp @@ -36,7 +36,7 @@ void ignore_test_files( /// Adds to @p cases every test under @p root: one per file for a directory, one per fixture for /// a named file, or one for a named file holding no fixture. Returns whether every test was -/// collected. -bool collect_tests(std::vector& cases, const std::filesystem::path& root, +/// collected. The tests hold @p settings and @p vm by reference, so both must outlive them. +[[nodiscard]] bool collect_tests(std::vector& cases, const std::filesystem::path& root, const TestSettings& settings, evmc::VM& vm); } // namespace evmone::test From abcda18c0ada379e0d3d7c156804128c349fcaba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 16:33:48 +0200 Subject: [PATCH 10/19] test: Share one document between the tests of a named file Each test captured the fixture it was made from, and the capture copies the subtree it names, so a file's whole document was held once more over again, and a listing paid for every copy to read nothing but the name. Hold the document once and index it when the test runs: peak memory on a 3MB file drops by about a quarter. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/utils/test_collector.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/utils/test_collector.cpp b/test/utils/test_collector.cpp index 53d5df257a..36f3e25cc5 100644 --- a/test/utils/test_collector.cpp +++ b/test/utils/test_collector.cpp @@ -5,6 +5,7 @@ #include "test_collector.hpp" #include #include +#include #include namespace evmone::test @@ -98,13 +99,16 @@ bool collect_tests( return false; } - for (const auto& [name, fixture] : file.items()) + // One document shared by every test of it. Capturing the fixture itself would copy its + // subtree into each test, which a listing pays for in full to read nothing but the name. + const auto doc = std::make_shared(std::move(file)); + for (const auto& [name, fixture] : doc->items()) { if (!settings.selects(name)) continue; cases.push_back( - {root.string() + "::" + name, [name, fixture, &settings, &vm](TestReport& report) { - run_fixture(name, fixture, settings, vm, report); + {root.string() + "::" + name, [doc, name = name, &settings, &vm](TestReport& report) { + run_fixture(name, doc->at(name), settings, vm, report); }}); } return true; From 5fa8e79e161f3d6e77bb6597d1b60d8484259013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 2 Sep 2026 16:36:49 +0200 Subject: [PATCH 11/19] test: Name the fixtures a file declined but ran beside Collected from a directory a whole file is one test, so a file which ran anything reported nothing about the fixtures it declined: two of three were dropped from a run which said "1 passed" and exited 0. The day a fixture stops being one this tool runs, its coverage would go with it and the count would not move. Name each on the error stream, which stays empty on a release whose files are each of one format. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/integration/CMakeLists.txt | 4 +++- test/utils/test_driver.cpp | 14 +++++++++----- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index bd562cd457..8abcba389e 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -102,9 +102,11 @@ DUP1,4 # Over a directory a test is a whole file, so a case the loader refuses must not take the # rest of the file with it: the file passes on the case beside it, the refused one first. + # The file's own verdict says nothing about what it declined, so each declined case is named. add_test(NAME ${PREFIX}/test_dir_declined COMMAND evmone-cli test ${DATA}/testcmd) set_tests_properties( - ${PREFIX}/test_dir_declined PROPERTIES PASS_REGULAR_EXPRESSION "= 1 passed in") + ${PREFIX}/test_dir_declined PROPERTIES PASS_REGULAR_EXPRESSION + "a_bad_rlp: tests with invalidly rlp-encoded blocks.*c_engine: unsupported fixture format.*= 1 passed in") # A PASS_REGULAR_EXPRESSION makes CTest ignore the exit code, so these two assert on nothing # else: a sound fixture directory succeeds, and a fault in one fails. diff --git a/test/utils/test_driver.cpp b/test/utils/test_driver.cpp index 9599760339..6753451f34 100644 --- a/test/utils/test_driver.cpp +++ b/test/utils/test_driver.cpp @@ -250,7 +250,7 @@ void run_fixture_file( // Named, because items() only borrows: iterating a temporary dangles. const auto contents = load_fixture_file(path); - std::optional declined; // The reason for the first fixture this tool declined. + std::vector> declined; // Fixture name and reason. bool any_ran = false; for (const auto& [name, fixture] : contents.items()) { @@ -263,8 +263,7 @@ void run_fixture_file( catch (const UnsupportedTestFeature& ex) { // This tool's own limit: a format it does not run, or a fixture its loader refuses. - if (!declined) - declined = ex.what(); + declined.emplace_back(name, ex.what()); continue; } catch (const std::exception& ex) @@ -285,8 +284,13 @@ void run_fixture_file( // TODO: A file whose cases -k all deselected still passes, as it did before this command // existed, so a filter which matches nothing turns a failing tree green. Skip it instead, // and an empty selection reaches NOTHING_VERIFIED on its own. - if (!any_ran && declined) - throw UnsupportedTestFeature{*declined}; + if (!any_ran && !declined.empty()) + throw UnsupportedTestFeature{declined.front().second}; + + // The file ran, so its own verdict says nothing about what it declined. Name those, or a + // fixture this tool stops running disappears from a passing run rather than being missed. + for (const auto& [name, reason] : declined) + std::cerr << path.string() << "::" << name << ": " << reason << '\n'; } void run_fixture(const std::string& name, const json::json& fixture, const TestSettings& settings, From 2c9e2c42875593b9aca4a88b15b4e61228a0f9cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Thu, 3 Sep 2026 13:38:20 +0200 Subject: [PATCH 12/19] tests review --- test/integration/CMakeLists.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index 8abcba389e..d5ff0ef140 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -54,6 +54,7 @@ if(TARGET evmone-cli) ${PREFIX}/version PROPERTIES PASS_REGULAR_EXPRESSION "evmone") # Test loading code from a file via @file syntax. + # TODO: Better to check the code.hex in. file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/code.hex "60 00 80 80 80 80 01 01 01 02 00") add_test(NAME ${PREFIX}/file_input COMMAND evmone-cli run @${CMAKE_CURRENT_BINARY_DIR}/code.hex) set_tests_properties( @@ -84,16 +85,15 @@ DUP1,4 set(DATA ${CMAKE_CURRENT_SOURCE_DIR}) - # One command runs both fixture formats, deciding per test case which one it is: the state - # test and the last one by shape, the blockchain test by the format its "_info" declares. - add_test(NAME ${PREFIX}/test_both_formats COMMAND evmone-cli test + # One command runs both fixture formats, deciding the fixture format on the go: + # file 2 has _info.fixture-type, others are recognized by field names. + add_test(NAME ${PREFIX}/test_mixed_formats COMMAND evmone-cli test ${DATA}/statetest/tests1/SuiteA/test1.json ${DATA}/blockchaintest/eip7778_block_gas.json ${DATA}/blockchaintest/unrecovered_sender_blob_gas.json) set_tests_properties( - ${PREFIX}/test_both_formats PROPERTIES PASS_REGULAR_EXPRESSION "= 3 passed in") + ${PREFIX}/test_mixed_formats PROPERTIES PASS_REGULAR_EXPRESSION "= 3 passed in") - # JSON which is not a test at all, as fixture directories hold beside their fixtures, is - # skipped whole rather than guessed at, even named on its own. + # A file recognized as "not a test" is skipped. add_test(NAME ${PREFIX}/test_not_a_test COMMAND evmone-cli test ${DATA}/statetest/tests1/SuiteA/index.json) set_tests_properties( From b1415d441b001d37fde049f3a4e5b595165c89a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Thu, 3 Sep 2026 13:40:37 +0200 Subject: [PATCH 13/19] test: Keep every faulty fixture in one file testcmd_cases and testcmd_fault each held a single file and pinned the same verdict: a directory whose one test fails. Merging them leaves one file whose four fixtures run, fault, throw and fail in turn, so the directory tests and the named-file tests share it. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/integration/CMakeLists.txt | 21 +-- .../testcmd_cases/case_after_exception.json | 70 --------- test/integration/testcmd_fault/faults.json | 135 ++++++++++++++++++ .../testcmd_fault/unrecognised_case.json | 67 --------- 4 files changed, 146 insertions(+), 147 deletions(-) delete mode 100644 test/integration/testcmd_cases/case_after_exception.json create mode 100644 test/integration/testcmd_fault/faults.json delete mode 100644 test/integration/testcmd_fault/unrecognised_case.json diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index d5ff0ef140..216d2003d4 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -133,14 +133,15 @@ DUP1,4 PASS_REGULAR_EXPRESSION "\"pass\":true" FAIL_REGULAR_EXPRESSION "failing_test_case") - # A case whose load throws must not abandon the cases after it, so both faults are reported. - add_test(NAME ${PREFIX}/test_after_exception COMMAND evmone-cli test ${DATA}/testcmd_cases) + # A fixture whose load throws must not abandon the fixtures after it, so all three faults + # are reported. + add_test(NAME ${PREFIX}/test_after_exception COMMAND evmone-cli test ${DATA}/testcmd_fault) set_tests_properties( ${PREFIX}/test_after_exception PROPERTIES # "collected 1 test" keeps this on the directory form, where the whole file is one test - # and the per-case catch is what keeps the second fault reachable. + # and the per-fixture catch is what keeps the later faults reachable. PASS_REGULAR_EXPRESSION - "collected 1 test.*a_load_error:\n exception.*b_bad_root:.*state root") + "collected 1 test.*c_load_error:\n exception.*d_bad_root:.*state root") # Collected from a directory a file is one test, so a file with nothing to run is skipped # whole, named by the first reason which explains it: all_unsupported declines twice, and @@ -155,17 +156,17 @@ DUP1,4 add_test(NAME ${PREFIX}/test_dir_skipped_exit COMMAND evmone-cli test ${DATA}/testcmd_skipped) set_tests_properties(${PREFIX}/test_dir_skipped_exit PROPERTIES WILL_FAIL TRUE) - # Naming the file instead of the directory makes each case its own test, and a fault in one - # is still a fault: the case beside it runs. + # Naming the file instead of the directory makes each fixture its own test, so a fault leaves + # the fixtures beside it running. add_test(NAME ${PREFIX}/test_file_fault COMMAND evmone-cli test - ${DATA}/testcmd_fault/unrecognised_case.json) + ${DATA}/testcmd_fault/faults.json) set_tests_properties( ${PREFIX}/test_file_fault PROPERTIES PASS_REGULAR_EXPRESSION - "FAILED[^\n]*unrecognised_case\\.json::b_not_a_test.*1 failed, 1 passed in") + "FAILED[^\n]*faults\\.json::b_not_a_test.*3 failed, 1 passed in") - # -k selects among the cases of a named file too. + # -k selects among the fixtures of a named file too. add_test(NAME ${PREFIX}/test_file_filter COMMAND evmone-cli test - ${DATA}/testcmd_fault/unrecognised_case.json -k a_runs) + ${DATA}/testcmd_fault/faults.json -k a_runs) set_tests_properties( ${PREFIX}/test_file_filter PROPERTIES PASS_REGULAR_EXPRESSION "= 1 passed in") diff --git a/test/integration/testcmd_cases/case_after_exception.json b/test/integration/testcmd_cases/case_after_exception.json deleted file mode 100644 index daac56ce62..0000000000 --- a/test/integration/testcmd_cases/case_after_exception.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "a_load_error": { - "_info": { - "fixture-format": "state_test", - "comment": "A state test with no pre state, so loading it throws." - } - }, - "b_bad_root": { - "_info": { - "comment": "Runs and fails on the state root. Only reported if the case before it did not abandon the file." - }, - "env": { - "currentBaseFee": "0x0a", - "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", - "currentDifficulty": "0x020000", - "currentGasLimit": "0xff112233445566", - "currentNumber": "0x01", - "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000", - "currentTimestamp": "0x03e8" - }, - "post": { - "London": [ - { - "hash": "0x1111111111111111111111111111111111111111111111111111111111111111", - "indexes": { - "data": 0, - "gas": 0, - "value": 0 - }, - "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" - } - ] - }, - "pre": { - "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { - "balance": "0x0de0b6b3a7640000", - "code": "0x600160010160005500", - "nonce": "0x00", - "storage": {} - }, - "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { - "balance": "0x00", - "code": "0x", - "nonce": "0x01", - "storage": {} - }, - "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { - "balance": "0x0de0b6b3a7640000", - "code": "0x", - "nonce": "0x00", - "storage": {} - } - }, - "transaction": { - "data": [ - "0x" - ], - "gasLimit": [ - "0x061a80" - ], - "gasPrice": "0x0a", - "nonce": "0x00", - "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", - "value": [ - "0x0186a0" - ] - } - } -} diff --git a/test/integration/testcmd_fault/faults.json b/test/integration/testcmd_fault/faults.json new file mode 100644 index 0000000000..5833bd3cee --- /dev/null +++ b/test/integration/testcmd_fault/faults.json @@ -0,0 +1,135 @@ +{ + "a_runs": { + "_info": {}, + "env": { + "currentBaseFee": "0x0a", + "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentDifficulty": "0x020000", + "currentGasLimit": "0xff112233445566", + "currentNumber": "0x01", + "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000", + "currentTimestamp": "0x03e8" + }, + "post": { + "London": [ + { + "hash": "0xe8010ce590f401c9d61fef8ab05bea9bcec24281b795e5868809bc4e515aa530", + "indexes": { + "data": 0, + "gas": 0, + "value": 0 + }, + "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" + } + ] + }, + "pre": { + "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { + "balance": "0x0de0b6b3a7640000", + "code": "0x600160010160005500", + "nonce": "0x00", + "storage": {} + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "balance": "0x00", + "code": "0x", + "nonce": "0x01", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x0de0b6b3a7640000", + "code": "0x", + "nonce": "0x00", + "storage": {} + } + }, + "transaction": { + "data": [ + "0x" + ], + "gasLimit": [ + "0x061a80" + ], + "gasPrice": "0x0a", + "nonce": "0x00", + "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", + "value": [ + "0x0186a0" + ] + } + }, + "b_not_a_test": { + "_info": { + "comment": "Not a test: no fixture fields at all. Beside a case which runs, so this is unmistakably a fixture file with one broken case in it." + } + }, + "c_load_error": { + "_info": { + "fixture-format": "state_test", + "comment": "A state test with no pre state, so loading it throws." + } + }, + "d_bad_root": { + "_info": { + "comment": "Runs and fails on the state root, reached only because the throw above did not abandon the file." + }, + "env": { + "currentBaseFee": "0x0a", + "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentDifficulty": "0x020000", + "currentGasLimit": "0xff112233445566", + "currentNumber": "0x01", + "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000", + "currentTimestamp": "0x03e8" + }, + "post": { + "London": [ + { + "hash": "0x1111111111111111111111111111111111111111111111111111111111111111", + "indexes": { + "data": 0, + "gas": 0, + "value": 0 + }, + "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" + } + ] + }, + "pre": { + "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { + "balance": "0x0de0b6b3a7640000", + "code": "0x600160010160005500", + "nonce": "0x00", + "storage": {} + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "balance": "0x00", + "code": "0x", + "nonce": "0x01", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x0de0b6b3a7640000", + "code": "0x", + "nonce": "0x00", + "storage": {} + } + }, + "transaction": { + "data": [ + "0x" + ], + "gasLimit": [ + "0x061a80" + ], + "gasPrice": "0x0a", + "nonce": "0x00", + "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", + "value": [ + "0x0186a0" + ] + } + } +} diff --git a/test/integration/testcmd_fault/unrecognised_case.json b/test/integration/testcmd_fault/unrecognised_case.json deleted file mode 100644 index 2e3a02ea41..0000000000 --- a/test/integration/testcmd_fault/unrecognised_case.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "a_runs": { - "_info": {}, - "env": { - "currentBaseFee": "0x0a", - "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", - "currentDifficulty": "0x020000", - "currentGasLimit": "0xff112233445566", - "currentNumber": "0x01", - "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000", - "currentTimestamp": "0x03e8" - }, - "post": { - "London": [ - { - "hash": "0xe8010ce590f401c9d61fef8ab05bea9bcec24281b795e5868809bc4e515aa530", - "indexes": { - "data": 0, - "gas": 0, - "value": 0 - }, - "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" - } - ] - }, - "pre": { - "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { - "balance": "0x0de0b6b3a7640000", - "code": "0x600160010160005500", - "nonce": "0x00", - "storage": {} - }, - "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { - "balance": "0x00", - "code": "0x", - "nonce": "0x01", - "storage": {} - }, - "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { - "balance": "0x0de0b6b3a7640000", - "code": "0x", - "nonce": "0x00", - "storage": {} - } - }, - "transaction": { - "data": [ - "0x" - ], - "gasLimit": [ - "0x061a80" - ], - "gasPrice": "0x0a", - "nonce": "0x00", - "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", - "value": [ - "0x0186a0" - ] - } - }, - "b_not_a_test": { - "_info": { - "comment": "Not a test: no fixture fields at all. Beside a case which runs, so this is unmistakably a fixture file with one broken case in it." - } - } -} From fa713ee305e2e446d77e928bbb25528d92ed1b5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Thu, 3 Sep 2026 13:40:52 +0200 Subject: [PATCH 14/19] test: Read the unparsable file the suite already keeps testcmd_broken held one truncated file to prove a named file which does not parse fails rather than skips. notes.txt, which the statetest suite already names for the same reason, does it without a directory of its own, and pins as well that a named file is read whatever its extension. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/integration/CMakeLists.txt | 7 ++++--- test/integration/testcmd_broken/truncated.json | 3 --- 2 files changed, 4 insertions(+), 6 deletions(-) delete mode 100644 test/integration/testcmd_broken/truncated.json diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index 216d2003d4..fd3925c864 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -193,12 +193,13 @@ DUP1,4 ${PREFIX}/test_trace PROPERTIES PASS_REGULAR_EXPRESSION "test1\\.json::test1\n\\{\"pc\":0,[^\n]*\"opName\":\"PUSH1\"\\}.*= 1 passed in") - # A named file which does not parse names itself, because no case in it can be. + # A named file is read whatever its extension, and one which does not parse names itself, + # because no fixture in it can be. add_test(NAME ${PREFIX}/test_file_unparsed COMMAND evmone-cli test - ${DATA}/testcmd_broken/truncated.json) + ${DATA}/statetest/tests1/SuiteA/notes.txt) set_tests_properties( ${PREFIX}/test_file_unparsed PROPERTIES PASS_REGULAR_EXPRESSION - "FAILED[^\n]*truncated\\.json - exception[^\n]*parse error.*1 failed, 0 passed in") + "FAILED[^\n]*notes\\.txt - exception[^\n]*parse error.*1 failed, 0 passed in") endif() diff --git a/test/integration/testcmd_broken/truncated.json b/test/integration/testcmd_broken/truncated.json deleted file mode 100644 index 0c72185851..0000000000 --- a/test/integration/testcmd_broken/truncated.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "a_case": { - "_info": {"comment": "The file ends here, mid-object, so it does not parse at all."} From 1320ac0a826bdb52848e809b5eab40ee84efb03c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Thu, 3 Sep 2026 13:41:10 +0200 Subject: [PATCH 15/19] test: Drop what a declining fixture never reads The blockchain loader refuses a_bad_rlp inside the block loop, so its postState, lastblockhash and the optional header fields were never read by anything. Its file is renamed for what it holds: three fixtures, two of which this tool declines. The three exit-code tests move together, so the note that a PASS_REGULAR_EXPRESSION makes CTest ignore the exit code is stated once rather than beside each pair. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/integration/CMakeLists.txt | 76 +++++++++---------- ...ne_unsupported_case.json => declined.json} | 24 +----- 2 files changed, 38 insertions(+), 62 deletions(-) rename test/integration/testcmd/{one_unsupported_case.json => declined.json} (83%) diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index fd3925c864..7e743be62e 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -86,7 +86,7 @@ DUP1,4 set(DATA ${CMAKE_CURRENT_SOURCE_DIR}) # One command runs both fixture formats, deciding the fixture format on the go: - # file 2 has _info.fixture-type, others are recognized by field names. + # file 2 has _info.fixture-format, others are recognized by field names. add_test(NAME ${PREFIX}/test_mixed_formats COMMAND evmone-cli test ${DATA}/statetest/tests1/SuiteA/test1.json ${DATA}/blockchaintest/eip7778_block_gas.json ${DATA}/blockchaintest/unrecovered_sender_blob_gas.json) @@ -100,32 +100,42 @@ DUP1,4 ${PREFIX}/test_not_a_test PROPERTIES PASS_REGULAR_EXPRESSION "SKIPPED[^\n]*index\\.json - not a test.*0 passed, 1 skipped in") - # Over a directory a test is a whole file, so a case the loader refuses must not take the - # rest of the file with it: the file passes on the case beside it, the refused one first. - # The file's own verdict says nothing about what it declined, so each declined case is named. - add_test(NAME ${PREFIX}/test_dir_declined COMMAND evmone-cli test ${DATA}/testcmd) - set_tests_properties( - ${PREFIX}/test_dir_declined PROPERTIES PASS_REGULAR_EXPRESSION - "a_bad_rlp: tests with invalidly rlp-encoded blocks.*c_engine: unsupported fixture format.*= 1 passed in") - - # A PASS_REGULAR_EXPRESSION makes CTest ignore the exit code, so these two assert on nothing - # else: a sound fixture directory succeeds, and a fault in one fails. + # A PASS_REGULAR_EXPRESSION makes CTest ignore the exit code, so these three assert on nothing + # else: a sound directory succeeds, one holding a fault fails, and so does one with nothing to + # run. Every test below pins what distinguishes them, which the exit code alone does not. add_test(NAME ${PREFIX}/test_exit_ok COMMAND evmone-cli test ${DATA}/testcmd) add_test(NAME ${PREFIX}/test_exit_fail COMMAND evmone-cli test ${DATA}/testcmd_fault) set_tests_properties(${PREFIX}/test_exit_fail PROPERTIES WILL_FAIL TRUE) - # WILL_FAIL above accepts any non-zero exit, and a fault downgraded to a skip exits non-zero - # too, so the counts are what distinguish the two. Whether a file is a fixture file is a - # property of the file, not of what -k selected: the case which is not a test still faults. + add_test(NAME ${PREFIX}/test_exit_skipped COMMAND evmone-cli test ${DATA}/testcmd_skipped) + set_tests_properties(${PREFIX}/test_exit_skipped PROPERTIES WILL_FAIL TRUE) + + # Over a directory a test is a whole file, so a fixture the loader refuses must not take the + # rest of the file with it: the file passes on the one fixture beside the two it declined. + # Its own verdict says nothing about those, so each is named. + add_test(NAME ${PREFIX}/test_dir_declined COMMAND evmone-cli test ${DATA}/testcmd) + set_tests_properties( + ${PREFIX}/test_dir_declined PROPERTIES PASS_REGULAR_EXPRESSION + "a_bad_rlp: tests with invalidly rlp-encoded blocks.*c_engine: unsupported fixture format.*= 1 passed in") + + # Whether a file holds fixtures is a property of the file, not of what -k selected: the + # fixture which is not a test faults even when it is the only one left. add_test(NAME ${PREFIX}/test_dir_fault COMMAND evmone-cli test ${DATA}/testcmd_fault -k b_not_a_test) set_tests_properties( ${PREFIX}/test_dir_fault PROPERTIES PASS_REGULAR_EXPRESSION "1 failed, 0 passed") - # Over a directory -k selects within the file's test, so the summary is what proves a case - # ran: a count alone would hold just as well if the filter had dropped every case. The - # summary appears only with --trace-summary, which nothing else here asks for. + # A fixture whose load throws must not abandon the fixtures after it. "collected 1 test" keeps + # this on the directory form, where the whole file is one test and the per-fixture catch is + # what keeps the later faults reachable. + add_test(NAME ${PREFIX}/test_after_exception COMMAND evmone-cli test ${DATA}/testcmd_fault) + set_tests_properties( + ${PREFIX}/test_after_exception PROPERTIES PASS_REGULAR_EXPRESSION + "collected 1 test.*c_load_error:\n exception.*d_bad_root:.*state root") + + # Over a directory -k selects within the file's test, so the summary is what proves a fixture + # ran: a count alone would hold just as well if the filter had dropped every one of them. add_test(NAME ${PREFIX}/test_dir_filter COMMAND evmone-cli test ${DATA}/statetest/filter -k passing_test_case --trace-summary) set_tests_properties( @@ -133,53 +143,37 @@ DUP1,4 PASS_REGULAR_EXPRESSION "\"pass\":true" FAIL_REGULAR_EXPRESSION "failing_test_case") - # A fixture whose load throws must not abandon the fixtures after it, so all three faults - # are reported. - add_test(NAME ${PREFIX}/test_after_exception COMMAND evmone-cli test ${DATA}/testcmd_fault) - set_tests_properties( - ${PREFIX}/test_after_exception PROPERTIES - # "collected 1 test" keeps this on the directory form, where the whole file is one test - # and the per-fixture catch is what keeps the later faults reachable. - PASS_REGULAR_EXPRESSION - "collected 1 test.*c_load_error:\n exception.*d_bad_root:.*state root") - # Collected from a directory a file is one test, so a file with nothing to run is skipped - # whole, named by the first reason which explains it: all_unsupported declines twice, and - # the format named is the first one's. + # whole, named by the first reason which explains it: all_unsupported declines twice, and the + # format named is the first one's. add_test(NAME ${PREFIX}/test_dir_skipped COMMAND evmone-cli test ${DATA}/testcmd_skipped) set_tests_properties( - ${PREFIX}/test_dir_skipped PROPERTIES - PASS_REGULAR_EXPRESSION + ${PREFIX}/test_dir_skipped PROPERTIES PASS_REGULAR_EXPRESSION "all_unsupported\\.json - unsupported fixture format: \"blockchain_test_engine\"\nSKIPPED[^\n]*empty\\.json - not a test.*pre_alloc\\.json - not a test.*0 passed, 5 skipped in") - # Nothing to run is not nothing to report: the run fails rather than passing empty. - add_test(NAME ${PREFIX}/test_dir_skipped_exit COMMAND evmone-cli test ${DATA}/testcmd_skipped) - set_tests_properties(${PREFIX}/test_dir_skipped_exit PROPERTIES WILL_FAIL TRUE) - # Naming the file instead of the directory makes each fixture its own test, so a fault leaves - # the fixtures beside it running. + # the fixtures beside it running, and -k selects among them. add_test(NAME ${PREFIX}/test_file_fault COMMAND evmone-cli test ${DATA}/testcmd_fault/faults.json) set_tests_properties( ${PREFIX}/test_file_fault PROPERTIES PASS_REGULAR_EXPRESSION "FAILED[^\n]*faults\\.json::b_not_a_test.*3 failed, 1 passed in") - # -k selects among the fixtures of a named file too. add_test(NAME ${PREFIX}/test_file_filter COMMAND evmone-cli test ${DATA}/testcmd_fault/faults.json -k a_runs) set_tests_properties( ${PREFIX}/test_file_filter PROPERTIES PASS_REGULAR_EXPRESSION "= 1 passed in") - # Naming a file whose cases this tool does not run skips them one by one, where collecting + # Naming a file whose fixtures this tool does not run skips them one by one, where collecting # the same file from a directory would skip it whole. add_test(NAME ${PREFIX}/test_file_declined COMMAND evmone-cli test - ${DATA}/testcmd/one_unsupported_case.json) + ${DATA}/testcmd/declined.json) set_tests_properties( ${PREFIX}/test_file_declined PROPERTIES PASS_REGULAR_EXPRESSION "SKIPPED[^\n]*c_engine - unsupported fixture format.*= 1 passed, 2 skipped in") - # --trace and --histogram reach the runner through the app-level flags, and turn the progress - # row into a line naming each test so what each writes can be told apart. + # --trace and --histogram reach the runner through the app-level flags, and each turns the + # progress row into a line naming the test so what it writes can be told apart. add_test(NAME ${PREFIX}/test_histogram COMMAND evmone-cli test --histogram ${DATA}/statetest/tests1/SuiteA/test1.json) set_tests_properties( diff --git a/test/integration/testcmd/one_unsupported_case.json b/test/integration/testcmd/declined.json similarity index 83% rename from test/integration/testcmd/one_unsupported_case.json rename to test/integration/testcmd/declined.json index 35eea677ad..8995f5d82a 100644 --- a/test/integration/testcmd/one_unsupported_case.json +++ b/test/integration/testcmd/declined.json @@ -1,7 +1,7 @@ { "a_bad_rlp": { "_info": { - "comment": "Declines to load: expectException without rlp_decoded. Nothing of this case runs, so the case beside it is what the file reports." + "comment": "Declines to load: expectException without rlp_decoded, which the loader refuses before reading anything after the blocks." }, "network": "Cancun", "genesisBlockHeader": { @@ -16,33 +16,15 @@ "gasUsed": "0x00", "timestamp": "0x00", "extraData": "0x00", - "baseFeePerGas": "0x10", - "withdrawalsRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "blobGasUsed": "0x00", - "excessBlobGas": "0x00", "hash": "0x0000000000000000000000000000000000000000000000000000000000000001" }, - "pre": { - "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { - "code": "0x", - "nonce": "0x00", - "balance": "0x02540be400" - } - }, + "pre": {}, "blocks": [ { "expectException": "TransactionException.INVALID_SIGNATURE_VRS", "rlp": "0x" } - ], - "lastblockhash": "0x0000000000000000000000000000000000000000000000000000000000000001", - "postState": { - "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { - "code": "0x", - "nonce": "0x00", - "balance": "0x02540be400" - } - } + ] }, "b_state": { "_info": { From 077666bf2dcd3d7a4f43d535a2af7684eab6d29c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Thu, 3 Sep 2026 14:54:45 +0200 Subject: [PATCH 16/19] test: Make a fixture file the only kind of test Naming a file collected one test per fixture in it, which naming a directory could not afford: enumerating fixtures means loading every file, and a release is gigabytes. The two modes reported different counts, filtered at different times and skipped at different granularity. A file is now one test either way. Collection loads nothing, so it can no longer fail, and the three tests which pinned the difference pin nothing: naming faults.json and naming its directory now produce byte-identical output. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/integration/CMakeLists.txt | 25 +--------- test/utils/test_collector.cpp | 58 +++++------------------ test/utils/test_collector.hpp | 8 ++-- test/utils/test_driver.cpp | 83 +++++++++++++++++---------------- test/utils/test_driver.hpp | 8 ---- tools/evmone/main.cpp | 12 ++--- 6 files changed, 66 insertions(+), 128 deletions(-) diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index 7e743be62e..8c48df4087 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -151,41 +151,20 @@ DUP1,4 ${PREFIX}/test_dir_skipped PROPERTIES PASS_REGULAR_EXPRESSION "all_unsupported\\.json - unsupported fixture format: \"blockchain_test_engine\"\nSKIPPED[^\n]*empty\\.json - not a test.*pre_alloc\\.json - not a test.*0 passed, 5 skipped in") - # Naming the file instead of the directory makes each fixture its own test, so a fault leaves - # the fixtures beside it running, and -k selects among them. - add_test(NAME ${PREFIX}/test_file_fault COMMAND evmone-cli test - ${DATA}/testcmd_fault/faults.json) - set_tests_properties( - ${PREFIX}/test_file_fault PROPERTIES PASS_REGULAR_EXPRESSION - "FAILED[^\n]*faults\\.json::b_not_a_test.*3 failed, 1 passed in") - - add_test(NAME ${PREFIX}/test_file_filter COMMAND evmone-cli test - ${DATA}/testcmd_fault/faults.json -k a_runs) - set_tests_properties( - ${PREFIX}/test_file_filter PROPERTIES PASS_REGULAR_EXPRESSION "= 1 passed in") - - # Naming a file whose fixtures this tool does not run skips them one by one, where collecting - # the same file from a directory would skip it whole. - add_test(NAME ${PREFIX}/test_file_declined COMMAND evmone-cli test - ${DATA}/testcmd/declined.json) - set_tests_properties( - ${PREFIX}/test_file_declined PROPERTIES PASS_REGULAR_EXPRESSION - "SKIPPED[^\n]*c_engine - unsupported fixture format.*= 1 passed, 2 skipped in") - # --trace and --histogram reach the runner through the app-level flags, and each turns the # progress row into a line naming the test so what it writes can be told apart. add_test(NAME ${PREFIX}/test_histogram COMMAND evmone-cli test --histogram ${DATA}/statetest/tests1/SuiteA/test1.json) set_tests_properties( ${PREFIX}/test_histogram PROPERTIES PASS_REGULAR_EXPRESSION - "test1\\.json::test1 + "test1\\.json --- # HISTOGRAM.*= 1 passed in") add_test(NAME ${PREFIX}/test_trace COMMAND evmone-cli test --trace ${DATA}/statetest/tests1/SuiteA/test1.json) set_tests_properties( ${PREFIX}/test_trace PROPERTIES PASS_REGULAR_EXPRESSION - "test1\\.json::test1\n\\{\"pc\":0,[^\n]*\"opName\":\"PUSH1\"\\}.*= 1 passed in") + "test1\\.json\n\\{\"pc\":0,[^\n]*\"opName\":\"PUSH1\"\\}.*= 1 passed in") # A named file is read whatever its extension, and one which does not parse names itself, # because no fixture in it can be. diff --git a/test/utils/test_collector.cpp b/test/utils/test_collector.cpp index 36f3e25cc5..6eb8aabafb 100644 --- a/test/utils/test_collector.cpp +++ b/test/utils/test_collector.cpp @@ -4,8 +4,6 @@ #include "test_collector.hpp" #include -#include -#include #include namespace evmone::test @@ -57,60 +55,28 @@ void ignore_test_files(std::vector& files, std::span i }); } -bool collect_tests( +void collect_tests( std::vector& cases, const fs::path& root, const TestSettings& settings, evmc::VM& vm) { + // A file is one test, whether it was named or found under a directory. Naming the fixtures + // in it instead would mean loading every file to collect, which a whole tree cannot afford. + std::vector files; if (is_directory(root)) { - auto files = collect_test_files(root); + files = collect_test_files(root); ignore_test_files(files, settings.ignored); - cases.reserve(cases.size() + files.size()); - for (const auto& file : files) - { - // Loaded when the test runs: loading a whole tree up front costs far more. - cases.push_back( - {file.path.string(), [path = file.path, &settings, &vm](TestReport& report) { - run_fixture_file(path, settings, vm, report); - }}); - } - return true; - } - - // Naming a file loads it now, to name the fixtures in it. One which cannot be loaded, or - // which is not a test, becomes a single test reporting why. - json::json file; - try - { - file = load_fixture_file(root); - } - catch (const UnsupportedTestFeature&) - { - // A skip, as when collected from a directory, not a broken collection. - cases.push_back({root.string(), - [error = std::current_exception()](auto&) { std::rethrow_exception(error); }}); - return true; - } - catch (const std::exception& ex) - { - // Also reported here: --collect-only never runs the test. - std::cerr << root.string() << ": " << ex.what() << '\n'; - cases.push_back({root.string(), - [error = std::current_exception()](auto&) { std::rethrow_exception(error); }}); - return false; } + else + files.push_back({root, {}}); - // One document shared by every test of it. Capturing the fixture itself would copy its - // subtree into each test, which a listing pays for in full to read nothing but the name. - const auto doc = std::make_shared(std::move(file)); - for (const auto& [name, fixture] : doc->items()) + cases.reserve(cases.size() + files.size()); + for (const auto& file : files) { - if (!settings.selects(name)) - continue; + // Loaded when the test runs: loading a whole tree up front costs far more. cases.push_back( - {root.string() + "::" + name, [doc, name = name, &settings, &vm](TestReport& report) { - run_fixture(name, doc->at(name), settings, vm, report); + {file.path.string(), [path = file.path, &settings, &vm](TestReport& report) { + run_fixture_file(path, settings, vm, report); }}); } - return true; } } // namespace evmone::test diff --git a/test/utils/test_collector.hpp b/test/utils/test_collector.hpp index 72abe1c132..55b573d33e 100644 --- a/test/utils/test_collector.hpp +++ b/test/utils/test_collector.hpp @@ -34,9 +34,9 @@ struct TestFile void ignore_test_files( std::vector& files, std::span ignored); -/// Adds to @p cases every test under @p root: one per file for a directory, one per fixture for -/// a named file, or one for a named file holding no fixture. Returns whether every test was -/// collected. The tests hold @p settings and @p vm by reference, so both must outlive them. -[[nodiscard]] bool collect_tests(std::vector& cases, const std::filesystem::path& root, +/// Adds to @p cases one test per fixture file under @p root, which is that file itself when it +/// is not a directory. The tests hold @p settings and @p vm by reference, so both must outlive +/// them. +void collect_tests(std::vector& cases, const std::filesystem::path& root, const TestSettings& settings, evmc::VM& vm); } // namespace evmone::test diff --git a/test/utils/test_driver.cpp b/test/utils/test_driver.cpp index 6753451f34..f4b6088609 100644 --- a/test/utils/test_driver.cpp +++ b/test/utils/test_driver.cpp @@ -105,6 +105,50 @@ Format classify(const json::json& fixture) return Format::not_a_test; } +/// Parses the fixture file at @p path. Throws UnsupportedTestFeature for a file with no fixture +/// in it, which is nothing to run: fixture directories hold other JSON beside the fixtures. +json::json load_fixture_file(const fs::path& path) +{ + std::ifstream f{path}; + // A stream which never opened reads as EOF, which parses as a syntax error in a file which + // has none. + if (!f) + throw std::runtime_error{"cannot open the file"}; + const auto contents = json::json::parse(f); + // Not one fixture in it: EEST keeps its shared pre-allocation and an index of the fixtures + // beside them, and neither is a test. Nor is a document which is not an object at all: + // items() would walk an array by index, naming its elements "0", "1", ... + if (!contents.is_object() || std::ranges::none_of(contents.items(), [](const auto& i) { + return classify(i.value()) != Format::not_a_test; + })) + throw UnsupportedTestFeature{"not a test"}; + return contents; +} + +/// Runs one fixture of a fixture file. One this tool does not recognise is a fault in the file; +/// one in a format it does not run is skipped. +void run_fixture(const std::string& name, const json::json& fixture, const TestSettings& settings, + evmc::VM& vm, TestReport& report) +{ + report.start_case(name); // Names whatever the load itself reports. + switch (classify(fixture)) + { + case Format::state_test: + run_state_test(make_state_test(name, fixture), vm, settings.trace_summary, report); + break; + case Format::blockchain_test: + run_blockchain_test(make_blockchain_test(name, fixture), vm, report); + break; + case Format::unsupported: + throw UnsupportedTestFeature{ + "unsupported fixture format: " + fixture.at("_info").at("fixture-format").dump()}; + case Format::not_a_test: + // The rest of the file holds fixtures, so this one is broken. + report.fail("not a test"); + break; + } +} + } // namespace int run_tests(std::span cases, std::ostream& out, const RunOptions& options) @@ -226,24 +270,6 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio return passed == 0 ? NOTHING_VERIFIED : SUCCESS; } -json::json load_fixture_file(const fs::path& path) -{ - std::ifstream f{path}; - // A stream which never opened reads as EOF, which parses as a syntax error in a file which - // has none. - if (!f) - throw std::runtime_error{"cannot open the file"}; - const auto contents = json::json::parse(f); - // Not one fixture in it: EEST keeps its shared pre-allocation and an index of the fixtures - // beside them, and neither is a test. Nor is a document which is not an object at all: - // items() would walk an array by index, naming its elements "0", "1", ... - if (!contents.is_object() || std::ranges::none_of(contents.items(), [](const auto& i) { - return classify(i.value()) != Format::not_a_test; - })) - throw UnsupportedTestFeature{"not a test"}; - return contents; -} - void run_fixture_file( const fs::path& path, const TestSettings& settings, evmc::VM& vm, TestReport& report) { @@ -293,25 +319,4 @@ void run_fixture_file( std::cerr << path.string() << "::" << name << ": " << reason << '\n'; } -void run_fixture(const std::string& name, const json::json& fixture, const TestSettings& settings, - evmc::VM& vm, TestReport& report) -{ - report.start_case(name); // Names whatever the load itself reports. - switch (classify(fixture)) - { - case Format::state_test: - run_state_test(make_state_test(name, fixture), vm, settings.trace_summary, report); - break; - case Format::blockchain_test: - run_blockchain_test(make_blockchain_test(name, fixture), vm, report); - break; - case Format::unsupported: - throw UnsupportedTestFeature{ - "unsupported fixture format: " + fixture.at("_info").at("fixture-format").dump()}; - case Format::not_a_test: - // The rest of the file holds fixtures, so this one is broken. - report.fail("not a test"); - break; - } -} } // namespace evmone::test diff --git a/test/utils/test_driver.hpp b/test/utils/test_driver.hpp index 27cd9fce65..edfb9e8368 100644 --- a/test/utils/test_driver.hpp +++ b/test/utils/test_driver.hpp @@ -69,17 +69,9 @@ struct TestSettings } }; -/// Parses the fixture file at @p path. Throws UnsupportedTestFeature for a file with no fixture -/// in it, which is nothing to run: fixture directories hold other JSON beside the fixtures. -[[nodiscard]] json::json load_fixture_file(const std::filesystem::path& path); - /// Runs every selected fixture of one fixture file, which together are one test. Throws /// UnsupportedTestFeature for a file this tool has nothing to run in. void run_fixture_file(const std::filesystem::path& path, const TestSettings& settings, evmc::VM& vm, TestReport& report); -/// Runs one fixture of a fixture file. One this tool does not recognise is a fault in the file; -/// one in a format it does not run is skipped. -void run_fixture(const std::string& name, const json::json& fixture, const TestSettings& settings, - evmc::VM& vm, TestReport& report); } // namespace evmone::test diff --git a/tools/evmone/main.cpp b/tools/evmone/main.cpp index 2dfafc0fe9..896e06e622 100644 --- a/tools/evmone/main.cpp +++ b/tools/evmone/main.cpp @@ -165,9 +165,8 @@ const CLI::App& setup_test_cmd(CLI::App& app, TestOptions& opts) { auto& cmd = *app.add_subcommand("test", "Run Ethereum tests")->fallthrough(); cmd.add_option("path", opts.paths, - "Test file or directory. Under a directory every .json file except index.json is " - "one test. A named file gives one test per fixture, or one for the file when it holds " - "no fixture.") + "Test file or directory. Every fixture file is one test: under a directory, each " + ".json file except index.json.") ->required() ->check(CLI::ExistingPath); cmd.add_option( @@ -196,13 +195,10 @@ int exec_test_cmd(evmc::VM& vm, TestOptions opts, bool trace, bool histogram) opts.run.progress = !(opts.settings.trace_summary || histogram); std::vector cases; - bool all_collected = true; for (const auto& p : opts.paths) - all_collected &= collect_tests(cases, p, opts.settings, vm); + collect_tests(cases, p, opts.settings, vm); - const auto exit_code = evmone::test::run_tests(cases, std::cout, opts.run); - // A file which could not be loaded fails the listing too, not only a run of it. - return all_collected ? exit_code : evmone::test::TESTS_FAILED; + return evmone::test::run_tests(cases, std::cout, opts.run); } } // namespace From 7c96b2bd6fd182a8dd627fdb21ef8c50bf6c6007 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Thu, 3 Sep 2026 15:03:21 +0200 Subject: [PATCH 17/19] test: Report a file's fixtures one result each A file was one test recording one outcome, so what it declined had nowhere to go but a warning on stderr: a fixture this tool stopped running vanished from a run which still passed on the fixtures beside it. Running a test now returns one result per fixture, and a file which never gets as far as a fixture returns the one result which says so. Collection and the counts stay per file, which is the only unit known without parsing the tree; the results name every fixture which did not pass. Claude-Session: https://claude.ai/code/session_016UHPAGwcwXMjqhTLpT31K7 --- test/blockchaintest/blockchaintest.cpp | 35 +++-- test/integration/CMakeLists.txt | 22 ++- test/statetest/statetest.cpp | 19 ++- test/unittests/test_driver_test.cpp | 92 +++++++++--- test/utils/test_collector.cpp | 6 +- test/utils/test_driver.cpp | 194 +++++++++++++------------ test/utils/test_driver.hpp | 39 ++++- 7 files changed, 251 insertions(+), 156 deletions(-) diff --git a/test/blockchaintest/blockchaintest.cpp b/test/blockchaintest/blockchaintest.cpp index 8026c70ba8..cd31f4fa4a 100644 --- a/test/blockchaintest/blockchaintest.cpp +++ b/test/blockchaintest/blockchaintest.cpp @@ -17,6 +17,13 @@ namespace { /// Adds to @p cases every test under @p root: one per file for a directory, one per test case in /// the file when the file itself is named. Returns whether every test was collected. +/// A test which is one case: the driver takes an array of results, and this is the one. +TestCase one_case(std::string name, std::function run) +{ + return {name, + [name, run = std::move(run)] { return std::vector{evmone::test::run_one(name, run)}; }}; +} + bool collect_tests(std::vector& cases, const fs::path& root, std::span ignored, evmc::VM& vm) { @@ -29,12 +36,12 @@ bool collect_tests(std::vector& cases, const fs::path& root, { // Loaded when the test runs: loading a whole tree up front costs far more. A // load which throws over an unsupported fixture reaches the driver, which skips. - cases.push_back( - {file.path.string(), [path = file.path, &vm](evmone::test::TestReport& report) { - std::ifstream f{path}; - for (const auto& test : evmone::test::load_blockchain_tests(f)) - evmone::test::run_blockchain_test(test, vm, report); - }}); + cases.push_back(one_case( + file.path.string(), [path = file.path, &vm](evmone::test::TestReport& report) { + std::ifstream f{path}; + for (const auto& test : evmone::test::load_blockchain_tests(f)) + evmone::test::run_blockchain_test(test, vm, report); + })); } } else // Treat as a file. @@ -50,25 +57,25 @@ bool collect_tests(std::vector& cases, const fs::path& root, catch (const evmone::test::UnsupportedTestFeature&) { // An unsupported fixture is a skip, not a broken collection. - cases.push_back({root.string(), - [error = std::current_exception()](auto&) { std::rethrow_exception(error); }}); + cases.push_back(one_case(root.string(), + [error = std::current_exception()](auto&) { std::rethrow_exception(error); })); return true; } catch (const std::exception& ex) { // Also reported here: --collect-only never runs the test. std::cerr << root.string() << ": " << ex.what() << '\n'; - cases.push_back({root.string(), - [error = std::current_exception()](auto&) { std::rethrow_exception(error); }}); + cases.push_back(one_case(root.string(), + [error = std::current_exception()](auto&) { std::rethrow_exception(error); })); return false; } for (const auto& test : tests) { - cases.push_back( - {root.string() + "::" + test.name, [test, &vm](evmone::test::TestReport& report) { - evmone::test::run_blockchain_test(test, vm, report); - }}); + cases.push_back(one_case( + root.string() + "::" + test.name, [test, &vm](evmone::test::TestReport& report) { + evmone::test::run_blockchain_test(test, vm, report); + })); } } return true; diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index 8c48df4087..b9448a6120 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -111,13 +111,13 @@ DUP1,4 add_test(NAME ${PREFIX}/test_exit_skipped COMMAND evmone-cli test ${DATA}/testcmd_skipped) set_tests_properties(${PREFIX}/test_exit_skipped PROPERTIES WILL_FAIL TRUE) - # Over a directory a test is a whole file, so a fixture the loader refuses must not take the - # rest of the file with it: the file passes on the one fixture beside the two it declined. - # Its own verdict says nothing about those, so each is named. + # A fixture the loader refuses must not take the rest of the file with it: the file passes on + # the one fixture beside the two it declined. Its own verdict says nothing about those, so + # each is named in the summary, where a count of files alone would lose them. add_test(NAME ${PREFIX}/test_dir_declined COMMAND evmone-cli test ${DATA}/testcmd) set_tests_properties( ${PREFIX}/test_dir_declined PROPERTIES PASS_REGULAR_EXPRESSION - "a_bad_rlp: tests with invalidly rlp-encoded blocks.*c_engine: unsupported fixture format.*= 1 passed in") + "SKIPPED[^\n]*::a_bad_rlp - tests with invalidly rlp-encoded blocks[^\n]*\nSKIPPED[^\n]*::c_engine - unsupported fixture format.*= 1 passed in") # Whether a file holds fixtures is a property of the file, not of what -k selected: the # fixture which is not a test faults even when it is the only one left. @@ -126,13 +126,12 @@ DUP1,4 set_tests_properties( ${PREFIX}/test_dir_fault PROPERTIES PASS_REGULAR_EXPRESSION "1 failed, 0 passed") - # A fixture whose load throws must not abandon the fixtures after it. "collected 1 test" keeps - # this on the directory form, where the whole file is one test and the per-fixture catch is - # what keeps the later faults reachable. + # A fixture whose load throws must not abandon the fixtures after it, and the file counts + # once for all three of them. add_test(NAME ${PREFIX}/test_after_exception COMMAND evmone-cli test ${DATA}/testcmd_fault) set_tests_properties( ${PREFIX}/test_after_exception PROPERTIES PASS_REGULAR_EXPRESSION - "collected 1 test.*c_load_error:\n exception.*d_bad_root:.*state root") + "collected 1 file\n.*c_load_error:\n exception.*d_bad_root:.*state root.*1 failed, 0 passed in") # Over a directory -k selects within the file's test, so the summary is what proves a fixture # ran: a count alone would hold just as well if the filter had dropped every one of them. @@ -143,13 +142,12 @@ DUP1,4 PASS_REGULAR_EXPRESSION "\"pass\":true" FAIL_REGULAR_EXPRESSION "failing_test_case") - # Collected from a directory a file is one test, so a file with nothing to run is skipped - # whole, named by the first reason which explains it: all_unsupported declines twice, and the - # format named is the first one's. + # A file is one test, so a file with nothing to run counts once as skipped however many + # fixtures it declined: all_unsupported declines twice and both are named. add_test(NAME ${PREFIX}/test_dir_skipped COMMAND evmone-cli test ${DATA}/testcmd_skipped) set_tests_properties( ${PREFIX}/test_dir_skipped PROPERTIES PASS_REGULAR_EXPRESSION - "all_unsupported\\.json - unsupported fixture format: \"blockchain_test_engine\"\nSKIPPED[^\n]*empty\\.json - not a test.*pre_alloc\\.json - not a test.*0 passed, 5 skipped in") + "all_unsupported\\.json::a_engine - unsupported fixture format: \"blockchain_test_engine\"\nSKIPPED[^\n]*::b_engine_x - .*empty\\.json - not a test.*pre_alloc\\.json - not a test.*0 passed, 5 skipped in") # --trace and --histogram reach the runner through the app-level flags, and each turns the # progress row into a line naming the test so what it writes can be told apart. diff --git a/test/statetest/statetest.cpp b/test/statetest/statetest.cpp index 56a747e698..45a9fa1e79 100644 --- a/test/statetest/statetest.cpp +++ b/test/statetest/statetest.cpp @@ -17,6 +17,13 @@ namespace { /// Adds to @p cases every test under @p root: one per file for a directory, one per test case in /// the file when the file itself is named. Returns whether every test was collected. +/// A test which is one case: the driver takes an array of results, and this is the one. +TestCase one_case(std::string name, std::function run) +{ + return {name, + [name, run = std::move(run)] { return std::vector{evmone::test::run_one(name, run)}; }}; +} + bool collect_tests(std::vector& cases, const fs::path& root, const std::optional& filter, std::span ignored, evmc::VM& vm, bool trace) @@ -35,7 +42,7 @@ bool collect_tests(std::vector& cases, const fs::path& root, for (const auto& file : files) { // Loaded when the test runs: loading a whole tree up front costs far more. - cases.push_back({file.path.string(), + cases.push_back(one_case(file.path.string(), [path = file.path, selected, &vm, trace](evmone::test::TestReport& report) { std::ifstream f{path}; for (const auto& test : evmone::test::load_state_tests(f)) @@ -43,7 +50,7 @@ bool collect_tests(std::vector& cases, const fs::path& root, if (selected(test)) evmone::test::run_state_test(test, vm, trace, report); } - }}); + })); } } else // Treat as a file. @@ -60,8 +67,8 @@ bool collect_tests(std::vector& cases, const fs::path& root, { // Also reported here: --collect-only never runs the test. std::cerr << root.string() << ": " << ex.what() << '\n'; - cases.push_back({root.string(), - [error = std::current_exception()](auto&) { std::rethrow_exception(error); }}); + cases.push_back(one_case(root.string(), + [error = std::current_exception()](auto&) { std::rethrow_exception(error); })); return false; } @@ -69,10 +76,10 @@ bool collect_tests(std::vector& cases, const fs::path& root, { if (!selected(test)) continue; - cases.push_back({root.string() + "::" + test.name, + cases.push_back(one_case(root.string() + "::" + test.name, [test, &vm, trace](evmone::test::TestReport& report) { evmone::test::run_state_test(test, vm, trace, report); - }}); + })); } } return true; diff --git a/test/unittests/test_driver_test.cpp b/test/unittests/test_driver_test.cpp index d2208aebd9..d4bc433811 100644 --- a/test/unittests/test_driver_test.cpp +++ b/test/unittests/test_driver_test.cpp @@ -22,6 +22,21 @@ Run run(std::span cases, const RunOptions& options = {}) const auto exit_code = run_tests(cases, out, options); return {exit_code, std::move(out).str()}; } + +/// A test holding one case, which is what a fixture file with one fixture in it comes to. +TestCase one(std::string name, std::function run) +{ + return {name, [name, run = std::move(run)] { return std::vector{run_one(name, run)}; }}; +} + +/// A test holding cases which only report the outcome given, running nothing. +TestCase holding(std::string name, std::initializer_list outcomes) +{ + std::vector results; + for (const auto outcome : outcomes) + results.push_back({name + "::case", outcome, "the reason", {}}); + return {std::move(name), [results = std::move(results)] { return results; }}; +} } // namespace TEST(test_driver, nothing_collected) @@ -29,13 +44,13 @@ TEST(test_driver, nothing_collected) const auto [exit_code, output] = run({}); EXPECT_EQ(NOTHING_VERIFIED, 5); // pytest's value, not just whatever we declared. EXPECT_EQ(exit_code, NOTHING_VERIFIED); - EXPECT_NE(output.find("collected 0 tests"), std::string::npos); + EXPECT_NE(output.find("collected 0 files"), std::string::npos); } TEST(test_driver, collect_only_lists_without_running) { bool ran = false; - const std::vector cases{{"a name", [&ran](TestReport&) { ran = true; }}}; + const std::vector cases{one("a name", [&ran](TestReport&) { ran = true; })}; const auto [exit_code, output] = run(cases, {.collect_only = true}); EXPECT_FALSE(ran); @@ -54,10 +69,10 @@ TEST(test_driver, exception_fails_only_its_own_test) { bool last_ran = false; const std::vector cases{ - {"ok", [](TestReport&) {}}, - {"throws", [](TestReport&) { throw std::runtime_error{"the reason"}; }}, - {"unknown", [](TestReport&) { throw 42; }}, // NOLINT(hicpp-exception-baseclass) - {"last", [&last_ran](TestReport&) { last_ran = true; }}, + one("ok", [](TestReport&) {}), + one("throws", [](TestReport&) { throw std::runtime_error{"the reason"}; }), + one("unknown", [](TestReport&) { throw 42; }), // NOLINT(hicpp-exception-baseclass) + one("last", [&last_ran](TestReport&) { last_ran = true; }), }; const auto [exit_code, output] = run(cases); @@ -71,8 +86,8 @@ TEST(test_driver, exception_fails_only_its_own_test) TEST(test_driver, unsupported_feature_skips) { const std::vector cases{ - {"ok", [](TestReport&) {}}, - {"skipped", [](TestReport&) { throw UnsupportedTestFeature{"no support for it"}; }}, + one("ok", [](TestReport&) {}), + one("skipped", [](TestReport&) { throw UnsupportedTestFeature{"no support for it"}; }), }; const auto [exit_code, output] = run(cases); @@ -84,7 +99,7 @@ TEST(test_driver, unsupported_feature_skips) TEST(test_driver, everything_skipped_verifies_nothing) { const std::vector cases{ - {"skipped", [](TestReport&) { throw UnsupportedTestFeature{"no support for it"}; }}}; + one("skipped", [](TestReport&) { throw UnsupportedTestFeature{"no support for it"}; })}; const auto [exit_code, output] = run(cases); EXPECT_EQ(exit_code, NOTHING_VERIFIED); @@ -94,7 +109,7 @@ TEST(test_driver, everything_skipped_verifies_nothing) TEST(test_driver, summary_names_the_check_which_failed) { const std::vector cases{ - {"mismatch", [](TestReport& report) { report.check_eq("a value", 1, 2); }}}; + one("mismatch", [](TestReport& report) { report.check_eq("a value", 1, 2); })}; const auto [exit_code, output] = run(cases); EXPECT_EQ(exit_code, 1); @@ -103,10 +118,10 @@ TEST(test_driver, summary_names_the_check_which_failed) TEST(test_driver, failure_outranks_a_later_exception) { - const std::vector cases{{"both", [](TestReport& report) { - report.check_eq("a value", 1, 2); - throw std::runtime_error{"gave up afterwards"}; - }}}; + const std::vector cases{one("both", [](TestReport& report) { + report.check_eq("a value", 1, 2); + throw std::runtime_error{"gave up afterwards"}; + })}; const auto [exit_code, output] = run(cases); EXPECT_EQ(exit_code, TESTS_FAILED); @@ -116,10 +131,10 @@ TEST(test_driver, failure_outranks_a_later_exception) TEST(test_driver, failure_outranks_a_later_skip) { - const std::vector cases{{"both", [](TestReport& report) { - report.check_eq("a value", 1, 2); - throw UnsupportedTestFeature{"gave up afterwards"}; - }}}; + const std::vector cases{one("both", [](TestReport& report) { + report.check_eq("a value", 1, 2); + throw UnsupportedTestFeature{"gave up afterwards"}; + })}; const auto [exit_code, output] = run(cases); EXPECT_EQ(exit_code, 1); @@ -127,3 +142,44 @@ TEST(test_driver, failure_outranks_a_later_skip) // The summary names the check which failed, not what the test then gave up on. EXPECT_NE(output.find("FAILED both - a value"), std::string::npos); } + +TEST(test_driver, a_file_counts_once_however_many_fixtures_it_holds) +{ + const std::vector cases{ + holding("a file", {Outcome::passed, Outcome::passed, Outcome::passed})}; + + const auto [exit_code, output] = run(cases); + EXPECT_EQ(exit_code, SUCCESS); + EXPECT_NE(output.find("collected 1 file"), std::string::npos); + EXPECT_NE(output.find("1 passed"), std::string::npos); +} + +TEST(test_driver, a_declined_fixture_is_named_though_its_file_passed) +{ + const std::vector cases{holding("a file", {Outcome::passed, Outcome::skipped})}; + + const auto [exit_code, output] = run(cases); + EXPECT_EQ(exit_code, SUCCESS); + // The file's own verdict says nothing about what it declined, so the fixture is named. + EXPECT_NE(output.find("1 passed in"), std::string::npos); + EXPECT_NE(output.find("SKIPPED a file::case - the reason"), std::string::npos); +} + +TEST(test_driver, one_failed_fixture_fails_its_file) +{ + const std::vector cases{ + holding("a file", {Outcome::passed, Outcome::failed, Outcome::skipped})}; + + const auto [exit_code, output] = run(cases); + EXPECT_EQ(exit_code, TESTS_FAILED); + EXPECT_NE(output.find("1 failed, 0 passed"), std::string::npos); +} + +TEST(test_driver, a_file_is_skipped_only_when_nothing_in_it_ran) +{ + const std::vector cases{holding("a file", {Outcome::skipped, Outcome::skipped})}; + + const auto [exit_code, output] = run(cases); + EXPECT_EQ(exit_code, NOTHING_VERIFIED); + EXPECT_NE(output.find("0 passed, 1 skipped"), std::string::npos); +} diff --git a/test/utils/test_collector.cpp b/test/utils/test_collector.cpp index 6eb8aabafb..e72ffc0c6c 100644 --- a/test/utils/test_collector.cpp +++ b/test/utils/test_collector.cpp @@ -73,10 +73,8 @@ void collect_tests( for (const auto& file : files) { // Loaded when the test runs: loading a whole tree up front costs far more. - cases.push_back( - {file.path.string(), [path = file.path, &settings, &vm](TestReport& report) { - run_fixture_file(path, settings, vm, report); - }}); + cases.push_back({file.path.string(), + [path = file.path, &settings, &vm] { return run_fixture_file(path, settings, vm); }}); } } } // namespace evmone::test diff --git a/test/utils/test_driver.cpp b/test/utils/test_driver.cpp index f4b6088609..08815f3675 100644 --- a/test/utils/test_driver.cpp +++ b/test/utils/test_driver.cpp @@ -21,14 +21,6 @@ namespace constexpr int LINE_WIDTH = 72; constexpr int PROGRESS_WIDTH = 60; -/// The outcome of one test, spelled as the progress character for it. -enum class Outcome : char -{ - passed = '.', - failed = 'F', - skipped = 's', -}; - void banner(std::ostream& out, std::string_view title, char fill = '=') { const auto padding = LINE_WIDTH - static_cast(title.size()) - 2; @@ -37,13 +29,12 @@ void banner(std::ostream& out, std::string_view title, char fill = '=') << std::string(static_cast(std::max(padding - left, 1)), fill) << '\n'; } -/// A test which did not pass: what the summary says about it and what it recorded. +/// A file with something to report: how it counts, and every fixture of it which did not pass. struct Note { - Outcome outcome; std::string name; - std::string reason; - std::vector failures; + Outcome outcome; + std::vector results; }; /// One progress character per test, wrapped, each line ending in the percentage done. @@ -151,6 +142,46 @@ void run_fixture(const std::string& name, const json::json& fixture, const TestS } // namespace +Result run_one(std::string name, const std::function& run) +{ + Result result{.name = std::move(name)}; + TestReport report{[&result](const Failure& failure) { result.failures.push_back(failure); }}; + report.start_case(result.name); // Names whatever the run itself reports. + + std::string exception_reason; + try + { + run(report); + } + catch (const UnsupportedTestFeature& ex) + { + result.outcome = Outcome::skipped; + result.reason = ex.what(); + } + catch (const std::exception& ex) + { + // One unloadable fixture in a tree of thousands fails its own test, not the run. + report.fail("exception", ex.what()); + exception_reason = concat("exception: ", ex.what()); + } + catch (...) + { + report.fail("exception", "not derived from std::exception"); + exception_reason = "exception not derived from std::exception"; + } + + // A recorded failure outranks giving up afterwards, in the summary too: the exception is + // the reason only when nothing failed before it threw. + if (!result.failures.empty()) + { + result.outcome = Outcome::failed; + result.reason = result.failures.size() == 1 && !exception_reason.empty() ? + std::move(exception_reason) : + result.failures.front().what; + } + return result; +} + int run_tests(std::span cases, std::ostream& out, const RunOptions& options) { if (options.collect_only) @@ -163,68 +194,59 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio const auto started = std::chrono::steady_clock::now(); banner(out, "test session starts"); - out << "collected " << cases.size() << (cases.size() == 1 ? " test\n\n" : " tests\n\n"); + out << "collected " << cases.size() << (cases.size() == 1 ? " file\n\n" : " files\n\n"); std::vector notes; Progress row{out, cases.size()}; + size_t failed = 0; + size_t skipped = 0; + size_t passed = 0; for (const auto& test : cases) { - // Held until the run ends, as pytest holds them, so nothing interleaves. - std::vector failures; - TestReport report{[&failures](const Failure& failure) { failures.push_back(failure); }}; - report.start_case(test.name); - - auto outcome = Outcome::passed; - std::string reason; - std::string exception_reason; if (!options.progress) out << test.name << '\n'; // The only thing naming what the test prints next. out << std::flush; + + // Held until the run ends, as pytest holds them, so nothing interleaves. + std::vector results; try { - test.run(report); - } - catch (const UnsupportedTestFeature& ex) - { - outcome = Outcome::skipped; - reason = ex.what(); - } - catch (const std::exception& ex) - { - // One unloadable fixture in a tree of thousands fails its own test, not the run. - report.fail("exception", ex.what()); - exception_reason = concat("exception: ", ex.what()); + results = test.run(); } catch (...) { - report.fail("exception", "not derived from std::exception"); - exception_reason = "exception not derived from std::exception"; + // A test which throws rather than reporting is the one result which says so. + const auto error = std::current_exception(); + results.push_back( + run_one(test.name, [&error](TestReport&) { std::rethrow_exception(error); })); } // A test writes its own output, an EVM trace above all, to another stream. std::clog << std::flush; - // A recorded failure outranks giving up afterwards, in the summary too: the exception - // is the reason only when nothing failed before it threw. - if (!failures.empty()) - { + // The file counts once, for the worst its fixtures reached. It is skipped only when + // nothing in it ran at all, so one fixture running is enough to give it a verdict. + static constexpr auto is = [](Outcome outcome) { + return [outcome](const Result& result) { return result.outcome == outcome; }; + }; + auto outcome = Outcome::passed; + if (std::ranges::any_of(results, is(Outcome::failed))) outcome = Outcome::failed; - reason = failures.size() == 1 && !exception_reason.empty() ? - std::move(exception_reason) : - failures.front().what; - } - if (outcome != Outcome::passed) - notes.push_back({outcome, test.name, std::move(reason), std::move(failures)}); + else if (!results.empty() && std::ranges::none_of(results, is(Outcome::passed))) + outcome = Outcome::skipped; + + ++(outcome == Outcome::failed ? failed : outcome == Outcome::skipped ? skipped : passed); + + // Every fixture which did not pass is named, including one declined by a file which + // passed on the fixtures beside it. Otherwise it would vanish from a green run. + std::erase_if(results, is(Outcome::passed)); + if (!results.empty()) + notes.push_back({test.name, outcome, std::move(results)}); if (options.progress) row.advance(outcome); } - // Every test which did not pass left exactly one note, so the counts follow from them. - const auto failed = std::ranges::count(notes, Outcome::failed, &Note::outcome); - const auto skipped = std::ranges::count(notes, Outcome::skipped, &Note::outcome); - const auto passed = cases.size() - notes.size(); - if (failed != 0) { out << '\n'; @@ -234,8 +256,11 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio if (note.outcome != Outcome::failed) continue; banner(out, note.name, '_'); - for (const auto& failure : note.failures) - out << failure << '\n'; + for (const auto& result : note.results) + { + for (const auto& failure : result.failures) + out << failure << '\n'; + } } } @@ -245,10 +270,13 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio banner(out, "short test summary info"); for (const auto& note : notes) { - out << (note.outcome == Outcome::failed ? "FAILED " : "SKIPPED ") << note.name; - if (!note.reason.empty()) - out << " - " << note.reason; - out << '\n'; + for (const auto& result : note.results) + { + out << (result.outcome == Outcome::failed ? "FAILED " : "SKIPPED ") << result.name; + if (!result.reason.empty()) + out << " - " << result.reason; + out << '\n'; + } } } @@ -270,53 +298,27 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio return passed == 0 ? NOTHING_VERIFIED : SUCCESS; } -void run_fixture_file( - const fs::path& path, const TestSettings& settings, evmc::VM& vm, TestReport& report) +std::vector run_fixture_file( + const fs::path& path, const TestSettings& settings, evmc::VM& vm) { // Named, because items() only borrows: iterating a temporary dangles. - const auto contents = load_fixture_file(path); - - std::vector> declined; // Fixture name and reason. - bool any_ran = false; + json::json contents; + // A file which does not parse, or holds no fixture at all, never gets as far as one: it is + // itself the only result there is to report. + if (auto loaded = + run_one(path.string(), [&](TestReport&) { contents = load_fixture_file(path); }); + loaded.outcome != Outcome::passed) + return {std::move(loaded)}; + + std::vector results; for (const auto& [name, fixture] : contents.items()) { if (!settings.selects(name)) continue; - try - { - run_fixture(name, fixture, settings, vm, report); - } - catch (const UnsupportedTestFeature& ex) - { - // This tool's own limit: a format it does not run, or a fixture its loader refuses. - declined.emplace_back(name, ex.what()); - continue; - } - catch (const std::exception& ex) - { - // The fixture is a test and it went wrong, which is this file's verdict but not the - // end of it: the fixtures after it are still worth running. Reported as run_tests - // reports one, so a what() of several lines keeps the indent of the rest. - report.fail("exception", ex.what()); - } - catch (...) - { - report.fail("exception", "not derived from std::exception"); - } - any_ran = true; + results.push_back(run_one(path.string() + "::" + name, + [&](TestReport& report) { run_fixture(name, fixture, settings, vm, report); })); } - - // A file in which this tool ran nothing it was asked for is skipped, not passed. - // TODO: A file whose cases -k all deselected still passes, as it did before this command - // existed, so a filter which matches nothing turns a failing tree green. Skip it instead, - // and an empty selection reaches NOTHING_VERIFIED on its own. - if (!any_ran && !declined.empty()) - throw UnsupportedTestFeature{declined.front().second}; - - // The file ran, so its own verdict says nothing about what it declined. Name those, or a - // fixture this tool stops running disappears from a passing run rather than being missed. - for (const auto& [name, reason] : declined) - std::cerr << path.string() << "::" << name << ": " << reason << '\n'; + return results; } } // namespace evmone::test diff --git a/test/utils/test_driver.hpp b/test/utils/test_driver.hpp index edfb9e8368..2f7790c6ba 100644 --- a/test/utils/test_driver.hpp +++ b/test/utils/test_driver.hpp @@ -23,15 +23,42 @@ constexpr int TESTS_FAILED = 1; /// value for the first of those; a test skipped has verified no more than a missing one. constexpr int NOTHING_VERIFIED = 5; +/// How one fixture ended, spelled as the character the progress row marks it with. +enum class Outcome : char +{ + passed = '.', + failed = 'F', + skipped = 's', +}; + +/// What running one fixture produced. +struct Result +{ + /// The fixture, as "::", or the file alone when it never got as far as one. + std::string name; + + Outcome outcome = Outcome::passed; + + /// Why it did not pass. Empty when it did. + std::string reason; + + std::vector failures; +}; + /// A single test: its name and how to run it. struct TestCase { std::string name; - /// Executes the test, recording what did not hold in the report. - std::function run; + /// Executes the test, returning what each of its fixtures produced. A test which never got + /// as far as a fixture returns the one result which says so, rather than throwing. + std::function()> run; }; +/// Runs @p run under a report of its own and says what it produced. What the run recorded +/// outranks how it ended: an exception is the reason only when nothing failed before it threw. +[[nodiscard]] Result run_one(std::string name, const std::function& run); + /// How to run and what to report. struct RunOptions { @@ -69,9 +96,9 @@ struct TestSettings } }; -/// Runs every selected fixture of one fixture file, which together are one test. Throws -/// UnsupportedTestFeature for a file this tool has nothing to run in. -void run_fixture_file(const std::filesystem::path& path, const TestSettings& settings, evmc::VM& vm, - TestReport& report); +/// Runs every selected fixture of one fixture file, which together are one test, and returns +/// what each produced. A file which holds no fixture, or does not parse, is the one result. +[[nodiscard]] std::vector run_fixture_file( + const std::filesystem::path& path, const TestSettings& settings, evmc::VM& vm); } // namespace evmone::test From 2038032c2cbdd7151b274da41f4b6e448d9b8788 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Thu, 3 Sep 2026 17:29:37 +0200 Subject: [PATCH 18/19] test review --- test/integration/CMakeLists.txt | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index b9448a6120..023c3b6dc9 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -83,6 +83,8 @@ DUP1,4 {\"pc\":6,\"op\":3,\"gas\":\"0xf4234\",\"gasCost\":\"0x3\",\"memSize\":0,\"stack\":\\[\"0x0\",\"0x4\"\\],\"depth\":1,\"refund\":0,\"opName\":\"SUB\"} ") + # FIXME: move `evmone-cli test` tests to subdir: integration/test (sibling to integration/t8n). + # we can add one more dir: integration/evmone-cli/{t8n,test,run}. set(DATA ${CMAKE_CURRENT_SOURCE_DIR}) # One command runs both fixture formats, deciding the fixture format on the go: @@ -100,41 +102,32 @@ DUP1,4 ${PREFIX}/test_not_a_test PROPERTIES PASS_REGULAR_EXPRESSION "SKIPPED[^\n]*index\\.json - not a test.*0 passed, 1 skipped in") - # A PASS_REGULAR_EXPRESSION makes CTest ignore the exit code, so these three assert on nothing - # else: a sound directory succeeds, one holding a fault fails, and so does one with nothing to - # run. Every test below pins what distinguishes them, which the exit code alone does not. + # Check exit code for: passed, failed, skipped (PASS_REGULAR_EXPRESSION ignores exit code). add_test(NAME ${PREFIX}/test_exit_ok COMMAND evmone-cli test ${DATA}/testcmd) - add_test(NAME ${PREFIX}/test_exit_fail COMMAND evmone-cli test ${DATA}/testcmd_fault) set_tests_properties(${PREFIX}/test_exit_fail PROPERTIES WILL_FAIL TRUE) - add_test(NAME ${PREFIX}/test_exit_skipped COMMAND evmone-cli test ${DATA}/testcmd_skipped) set_tests_properties(${PREFIX}/test_exit_skipped PROPERTIES WILL_FAIL TRUE) - # A fixture the loader refuses must not take the rest of the file with it: the file passes on - # the one fixture beside the two it declined. Its own verdict says nothing about those, so - # each is named in the summary, where a count of files alone would lose them. + # A declined fixture should not kill the whole file. add_test(NAME ${PREFIX}/test_dir_declined COMMAND evmone-cli test ${DATA}/testcmd) set_tests_properties( ${PREFIX}/test_dir_declined PROPERTIES PASS_REGULAR_EXPRESSION "SKIPPED[^\n]*::a_bad_rlp - tests with invalidly rlp-encoded blocks[^\n]*\nSKIPPED[^\n]*::c_engine - unsupported fixture format.*= 1 passed in") - # Whether a file holds fixtures is a property of the file, not of what -k selected: the - # fixture which is not a test faults even when it is the only one left. + # The -k filter should not filter out a "not a test". add_test(NAME ${PREFIX}/test_dir_fault COMMAND evmone-cli test ${DATA}/testcmd_fault -k b_not_a_test) set_tests_properties( ${PREFIX}/test_dir_fault PROPERTIES PASS_REGULAR_EXPRESSION "1 failed, 0 passed") - # A fixture whose load throws must not abandon the fixtures after it, and the file counts - # once for all three of them. + # A failed fixture should not kill the whole file. add_test(NAME ${PREFIX}/test_after_exception COMMAND evmone-cli test ${DATA}/testcmd_fault) set_tests_properties( ${PREFIX}/test_after_exception PROPERTIES PASS_REGULAR_EXPRESSION "collected 1 file\n.*c_load_error:\n exception.*d_bad_root:.*state root.*1 failed, 0 passed in") - # Over a directory -k selects within the file's test, so the summary is what proves a fixture - # ran: a count alone would hold just as well if the filter had dropped every one of them. + # The -k filter should filter out the failing fixture. add_test(NAME ${PREFIX}/test_dir_filter COMMAND evmone-cli test ${DATA}/statetest/filter -k passing_test_case --trace-summary) set_tests_properties( @@ -149,8 +142,8 @@ DUP1,4 ${PREFIX}/test_dir_skipped PROPERTIES PASS_REGULAR_EXPRESSION "all_unsupported\\.json::a_engine - unsupported fixture format: \"blockchain_test_engine\"\nSKIPPED[^\n]*::b_engine_x - .*empty\\.json - not a test.*pre_alloc\\.json - not a test.*0 passed, 5 skipped in") - # --trace and --histogram reach the runner through the app-level flags, and each turns the - # progress row into a line naming the test so what it writes can be told apart. + # The --histogram enables also test name printing. + # FIXME: This should print the fixture name, not the test file. This can be fixed later when we will add --verbosity flag. If so, remove file printing now. add_test(NAME ${PREFIX}/test_histogram COMMAND evmone-cli test --histogram ${DATA}/statetest/tests1/SuiteA/test1.json) set_tests_properties( @@ -158,14 +151,14 @@ DUP1,4 "test1\\.json --- # HISTOGRAM.*= 1 passed in") + # The --trace enables also test name printing. add_test(NAME ${PREFIX}/test_trace COMMAND evmone-cli test --trace ${DATA}/statetest/tests1/SuiteA/test1.json) set_tests_properties( ${PREFIX}/test_trace PROPERTIES PASS_REGULAR_EXPRESSION "test1\\.json\n\\{\"pc\":0,[^\n]*\"opName\":\"PUSH1\"\\}.*= 1 passed in") - # A named file is read whatever its extension, and one which does not parse names itself, - # because no fixture in it can be. + # A named file is loaded whatever its extension. add_test(NAME ${PREFIX}/test_file_unparsed COMMAND evmone-cli test ${DATA}/statetest/tests1/SuiteA/notes.txt) set_tests_properties( From 0ef22cb9ca04ce89d6d2ac5f05167f450a7d09d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Thu, 3 Sep 2026 17:40:07 +0200 Subject: [PATCH 19/19] delete evmone-{statetest,blockchaintest} --- test/CMakeLists.txt | 4 +- test/blockchaintest/.clang-tidy | 3 - test/blockchaintest/CMakeLists.txt | 10 -- test/blockchaintest/blockchaintest.cpp | 140 ---------------- .../integration/blockchaintest/CMakeLists.txt | 3 +- test/statetest/.clang-tidy | 3 - test/statetest/CMakeLists.txt | 10 -- test/statetest/statetest.cpp | 154 ------------------ 8 files changed, 3 insertions(+), 324 deletions(-) delete mode 100644 test/blockchaintest/.clang-tidy delete mode 100644 test/blockchaintest/CMakeLists.txt delete mode 100644 test/blockchaintest/blockchaintest.cpp delete mode 100644 test/statetest/.clang-tidy delete mode 100644 test/statetest/CMakeLists.txt delete mode 100644 test/statetest/statetest.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b8105f8238..c56bf1526f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -53,15 +53,13 @@ include(CableBuildInfo) cable_add_buildinfo_library(PROJECT_NAME evmone) add_subdirectory(bench) -add_subdirectory(blockchaintest) add_subdirectory(experimental) add_subdirectory(integration) add_subdirectory(internal_benchmarks) add_subdirectory(precompiles_bench) -add_subdirectory(statetest) add_subdirectory(unittests) -set(targets evmone-bench evmone-bench-internal evmone-blockchaintest evmone-precompiles-bench evmone-state evmone-statetest evmone-unittests) +set(targets evmone-bench evmone-bench-internal evmone-precompiles-bench evmone-state evmone-unittests) if(EVMONE_FUZZING) add_subdirectory(fuzzer) diff --git a/test/blockchaintest/.clang-tidy b/test/blockchaintest/.clang-tidy deleted file mode 100644 index efc628c8e9..0000000000 --- a/test/blockchaintest/.clang-tidy +++ /dev/null @@ -1,3 +0,0 @@ -InheritParentConfig: true -Checks: > - -clang-analyzer-cplusplus.NewDeleteLeaks diff --git a/test/blockchaintest/CMakeLists.txt b/test/blockchaintest/CMakeLists.txt deleted file mode 100644 index 79b42d9085..0000000000 --- a/test/blockchaintest/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# evmone: Fast Ethereum Virtual Machine implementation -# Copyright 2023 The evmone Authors. -# SPDX-License-Identifier: Apache-2.0 - -add_executable(evmone-blockchaintest) -target_link_libraries(evmone-blockchaintest PRIVATE evmone::testutils evmone evmone-buildinfo CLI11::CLI11) -target_sources( - evmone-blockchaintest PRIVATE - blockchaintest.cpp -) diff --git a/test/blockchaintest/blockchaintest.cpp b/test/blockchaintest/blockchaintest.cpp deleted file mode 100644 index cd31f4fa4a..0000000000 --- a/test/blockchaintest/blockchaintest.cpp +++ /dev/null @@ -1,140 +0,0 @@ -// evmone: Fast Ethereum Virtual Machine implementation -// Copyright 2023 The evmone Authors. -// SPDX-License-Identifier: Apache-2.0 - -#include -#include -#include -#include -#include -#include -#include - -namespace fs = std::filesystem; -using evmone::test::TestCase; - -namespace -{ -/// Adds to @p cases every test under @p root: one per file for a directory, one per test case in -/// the file when the file itself is named. Returns whether every test was collected. -/// A test which is one case: the driver takes an array of results, and this is the one. -TestCase one_case(std::string name, std::function run) -{ - return {name, - [name, run = std::move(run)] { return std::vector{evmone::test::run_one(name, run)}; }}; -} - -bool collect_tests(std::vector& cases, const fs::path& root, - std::span ignored, evmc::VM& vm) -{ - if (is_directory(root)) - { - auto files = evmone::test::collect_test_files(root); - evmone::test::ignore_test_files(files, ignored); - cases.reserve(cases.size() + files.size()); - for (const auto& file : files) - { - // Loaded when the test runs: loading a whole tree up front costs far more. A - // load which throws over an unsupported fixture reaches the driver, which skips. - cases.push_back(one_case( - file.path.string(), [path = file.path, &vm](evmone::test::TestReport& report) { - std::ifstream f{path}; - for (const auto& test : evmone::test::load_blockchain_tests(f)) - evmone::test::run_blockchain_test(test, vm, report); - })); - } - } - else // Treat as a file. - { - // Naming a file loads it now, to name the test cases in it. One which cannot be - // loaded becomes a single test the driver skips or fails. - std::vector tests; - try - { - std::ifstream f{root}; - tests = evmone::test::load_blockchain_tests(f); - } - catch (const evmone::test::UnsupportedTestFeature&) - { - // An unsupported fixture is a skip, not a broken collection. - cases.push_back(one_case(root.string(), - [error = std::current_exception()](auto&) { std::rethrow_exception(error); })); - return true; - } - catch (const std::exception& ex) - { - // Also reported here: --collect-only never runs the test. - std::cerr << root.string() << ": " << ex.what() << '\n'; - cases.push_back(one_case(root.string(), - [error = std::current_exception()](auto&) { std::rethrow_exception(error); })); - return false; - } - - for (const auto& test : tests) - { - cases.push_back(one_case( - root.string() + "::" + test.name, [test, &vm](evmone::test::TestReport& report) { - evmone::test::run_blockchain_test(test, vm, report); - })); - } - } - return true; -} -} // namespace - - -int main(int argc, char* argv[]) -{ - try - { - CLI::App app{"evmone blockchain test runner"}; - - app.set_version_flag("--version", "evmone-blockchaintest " EVMONE_VERSION); - - std::vector paths; - app.add_option("path", paths, - "Path to test file or directory. For a directory, all .json " - "files (except index.json) are considered test files, and each file is treated as a " - "separate test. For a file, all tests in the file are treated as separate tests.") - ->required() - ->check(CLI::ExistingPath); - - std::vector ignored; - app.add_option("--ignore", ignored, - "Path, relative to a test directory, not to collect tests from. May be given more " - "than once. Whole path components are matched, so --ignore bc4895 keeps " - "bc4895-withdrawals.") - // Without this the option is variadic and swallows the positional paths after it. - ->allow_extra_args(false); - - bool collect_only = false; - app.add_flag("--collect-only", collect_only, - "List the path of each collected test, one per line, and exit."); - - bool trace_flag = false; - app.add_flag("--trace", trace_flag, "Enable EVM tracing"); - - CLI11_PARSE(app, argc, argv); - - evmc::VM vm{evmc_create_evmone()}; - - if (trace_flag) - vm.set_option("trace", "1"); - - std::vector cases; - bool all_collected = true; - for (const auto& p : paths) - all_collected &= collect_tests(cases, p, ignored, vm); - - const evmone::test::RunOptions options{ - .collect_only = collect_only, .progress = !trace_flag}; - const auto exit_code = evmone::test::run_tests(cases, std::cout, options); - // A file which could not be loaded fails the listing too, not only a run of it. - return all_collected ? exit_code : evmone::test::TESTS_FAILED; - } - catch (const std::exception& ex) - { - std::cerr << ex.what() << "\n"; - return -1; - } -} diff --git a/test/integration/blockchaintest/CMakeLists.txt b/test/integration/blockchaintest/CMakeLists.txt index 06e6f0f7ce..ef12ee88cb 100644 --- a/test/integration/blockchaintest/CMakeLists.txt +++ b/test/integration/blockchaintest/CMakeLists.txt @@ -9,11 +9,12 @@ set(TESTS1 ${CMAKE_CURRENT_SOURCE_DIR}) add_test( NAME ${PREFIX}/json_test - COMMAND evmone-blockchaintest ${TESTS1}/test.json + COMMAND evmone-cli test ${TESTS1}/test.json ) set_tests_properties( ${PREFIX}/json_test PROPERTIES # Make sure both tests in the file are executed (both should fail). + # FIXME: evmone test should report failed tests, not failed files. maybe also report all tests as this is not known up front. PASS_REGULAR_EXPRESSION "2 failed, 0 passed" ) diff --git a/test/statetest/.clang-tidy b/test/statetest/.clang-tidy deleted file mode 100644 index efc628c8e9..0000000000 --- a/test/statetest/.clang-tidy +++ /dev/null @@ -1,3 +0,0 @@ -InheritParentConfig: true -Checks: > - -clang-analyzer-cplusplus.NewDeleteLeaks diff --git a/test/statetest/CMakeLists.txt b/test/statetest/CMakeLists.txt deleted file mode 100644 index 15e07d1618..0000000000 --- a/test/statetest/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# evmone: Fast Ethereum Virtual Machine implementation -# Copyright 2022 The evmone Authors. -# SPDX-License-Identifier: Apache-2.0 - -add_executable(evmone-statetest) -target_link_libraries(evmone-statetest PRIVATE evmone::testutils evmone evmone-buildinfo CLI11::CLI11) -target_sources( - evmone-statetest PRIVATE - statetest.cpp -) diff --git a/test/statetest/statetest.cpp b/test/statetest/statetest.cpp deleted file mode 100644 index 45a9fa1e79..0000000000 --- a/test/statetest/statetest.cpp +++ /dev/null @@ -1,154 +0,0 @@ -// evmone: Fast Ethereum Virtual Machine implementation -// Copyright 2022 The evmone Authors. -// SPDX-License-Identifier: Apache-2.0 - -#include -#include -#include -#include -#include -#include -#include - -namespace fs = std::filesystem; -using evmone::test::TestCase; - -namespace -{ -/// Adds to @p cases every test under @p root: one per file for a directory, one per test case in -/// the file when the file itself is named. Returns whether every test was collected. -/// A test which is one case: the driver takes an array of results, and this is the one. -TestCase one_case(std::string name, std::function run) -{ - return {name, - [name, run = std::move(run)] { return std::vector{evmone::test::run_one(name, run)}; }}; -} - -bool collect_tests(std::vector& cases, const fs::path& root, - const std::optional& filter, std::span ignored, evmc::VM& vm, - bool trace) -{ - // Which cases -k keeps. Over a directory it selects within the file's test, because - // naming the cases up front would mean loading the whole tree. - const auto selected = [&filter](const evmone::test::StateTransitionTest& test) { - return !filter.has_value() || test.name.find(*filter) != std::string::npos; - }; - - if (is_directory(root)) - { - auto files = evmone::test::collect_test_files(root); - evmone::test::ignore_test_files(files, ignored); - cases.reserve(cases.size() + files.size()); - for (const auto& file : files) - { - // Loaded when the test runs: loading a whole tree up front costs far more. - cases.push_back(one_case(file.path.string(), - [path = file.path, selected, &vm, trace](evmone::test::TestReport& report) { - std::ifstream f{path}; - for (const auto& test : evmone::test::load_state_tests(f)) - { - if (selected(test)) - evmone::test::run_state_test(test, vm, trace, report); - } - })); - } - } - else // Treat as a file. - { - // Naming a file loads it now, to name the test cases in it. One which cannot be - // loaded becomes a single test reporting why. - std::vector tests; - try - { - std::ifstream f{root}; - tests = evmone::test::load_state_tests(f); - } - catch (const std::exception& ex) - { - // Also reported here: --collect-only never runs the test. - std::cerr << root.string() << ": " << ex.what() << '\n'; - cases.push_back(one_case(root.string(), - [error = std::current_exception()](auto&) { std::rethrow_exception(error); })); - return false; - } - - for (const auto& test : tests) - { - if (!selected(test)) - continue; - cases.push_back(one_case(root.string() + "::" + test.name, - [test, &vm, trace](evmone::test::TestReport& report) { - evmone::test::run_state_test(test, vm, trace, report); - })); - } - } - return true; -} -} // namespace - - -int main(int argc, char* argv[]) -{ - try - { - CLI::App app{"evmone state test runner"}; - - app.set_version_flag("--version", "evmone-statetest " EVMONE_VERSION); - - std::vector paths; - app.add_option("path", paths, - "Path to test file or directory. For a directory, all .json " - "files (except index.json) are considered test files, and each file is treated as a " - "separate test. For a file, all tests in the file are treated as separate tests.") - ->required() - ->check(CLI::ExistingPath); - - std::optional filter; - app.add_option("-k", filter, - "Test name filter. Run only tests with names containing the specified string."); - - std::vector ignored; - app.add_option("--ignore", ignored, - "Path, relative to a test directory, not to collect tests from. May be given more " - "than once. Whole path components are matched, so --ignore bc4895 keeps " - "bc4895-withdrawals.") - // Without this the option is variadic and swallows the positional paths after it. - ->allow_extra_args(false); - - bool collect_only = false; - app.add_flag("--collect-only", collect_only, - "List the path of each collected test, one per line, and exit."); - - bool trace = false; - bool trace_summary = false; - const auto trace_opt = app.add_flag("--trace", trace, "Enable EVM tracing"); - app.add_flag("--trace-summary", trace_summary, "Output trace summary only") - ->excludes(trace_opt); - - CLI11_PARSE(app, argc, argv); - - evmc::VM vm{evmc_create_evmone(), {{"O", "0"}}}; - - if (trace) - { - std::ios::sync_with_stdio(false); - vm.set_option("trace", "1"); - } - - std::vector cases; - bool all_collected = true; - for (const auto& p : paths) - all_collected &= collect_tests(cases, p, filter, ignored, vm, trace || trace_summary); - - const evmone::test::RunOptions options{ - .collect_only = collect_only, .progress = !(trace || trace_summary)}; - const auto exit_code = evmone::test::run_tests(cases, std::cout, options); - // A file which could not be loaded fails the listing too, not only a run of it. - return all_collected ? exit_code : evmone::test::TESTS_FAILED; - } - catch (const std::exception& ex) - { - std::cerr << ex.what() << "\n"; - return -1; - } -}