diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b8105f8238..c56bf1526f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -53,15 +53,13 @@ include(CableBuildInfo) cable_add_buildinfo_library(PROJECT_NAME evmone) add_subdirectory(bench) -add_subdirectory(blockchaintest) add_subdirectory(experimental) add_subdirectory(integration) add_subdirectory(internal_benchmarks) add_subdirectory(precompiles_bench) -add_subdirectory(statetest) add_subdirectory(unittests) -set(targets evmone-bench evmone-bench-internal evmone-blockchaintest evmone-precompiles-bench evmone-state evmone-statetest evmone-unittests) +set(targets evmone-bench evmone-bench-internal evmone-precompiles-bench evmone-state evmone-unittests) if(EVMONE_FUZZING) add_subdirectory(fuzzer) diff --git a/test/blockchaintest/.clang-tidy b/test/blockchaintest/.clang-tidy deleted file mode 100644 index efc628c8e9..0000000000 --- a/test/blockchaintest/.clang-tidy +++ /dev/null @@ -1,3 +0,0 @@ -InheritParentConfig: true -Checks: > - -clang-analyzer-cplusplus.NewDeleteLeaks diff --git a/test/blockchaintest/CMakeLists.txt b/test/blockchaintest/CMakeLists.txt deleted file mode 100644 index 79b42d9085..0000000000 --- a/test/blockchaintest/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# evmone: Fast Ethereum Virtual Machine implementation -# Copyright 2023 The evmone Authors. -# SPDX-License-Identifier: Apache-2.0 - -add_executable(evmone-blockchaintest) -target_link_libraries(evmone-blockchaintest PRIVATE evmone::testutils evmone evmone-buildinfo CLI11::CLI11) -target_sources( - evmone-blockchaintest PRIVATE - blockchaintest.cpp -) diff --git a/test/blockchaintest/blockchaintest.cpp b/test/blockchaintest/blockchaintest.cpp deleted file mode 100644 index 8026c70ba8..0000000000 --- a/test/blockchaintest/blockchaintest.cpp +++ /dev/null @@ -1,133 +0,0 @@ -// evmone: Fast Ethereum Virtual Machine implementation -// Copyright 2023 The evmone Authors. -// SPDX-License-Identifier: Apache-2.0 - -#include -#include -#include -#include -#include -#include -#include - -namespace fs = std::filesystem; -using evmone::test::TestCase; - -namespace -{ -/// Adds to @p cases every test under @p root: one per file for a directory, one per test case in -/// the file when the file itself is named. Returns whether every test was collected. -bool collect_tests(std::vector& cases, const fs::path& root, - std::span ignored, evmc::VM& vm) -{ - if (is_directory(root)) - { - auto files = evmone::test::collect_test_files(root); - evmone::test::ignore_test_files(files, ignored); - cases.reserve(cases.size() + files.size()); - for (const auto& file : files) - { - // Loaded when the test runs: loading a whole tree up front costs far more. A - // load which throws over an unsupported fixture reaches the driver, which skips. - cases.push_back( - {file.path.string(), [path = file.path, &vm](evmone::test::TestReport& report) { - std::ifstream f{path}; - for (const auto& test : evmone::test::load_blockchain_tests(f)) - evmone::test::run_blockchain_test(test, vm, report); - }}); - } - } - else // Treat as a file. - { - // Naming a file loads it now, to name the test cases in it. One which cannot be - // loaded becomes a single test the driver skips or fails. - std::vector tests; - try - { - std::ifstream f{root}; - tests = evmone::test::load_blockchain_tests(f); - } - catch (const evmone::test::UnsupportedTestFeature&) - { - // An unsupported fixture is a skip, not a broken collection. - cases.push_back({root.string(), - [error = std::current_exception()](auto&) { std::rethrow_exception(error); }}); - return true; - } - catch (const std::exception& ex) - { - // Also reported here: --collect-only never runs the test. - std::cerr << root.string() << ": " << ex.what() << '\n'; - cases.push_back({root.string(), - [error = std::current_exception()](auto&) { std::rethrow_exception(error); }}); - return false; - } - - for (const auto& test : tests) - { - cases.push_back( - {root.string() + "::" + test.name, [test, &vm](evmone::test::TestReport& report) { - evmone::test::run_blockchain_test(test, vm, report); - }}); - } - } - return true; -} -} // namespace - - -int main(int argc, char* argv[]) -{ - try - { - CLI::App app{"evmone blockchain test runner"}; - - app.set_version_flag("--version", "evmone-blockchaintest " EVMONE_VERSION); - - std::vector paths; - app.add_option("path", paths, - "Path to test file or directory. For a directory, all .json " - "files (except index.json) are considered test files, and each file is treated as a " - "separate test. For a file, all tests in the file are treated as separate tests.") - ->required() - ->check(CLI::ExistingPath); - - std::vector ignored; - app.add_option("--ignore", ignored, - "Path, relative to a test directory, not to collect tests from. May be given more " - "than once. Whole path components are matched, so --ignore bc4895 keeps " - "bc4895-withdrawals.") - // Without this the option is variadic and swallows the positional paths after it. - ->allow_extra_args(false); - - bool collect_only = false; - app.add_flag("--collect-only", collect_only, - "List the path of each collected test, one per line, and exit."); - - bool trace_flag = false; - app.add_flag("--trace", trace_flag, "Enable EVM tracing"); - - CLI11_PARSE(app, argc, argv); - - evmc::VM vm{evmc_create_evmone()}; - - if (trace_flag) - vm.set_option("trace", "1"); - - std::vector cases; - bool all_collected = true; - for (const auto& p : paths) - all_collected &= collect_tests(cases, p, ignored, vm); - - const evmone::test::RunOptions options{ - .collect_only = collect_only, .progress = !trace_flag}; - const auto exit_code = evmone::test::run_tests(cases, std::cout, options); - // A file which could not be loaded fails the listing too, not only a run of it. - return all_collected ? exit_code : evmone::test::TESTS_FAILED; - } - catch (const std::exception& ex) - { - std::cerr << ex.what() << "\n"; - return -1; - } -} diff --git a/test/integration/CMakeLists.txt b/test/integration/CMakeLists.txt index 4abbe17ece..023c3b6dc9 100644 --- a/test/integration/CMakeLists.txt +++ b/test/integration/CMakeLists.txt @@ -54,6 +54,7 @@ if(TARGET evmone-cli) ${PREFIX}/version PROPERTIES PASS_REGULAR_EXPRESSION "evmone") # Test loading code from a file via @file syntax. + # TODO: Better to check the code.hex in. file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/code.hex "60 00 80 80 80 80 01 01 01 02 00") add_test(NAME ${PREFIX}/file_input COMMAND evmone-cli run @${CMAKE_CURRENT_BINARY_DIR}/code.hex) set_tests_properties( @@ -82,39 +83,89 @@ DUP1,4 {\"pc\":6,\"op\":3,\"gas\":\"0xf4234\",\"gasCost\":\"0x3\",\"memSize\":0,\"stack\":\\[\"0x0\",\"0x4\"\\],\"depth\":1,\"refund\":0,\"opName\":\"SUB\"} ") -endif() + # FIXME: move `evmone-cli test` tests to subdir: integration/test (sibling to integration/t8n). + # we can add one more dir: integration/evmone-cli/{t8n,test,run}. + set(DATA ${CMAKE_CURRENT_SOURCE_DIR}) + + # One command runs both fixture formats, deciding the fixture format on the go: + # file 2 has _info.fixture-format, others are recognized by field names. + add_test(NAME ${PREFIX}/test_mixed_formats COMMAND evmone-cli test + ${DATA}/statetest/tests1/SuiteA/test1.json ${DATA}/blockchaintest/eip7778_block_gas.json + ${DATA}/blockchaintest/unrecovered_sender_blob_gas.json) + set_tests_properties( + ${PREFIX}/test_mixed_formats PROPERTIES PASS_REGULAR_EXPRESSION "= 3 passed in") + + # A file recognized as "not a test" is skipped. + add_test(NAME ${PREFIX}/test_not_a_test COMMAND evmone-cli test + ${DATA}/statetest/tests1/SuiteA/index.json) + set_tests_properties( + ${PREFIX}/test_not_a_test PROPERTIES PASS_REGULAR_EXPRESSION + "SKIPPED[^\n]*index\\.json - not a test.*0 passed, 1 skipped in") + + # Check exit code for: passed, failed, skipped (PASS_REGULAR_EXPRESSION ignores exit code). + add_test(NAME ${PREFIX}/test_exit_ok COMMAND evmone-cli test ${DATA}/testcmd) + add_test(NAME ${PREFIX}/test_exit_fail COMMAND evmone-cli test ${DATA}/testcmd_fault) + set_tests_properties(${PREFIX}/test_exit_fail PROPERTIES WILL_FAIL TRUE) + add_test(NAME ${PREFIX}/test_exit_skipped COMMAND evmone-cli test ${DATA}/testcmd_skipped) + set_tests_properties(${PREFIX}/test_exit_skipped PROPERTIES WILL_FAIL TRUE) + + # A declined fixture should not kill the whole file. + add_test(NAME ${PREFIX}/test_dir_declined COMMAND evmone-cli test ${DATA}/testcmd) + set_tests_properties( + ${PREFIX}/test_dir_declined PROPERTIES PASS_REGULAR_EXPRESSION + "SKIPPED[^\n]*::a_bad_rlp - tests with invalidly rlp-encoded blocks[^\n]*\nSKIPPED[^\n]*::c_engine - unsupported fixture format.*= 1 passed in") -# A file holding a case which is not a fixture at all, beside one which runs. -add_test(NAME ${PREFIX}/fixture_fault COMMAND evmone-statetest - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault) -set_tests_properties(${PREFIX}/fixture_fault PROPERTIES WILL_FAIL TRUE) - -# WILL_FAIL above accepts any non-zero exit, and a fault downgraded to a skip exits non-zero too, -# so the counts are what distinguish the two. -add_test(NAME ${PREFIX}/fixture_fault_is_not_a_skip COMMAND evmone-statetest - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault) -set_tests_properties( - ${PREFIX}/fixture_fault_is_not_a_skip PROPERTIES PASS_REGULAR_EXPRESSION "1 failed, 0 passed") - -# Selecting only the case which is not a fixture must still fault. -add_test(NAME ${PREFIX}/fixture_fault_survives_filter COMMAND evmone-statetest - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_fault -k b_not_a_fixture) -set_tests_properties( - ${PREFIX}/fixture_fault_survives_filter PROPERTIES - PASS_REGULAR_EXPRESSION "1 failed, 0 passed") - -# A case whose load throws takes the rest of the file with it: the whole file is loaded before -# any of it runs, so the case after it is never reached and the failure is named after the file -# rather than the case it came from. FAILED pins that as a failure: a PASS_REGULAR_EXPRESSION -# makes CTest ignore the exit code, and a fault downgraded to a skip names the file too. -add_test(NAME ${PREFIX}/case_after_exception COMMAND evmone-statetest - ${CMAKE_CURRENT_SOURCE_DIR}/testcmd_cases) -set_tests_properties( - ${PREFIX}/case_after_exception PROPERTIES - PASS_REGULAR_EXPRESSION - "collected 1 test.*FAILED[^\n]*case_after_exception\\.json[^\n]*exception" - FAIL_REGULAR_EXPRESSION "b_wrong_state_root" -) + # The -k filter should not filter out a "not a test". + add_test(NAME ${PREFIX}/test_dir_fault COMMAND evmone-cli test + ${DATA}/testcmd_fault -k b_not_a_test) + set_tests_properties( + ${PREFIX}/test_dir_fault PROPERTIES PASS_REGULAR_EXPRESSION "1 failed, 0 passed") + + # A failed fixture should not kill the whole file. + add_test(NAME ${PREFIX}/test_after_exception COMMAND evmone-cli test ${DATA}/testcmd_fault) + set_tests_properties( + ${PREFIX}/test_after_exception PROPERTIES PASS_REGULAR_EXPRESSION + "collected 1 file\n.*c_load_error:\n exception.*d_bad_root:.*state root.*1 failed, 0 passed in") + + # The -k filter should filter out the failing fixture. + add_test(NAME ${PREFIX}/test_dir_filter COMMAND evmone-cli test + ${DATA}/statetest/filter -k passing_test_case --trace-summary) + set_tests_properties( + ${PREFIX}/test_dir_filter PROPERTIES + PASS_REGULAR_EXPRESSION "\"pass\":true" + FAIL_REGULAR_EXPRESSION "failing_test_case") + + # A file is one test, so a file with nothing to run counts once as skipped however many + # fixtures it declined: all_unsupported declines twice and both are named. + add_test(NAME ${PREFIX}/test_dir_skipped COMMAND evmone-cli test ${DATA}/testcmd_skipped) + set_tests_properties( + ${PREFIX}/test_dir_skipped PROPERTIES PASS_REGULAR_EXPRESSION + "all_unsupported\\.json::a_engine - unsupported fixture format: \"blockchain_test_engine\"\nSKIPPED[^\n]*::b_engine_x - .*empty\\.json - not a test.*pre_alloc\\.json - not a test.*0 passed, 5 skipped in") + + # The --histogram enables also test name printing. + # FIXME: This should print the fixture name, not the test file. This can be fixed later when we will add --verbosity flag. If so, remove file printing now. + add_test(NAME ${PREFIX}/test_histogram COMMAND evmone-cli test --histogram + ${DATA}/statetest/tests1/SuiteA/test1.json) + set_tests_properties( + ${PREFIX}/test_histogram PROPERTIES PASS_REGULAR_EXPRESSION + "test1\\.json +--- # HISTOGRAM.*= 1 passed in") + + # The --trace enables also test name printing. + add_test(NAME ${PREFIX}/test_trace COMMAND evmone-cli test --trace + ${DATA}/statetest/tests1/SuiteA/test1.json) + set_tests_properties( + ${PREFIX}/test_trace PROPERTIES PASS_REGULAR_EXPRESSION + "test1\\.json\n\\{\"pc\":0,[^\n]*\"opName\":\"PUSH1\"\\}.*= 1 passed in") + + # A named file is loaded whatever its extension. + add_test(NAME ${PREFIX}/test_file_unparsed COMMAND evmone-cli test + ${DATA}/statetest/tests1/SuiteA/notes.txt) + set_tests_properties( + ${PREFIX}/test_file_unparsed PROPERTIES PASS_REGULAR_EXPRESSION + "FAILED[^\n]*notes\\.txt - exception[^\n]*parse error.*1 failed, 0 passed in") + +endif() add_subdirectory(blockchaintest) add_subdirectory(export) diff --git a/test/integration/blockchaintest/CMakeLists.txt b/test/integration/blockchaintest/CMakeLists.txt index 06e6f0f7ce..ef12ee88cb 100644 --- a/test/integration/blockchaintest/CMakeLists.txt +++ b/test/integration/blockchaintest/CMakeLists.txt @@ -9,11 +9,12 @@ set(TESTS1 ${CMAKE_CURRENT_SOURCE_DIR}) add_test( NAME ${PREFIX}/json_test - COMMAND evmone-blockchaintest ${TESTS1}/test.json + COMMAND evmone-cli test ${TESTS1}/test.json ) set_tests_properties( ${PREFIX}/json_test PROPERTIES # Make sure both tests in the file are executed (both should fail). + # FIXME: evmone test should report failed tests, not failed files. maybe also report all tests as this is not known up front. PASS_REGULAR_EXPRESSION "2 failed, 0 passed" ) diff --git a/test/integration/blockchaintest/eip7778_block_gas.json b/test/integration/blockchaintest/eip7778_block_gas.json index 326c57ef0b..f7bf7042c1 100644 --- a/test/integration/blockchaintest/eip7778_block_gas.json +++ b/test/integration/blockchaintest/eip7778_block_gas.json @@ -130,6 +130,9 @@ }, "genesisRLP": "0x", "sealEngine": "NoProof", - "_info": {} + "_info": { + "fixture-format": "blockchain_test", + "comment": "The format every EEST blockchain fixture declares." + } } } diff --git a/test/integration/testcmd/declined.json b/test/integration/testcmd/declined.json new file mode 100644 index 0000000000..8995f5d82a --- /dev/null +++ b/test/integration/testcmd/declined.json @@ -0,0 +1,98 @@ +{ + "a_bad_rlp": { + "_info": { + "comment": "Declines to load: expectException without rlp_decoded, which the loader refuses before reading anything after the blocks." + }, + "network": "Cancun", + "genesisBlockHeader": { + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "coinbase": "0x0000000000000000000000000000000000000000", + "stateRoot": "0x0000000000000000000000000000000000000000000000000000000000000000", + "transactionsTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptTrie": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "bloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "number": "0x00", + "gasLimit": "0x01000000", + "gasUsed": "0x00", + "timestamp": "0x00", + "extraData": "0x00", + "hash": "0x0000000000000000000000000000000000000000000000000000000000000001" + }, + "pre": {}, + "blocks": [ + { + "expectException": "TransactionException.INVALID_SIGNATURE_VRS", + "rlp": "0x" + } + ] + }, + "b_state": { + "_info": { + "comment": "A state test beside blockchain ones, which is the whole point of deciding the format per case." + }, + "env": { + "currentBaseFee": "0x0a", + "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentDifficulty": "0x020000", + "currentGasLimit": "0xff112233445566", + "currentNumber": "0x01", + "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000", + "currentTimestamp": "0x03e8" + }, + "post": { + "London": [ + { + "hash": "0xe8010ce590f401c9d61fef8ab05bea9bcec24281b795e5868809bc4e515aa530", + "indexes": { + "data": 0, + "gas": 0, + "value": 0 + }, + "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" + } + ] + }, + "pre": { + "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { + "balance": "0x0de0b6b3a7640000", + "code": "0x600160010160005500", + "nonce": "0x00", + "storage": {} + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "balance": "0x00", + "code": "0x", + "nonce": "0x01", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x0de0b6b3a7640000", + "code": "0x", + "nonce": "0x00", + "storage": {} + } + }, + "transaction": { + "data": [ + "0x" + ], + "gasLimit": [ + "0x061a80" + ], + "gasPrice": "0x0a", + "nonce": "0x00", + "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", + "value": [ + "0x0186a0" + ] + } + }, + "c_engine": { + "_info": { + "fixture-format": "blockchain_test_engine", + "comment": "Carries blocks, so the shape alone would run it; the declared format is what decides, and this tool does not run that one." + }, + "blocks": [] + } +} diff --git a/test/integration/testcmd_cases/case_after_exception.json b/test/integration/testcmd_cases/case_after_exception.json deleted file mode 100644 index 0b8f8e37f7..0000000000 --- a/test/integration/testcmd_cases/case_after_exception.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "a_load_error": { - "_info": { - "fixture-format": "state_test", - "comment": "A state test with no pre state, so loading it throws." - } - }, - "b_wrong_state_root": { - "_info": { - "comment": "Runs and fails on the state root. Only reported if the case before it did not abandon the file." - }, - "env": { - "currentBaseFee": "0x0a", - "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", - "currentDifficulty": "0x020000", - "currentGasLimit": "0xff112233445566", - "currentNumber": "0x01", - "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000", - "currentTimestamp": "0x03e8" - }, - "post": { - "London": [ - { - "hash": "0x1111111111111111111111111111111111111111111111111111111111111111", - "indexes": { - "data": 0, - "gas": 0, - "value": 0 - }, - "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" - } - ] - }, - "pre": { - "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { - "balance": "0x0de0b6b3a7640000", - "code": "0x600160010160005500", - "nonce": "0x00", - "storage": {} - }, - "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { - "balance": "0x00", - "code": "0x", - "nonce": "0x01", - "storage": {} - }, - "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { - "balance": "0x0de0b6b3a7640000", - "code": "0x", - "nonce": "0x00", - "storage": {} - } - }, - "transaction": { - "data": [ - "0x" - ], - "gasLimit": [ - "0x061a80" - ], - "gasPrice": "0x0a", - "nonce": "0x00", - "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", - "value": [ - "0x0186a0" - ] - } - } -} diff --git a/test/integration/testcmd_fault/faults.json b/test/integration/testcmd_fault/faults.json new file mode 100644 index 0000000000..5833bd3cee --- /dev/null +++ b/test/integration/testcmd_fault/faults.json @@ -0,0 +1,135 @@ +{ + "a_runs": { + "_info": {}, + "env": { + "currentBaseFee": "0x0a", + "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentDifficulty": "0x020000", + "currentGasLimit": "0xff112233445566", + "currentNumber": "0x01", + "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000", + "currentTimestamp": "0x03e8" + }, + "post": { + "London": [ + { + "hash": "0xe8010ce590f401c9d61fef8ab05bea9bcec24281b795e5868809bc4e515aa530", + "indexes": { + "data": 0, + "gas": 0, + "value": 0 + }, + "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" + } + ] + }, + "pre": { + "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { + "balance": "0x0de0b6b3a7640000", + "code": "0x600160010160005500", + "nonce": "0x00", + "storage": {} + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "balance": "0x00", + "code": "0x", + "nonce": "0x01", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x0de0b6b3a7640000", + "code": "0x", + "nonce": "0x00", + "storage": {} + } + }, + "transaction": { + "data": [ + "0x" + ], + "gasLimit": [ + "0x061a80" + ], + "gasPrice": "0x0a", + "nonce": "0x00", + "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", + "value": [ + "0x0186a0" + ] + } + }, + "b_not_a_test": { + "_info": { + "comment": "Not a test: no fixture fields at all. Beside a case which runs, so this is unmistakably a fixture file with one broken case in it." + } + }, + "c_load_error": { + "_info": { + "fixture-format": "state_test", + "comment": "A state test with no pre state, so loading it throws." + } + }, + "d_bad_root": { + "_info": { + "comment": "Runs and fails on the state root, reached only because the throw above did not abandon the file." + }, + "env": { + "currentBaseFee": "0x0a", + "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentDifficulty": "0x020000", + "currentGasLimit": "0xff112233445566", + "currentNumber": "0x01", + "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000", + "currentTimestamp": "0x03e8" + }, + "post": { + "London": [ + { + "hash": "0x1111111111111111111111111111111111111111111111111111111111111111", + "indexes": { + "data": 0, + "gas": 0, + "value": 0 + }, + "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" + } + ] + }, + "pre": { + "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { + "balance": "0x0de0b6b3a7640000", + "code": "0x600160010160005500", + "nonce": "0x00", + "storage": {} + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "balance": "0x00", + "code": "0x", + "nonce": "0x01", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x0de0b6b3a7640000", + "code": "0x", + "nonce": "0x00", + "storage": {} + } + }, + "transaction": { + "data": [ + "0x" + ], + "gasLimit": [ + "0x061a80" + ], + "gasPrice": "0x0a", + "nonce": "0x00", + "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", + "value": [ + "0x0186a0" + ] + } + } +} diff --git a/test/integration/testcmd_fault/unrecognised_case.json b/test/integration/testcmd_fault/unrecognised_case.json deleted file mode 100644 index e389b69953..0000000000 --- a/test/integration/testcmd_fault/unrecognised_case.json +++ /dev/null @@ -1,67 +0,0 @@ -{ - "a_runs": { - "_info": {}, - "env": { - "currentBaseFee": "0x0a", - "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", - "currentDifficulty": "0x020000", - "currentGasLimit": "0xff112233445566", - "currentNumber": "0x01", - "currentRandom": "0x0000000000000000000000000000000000000000000000000000000000020000", - "currentTimestamp": "0x03e8" - }, - "post": { - "London": [ - { - "hash": "0xe8010ce590f401c9d61fef8ab05bea9bcec24281b795e5868809bc4e515aa530", - "indexes": { - "data": 0, - "gas": 0, - "value": 0 - }, - "logs": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347" - } - ] - }, - "pre": { - "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { - "balance": "0x0de0b6b3a7640000", - "code": "0x600160010160005500", - "nonce": "0x00", - "storage": {} - }, - "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { - "balance": "0x00", - "code": "0x", - "nonce": "0x01", - "storage": {} - }, - "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { - "balance": "0x0de0b6b3a7640000", - "code": "0x", - "nonce": "0x00", - "storage": {} - } - }, - "transaction": { - "data": [ - "0x" - ], - "gasLimit": [ - "0x061a80" - ], - "gasPrice": "0x0a", - "nonce": "0x00", - "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", - "value": [ - "0x0186a0" - ] - } - }, - "b_not_a_fixture": { - "_info": { - "comment": "Not a test: no fixture fields at all. Beside a case which runs, so this is unmistakably a fixture file with one broken case in it." - } - } -} diff --git a/test/integration/testcmd_skipped/all_unsupported.json b/test/integration/testcmd_skipped/all_unsupported.json new file mode 100644 index 0000000000..2a9fb9915b --- /dev/null +++ b/test/integration/testcmd_skipped/all_unsupported.json @@ -0,0 +1,15 @@ +{ + "a_engine": { + "_info": { + "fixture-format": "blockchain_test_engine", + "comment": "The file is named by this reason, the first collected, not the one below." + }, + "blocks": [] + }, + "b_engine_x": { + "_info": { + "fixture-format": "blockchain_test_engine_x" + }, + "blocks": [] + } +} diff --git a/test/integration/testcmd_skipped/empty.json b/test/integration/testcmd_skipped/empty.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/test/integration/testcmd_skipped/empty.json @@ -0,0 +1 @@ +{} diff --git a/test/integration/testcmd_skipped/not_an_object.json b/test/integration/testcmd_skipped/not_an_object.json new file mode 100644 index 0000000000..723839b091 --- /dev/null +++ b/test/integration/testcmd_skipped/not_an_object.json @@ -0,0 +1,7 @@ +[ + { + "comment": "A top-level array, so it holds no named fixture. Its one element would be taken for a blockchain test if the array were walked by index.", + "pre": {}, + "blocks": [] + } +] diff --git a/test/integration/testcmd_skipped/partial_shapes.json b/test/integration/testcmd_skipped/partial_shapes.json new file mode 100644 index 0000000000..5d85ea8afa --- /dev/null +++ b/test/integration/testcmd_skipped/partial_shapes.json @@ -0,0 +1,24 @@ +{ + "a_blocks_no_pre": { + "comment": "Carries one of the two keys a blockchain test is named by, so the shape does not name it and nothing else does either.", + "blocks": [] + }, + "b_tx_no_pre": { + "comment": "The same for a state test: a transaction to apply, but no state to apply it to.", + "transaction": { + "data": [ + "0x" + ], + "gasLimit": [ + "0x061a80" + ], + "gasPrice": "0x0a", + "nonce": "0x00", + "sender": "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "to": "0x095e7baea6a6c7c4c2dfeb977efac326af552d87", + "value": [ + "0x0186a0" + ] + } + } +} diff --git a/test/integration/testcmd_skipped/pre_alloc.json b/test/integration/testcmd_skipped/pre_alloc.json new file mode 100644 index 0000000000..015cd24e1d --- /dev/null +++ b/test/integration/testcmd_skipped/pre_alloc.json @@ -0,0 +1,27 @@ +{ + "testIds": [ + "tests/ported_static/vmIOandFlowOperations/test_mload.py::test_mload[fork_Osaka-blockchain_test_engine_x]" + ], + "environment": { + "currentCoinbase": "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba", + "currentGasLimit": "0xff112233445566", + "currentNumber": "0x01" + }, + "network": "Osaka", + "chainId": 1, + "pre": { + "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { + "balance": "0x0de0b6b3a7640000", + "code": "0x600160010160005500", + "nonce": "0x00", + "storage": {} + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x0de0b6b3a7640000", + "code": "0x", + "nonce": "0x00", + "storage": {} + } + }, + "comment": "EEST keeps the pre-allocation its fixtures share beside them, and none of it is a test. Several entries, none of which names a format or carries a shape." +} diff --git a/test/statetest/.clang-tidy b/test/statetest/.clang-tidy deleted file mode 100644 index efc628c8e9..0000000000 --- a/test/statetest/.clang-tidy +++ /dev/null @@ -1,3 +0,0 @@ -InheritParentConfig: true -Checks: > - -clang-analyzer-cplusplus.NewDeleteLeaks diff --git a/test/statetest/CMakeLists.txt b/test/statetest/CMakeLists.txt deleted file mode 100644 index 15e07d1618..0000000000 --- a/test/statetest/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# evmone: Fast Ethereum Virtual Machine implementation -# Copyright 2022 The evmone Authors. -# SPDX-License-Identifier: Apache-2.0 - -add_executable(evmone-statetest) -target_link_libraries(evmone-statetest PRIVATE evmone::testutils evmone evmone-buildinfo CLI11::CLI11) -target_sources( - evmone-statetest PRIVATE - statetest.cpp -) diff --git a/test/statetest/statetest.cpp b/test/statetest/statetest.cpp deleted file mode 100644 index 56a747e698..0000000000 --- a/test/statetest/statetest.cpp +++ /dev/null @@ -1,147 +0,0 @@ -// evmone: Fast Ethereum Virtual Machine implementation -// Copyright 2022 The evmone Authors. -// SPDX-License-Identifier: Apache-2.0 - -#include -#include -#include -#include -#include -#include -#include - -namespace fs = std::filesystem; -using evmone::test::TestCase; - -namespace -{ -/// Adds to @p cases every test under @p root: one per file for a directory, one per test case in -/// the file when the file itself is named. Returns whether every test was collected. -bool collect_tests(std::vector& cases, const fs::path& root, - const std::optional& filter, std::span ignored, evmc::VM& vm, - bool trace) -{ - // Which cases -k keeps. Over a directory it selects within the file's test, because - // naming the cases up front would mean loading the whole tree. - const auto selected = [&filter](const evmone::test::StateTransitionTest& test) { - return !filter.has_value() || test.name.find(*filter) != std::string::npos; - }; - - if (is_directory(root)) - { - auto files = evmone::test::collect_test_files(root); - evmone::test::ignore_test_files(files, ignored); - cases.reserve(cases.size() + files.size()); - for (const auto& file : files) - { - // Loaded when the test runs: loading a whole tree up front costs far more. - cases.push_back({file.path.string(), - [path = file.path, selected, &vm, trace](evmone::test::TestReport& report) { - std::ifstream f{path}; - for (const auto& test : evmone::test::load_state_tests(f)) - { - if (selected(test)) - evmone::test::run_state_test(test, vm, trace, report); - } - }}); - } - } - else // Treat as a file. - { - // Naming a file loads it now, to name the test cases in it. One which cannot be - // loaded becomes a single test reporting why. - std::vector tests; - try - { - std::ifstream f{root}; - tests = evmone::test::load_state_tests(f); - } - catch (const std::exception& ex) - { - // Also reported here: --collect-only never runs the test. - std::cerr << root.string() << ": " << ex.what() << '\n'; - cases.push_back({root.string(), - [error = std::current_exception()](auto&) { std::rethrow_exception(error); }}); - return false; - } - - for (const auto& test : tests) - { - if (!selected(test)) - continue; - cases.push_back({root.string() + "::" + test.name, - [test, &vm, trace](evmone::test::TestReport& report) { - evmone::test::run_state_test(test, vm, trace, report); - }}); - } - } - return true; -} -} // namespace - - -int main(int argc, char* argv[]) -{ - try - { - CLI::App app{"evmone state test runner"}; - - app.set_version_flag("--version", "evmone-statetest " EVMONE_VERSION); - - std::vector paths; - app.add_option("path", paths, - "Path to test file or directory. For a directory, all .json " - "files (except index.json) are considered test files, and each file is treated as a " - "separate test. For a file, all tests in the file are treated as separate tests.") - ->required() - ->check(CLI::ExistingPath); - - std::optional filter; - app.add_option("-k", filter, - "Test name filter. Run only tests with names containing the specified string."); - - std::vector ignored; - app.add_option("--ignore", ignored, - "Path, relative to a test directory, not to collect tests from. May be given more " - "than once. Whole path components are matched, so --ignore bc4895 keeps " - "bc4895-withdrawals.") - // Without this the option is variadic and swallows the positional paths after it. - ->allow_extra_args(false); - - bool collect_only = false; - app.add_flag("--collect-only", collect_only, - "List the path of each collected test, one per line, and exit."); - - bool trace = false; - bool trace_summary = false; - const auto trace_opt = app.add_flag("--trace", trace, "Enable EVM tracing"); - app.add_flag("--trace-summary", trace_summary, "Output trace summary only") - ->excludes(trace_opt); - - CLI11_PARSE(app, argc, argv); - - evmc::VM vm{evmc_create_evmone(), {{"O", "0"}}}; - - if (trace) - { - std::ios::sync_with_stdio(false); - vm.set_option("trace", "1"); - } - - std::vector cases; - bool all_collected = true; - for (const auto& p : paths) - all_collected &= collect_tests(cases, p, filter, ignored, vm, trace || trace_summary); - - const evmone::test::RunOptions options{ - .collect_only = collect_only, .progress = !(trace || trace_summary)}; - const auto exit_code = evmone::test::run_tests(cases, std::cout, options); - // A file which could not be loaded fails the listing too, not only a run of it. - return all_collected ? exit_code : evmone::test::TESTS_FAILED; - } - catch (const std::exception& ex) - { - std::cerr << ex.what() << "\n"; - return -1; - } -} diff --git a/test/unittests/test_driver_test.cpp b/test/unittests/test_driver_test.cpp index d2208aebd9..d4bc433811 100644 --- a/test/unittests/test_driver_test.cpp +++ b/test/unittests/test_driver_test.cpp @@ -22,6 +22,21 @@ Run run(std::span cases, const RunOptions& options = {}) const auto exit_code = run_tests(cases, out, options); return {exit_code, std::move(out).str()}; } + +/// A test holding one case, which is what a fixture file with one fixture in it comes to. +TestCase one(std::string name, std::function run) +{ + return {name, [name, run = std::move(run)] { return std::vector{run_one(name, run)}; }}; +} + +/// A test holding cases which only report the outcome given, running nothing. +TestCase holding(std::string name, std::initializer_list outcomes) +{ + std::vector results; + for (const auto outcome : outcomes) + results.push_back({name + "::case", outcome, "the reason", {}}); + return {std::move(name), [results = std::move(results)] { return results; }}; +} } // namespace TEST(test_driver, nothing_collected) @@ -29,13 +44,13 @@ TEST(test_driver, nothing_collected) const auto [exit_code, output] = run({}); EXPECT_EQ(NOTHING_VERIFIED, 5); // pytest's value, not just whatever we declared. EXPECT_EQ(exit_code, NOTHING_VERIFIED); - EXPECT_NE(output.find("collected 0 tests"), std::string::npos); + EXPECT_NE(output.find("collected 0 files"), std::string::npos); } TEST(test_driver, collect_only_lists_without_running) { bool ran = false; - const std::vector cases{{"a name", [&ran](TestReport&) { ran = true; }}}; + const std::vector cases{one("a name", [&ran](TestReport&) { ran = true; })}; const auto [exit_code, output] = run(cases, {.collect_only = true}); EXPECT_FALSE(ran); @@ -54,10 +69,10 @@ TEST(test_driver, exception_fails_only_its_own_test) { bool last_ran = false; const std::vector cases{ - {"ok", [](TestReport&) {}}, - {"throws", [](TestReport&) { throw std::runtime_error{"the reason"}; }}, - {"unknown", [](TestReport&) { throw 42; }}, // NOLINT(hicpp-exception-baseclass) - {"last", [&last_ran](TestReport&) { last_ran = true; }}, + one("ok", [](TestReport&) {}), + one("throws", [](TestReport&) { throw std::runtime_error{"the reason"}; }), + one("unknown", [](TestReport&) { throw 42; }), // NOLINT(hicpp-exception-baseclass) + one("last", [&last_ran](TestReport&) { last_ran = true; }), }; const auto [exit_code, output] = run(cases); @@ -71,8 +86,8 @@ TEST(test_driver, exception_fails_only_its_own_test) TEST(test_driver, unsupported_feature_skips) { const std::vector cases{ - {"ok", [](TestReport&) {}}, - {"skipped", [](TestReport&) { throw UnsupportedTestFeature{"no support for it"}; }}, + one("ok", [](TestReport&) {}), + one("skipped", [](TestReport&) { throw UnsupportedTestFeature{"no support for it"}; }), }; const auto [exit_code, output] = run(cases); @@ -84,7 +99,7 @@ TEST(test_driver, unsupported_feature_skips) TEST(test_driver, everything_skipped_verifies_nothing) { const std::vector cases{ - {"skipped", [](TestReport&) { throw UnsupportedTestFeature{"no support for it"}; }}}; + one("skipped", [](TestReport&) { throw UnsupportedTestFeature{"no support for it"}; })}; const auto [exit_code, output] = run(cases); EXPECT_EQ(exit_code, NOTHING_VERIFIED); @@ -94,7 +109,7 @@ TEST(test_driver, everything_skipped_verifies_nothing) TEST(test_driver, summary_names_the_check_which_failed) { const std::vector cases{ - {"mismatch", [](TestReport& report) { report.check_eq("a value", 1, 2); }}}; + one("mismatch", [](TestReport& report) { report.check_eq("a value", 1, 2); })}; const auto [exit_code, output] = run(cases); EXPECT_EQ(exit_code, 1); @@ -103,10 +118,10 @@ TEST(test_driver, summary_names_the_check_which_failed) TEST(test_driver, failure_outranks_a_later_exception) { - const std::vector cases{{"both", [](TestReport& report) { - report.check_eq("a value", 1, 2); - throw std::runtime_error{"gave up afterwards"}; - }}}; + const std::vector cases{one("both", [](TestReport& report) { + report.check_eq("a value", 1, 2); + throw std::runtime_error{"gave up afterwards"}; + })}; const auto [exit_code, output] = run(cases); EXPECT_EQ(exit_code, TESTS_FAILED); @@ -116,10 +131,10 @@ TEST(test_driver, failure_outranks_a_later_exception) TEST(test_driver, failure_outranks_a_later_skip) { - const std::vector cases{{"both", [](TestReport& report) { - report.check_eq("a value", 1, 2); - throw UnsupportedTestFeature{"gave up afterwards"}; - }}}; + const std::vector cases{one("both", [](TestReport& report) { + report.check_eq("a value", 1, 2); + throw UnsupportedTestFeature{"gave up afterwards"}; + })}; const auto [exit_code, output] = run(cases); EXPECT_EQ(exit_code, 1); @@ -127,3 +142,44 @@ TEST(test_driver, failure_outranks_a_later_skip) // The summary names the check which failed, not what the test then gave up on. EXPECT_NE(output.find("FAILED both - a value"), std::string::npos); } + +TEST(test_driver, a_file_counts_once_however_many_fixtures_it_holds) +{ + const std::vector cases{ + holding("a file", {Outcome::passed, Outcome::passed, Outcome::passed})}; + + const auto [exit_code, output] = run(cases); + EXPECT_EQ(exit_code, SUCCESS); + EXPECT_NE(output.find("collected 1 file"), std::string::npos); + EXPECT_NE(output.find("1 passed"), std::string::npos); +} + +TEST(test_driver, a_declined_fixture_is_named_though_its_file_passed) +{ + const std::vector cases{holding("a file", {Outcome::passed, Outcome::skipped})}; + + const auto [exit_code, output] = run(cases); + EXPECT_EQ(exit_code, SUCCESS); + // The file's own verdict says nothing about what it declined, so the fixture is named. + EXPECT_NE(output.find("1 passed in"), std::string::npos); + EXPECT_NE(output.find("SKIPPED a file::case - the reason"), std::string::npos); +} + +TEST(test_driver, one_failed_fixture_fails_its_file) +{ + const std::vector cases{ + holding("a file", {Outcome::passed, Outcome::failed, Outcome::skipped})}; + + const auto [exit_code, output] = run(cases); + EXPECT_EQ(exit_code, TESTS_FAILED); + EXPECT_NE(output.find("1 failed, 0 passed"), std::string::npos); +} + +TEST(test_driver, a_file_is_skipped_only_when_nothing_in_it_ran) +{ + const std::vector cases{holding("a file", {Outcome::skipped, Outcome::skipped})}; + + const auto [exit_code, output] = run(cases); + EXPECT_EQ(exit_code, NOTHING_VERIFIED); + EXPECT_NE(output.find("0 passed, 1 skipped"), std::string::npos); +} diff --git a/test/utils/test_collector.cpp b/test/utils/test_collector.cpp index ab30751b91..e72ffc0c6c 100644 --- a/test/utils/test_collector.cpp +++ b/test/utils/test_collector.cpp @@ -54,4 +54,27 @@ void ignore_test_files(std::vector& files, std::span i ignored, [&relative](const fs::path& prefix) { return is_under(relative, prefix); }); }); } + +void collect_tests( + std::vector& cases, const fs::path& root, const TestSettings& settings, evmc::VM& vm) +{ + // A file is one test, whether it was named or found under a directory. Naming the fixtures + // in it instead would mean loading every file to collect, which a whole tree cannot afford. + std::vector files; + if (is_directory(root)) + { + files = collect_test_files(root); + ignore_test_files(files, settings.ignored); + } + else + files.push_back({root, {}}); + + cases.reserve(cases.size() + files.size()); + for (const auto& file : files) + { + // Loaded when the test runs: loading a whole tree up front costs far more. + cases.push_back({file.path.string(), + [path = file.path, &settings, &vm] { return run_fixture_file(path, settings, vm); }}); + } +} } // namespace evmone::test diff --git a/test/utils/test_collector.hpp b/test/utils/test_collector.hpp index f116d6639e..55b573d33e 100644 --- a/test/utils/test_collector.hpp +++ b/test/utils/test_collector.hpp @@ -3,6 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 #pragma once +#include #include #include #include @@ -32,4 +33,10 @@ struct TestFile /// "bc4895-withdrawals". void ignore_test_files( std::vector& files, std::span ignored); + +/// Adds to @p cases one test per fixture file under @p root, which is that file itself when it +/// is not a directory. The tests hold @p settings and @p vm by reference, so both must outlive +/// them. +void collect_tests(std::vector& cases, const std::filesystem::path& root, + const TestSettings& settings, evmc::VM& vm); } // namespace evmone::test diff --git a/test/utils/test_driver.cpp b/test/utils/test_driver.cpp index 5ad47d8382..08815f3675 100644 --- a/test/utils/test_driver.cpp +++ b/test/utils/test_driver.cpp @@ -3,27 +3,24 @@ // SPDX-License-Identifier: Apache-2.0 #include "test_driver.hpp" +#include +#include #include +#include #include #include #include namespace evmone::test { +namespace fs = std::filesystem; + namespace { /// The report is laid out like pytest's. constexpr int LINE_WIDTH = 72; constexpr int PROGRESS_WIDTH = 60; -/// The outcome of one test, spelled as the progress character for it. -enum class Outcome : char -{ - passed = '.', - failed = 'F', - skipped = 's', -}; - void banner(std::ostream& out, std::string_view title, char fill = '=') { const auto padding = LINE_WIDTH - static_cast(title.size()) - 2; @@ -32,13 +29,12 @@ void banner(std::ostream& out, std::string_view title, char fill = '=') << std::string(static_cast(std::max(padding - left, 1)), fill) << '\n'; } -/// A test which did not pass: what the summary says about it and what it recorded. +/// A file with something to report: how it counts, and every fixture of it which did not pass. struct Note { - Outcome outcome; std::string name; - std::string reason; - std::vector failures; + Outcome outcome; + std::vector results; }; /// One progress character per test, wrapped, each line ending in the percentage done. @@ -65,8 +61,127 @@ class Progress m_column = 0; } }; + +/// What this tool makes of one fixture. EEST names the format in each fixture's "_info", and a +/// heuristic covers the hand-written and pre-EEST files which have no "_info" at all. +enum class Format +{ + state_test, + blockchain_test, + unsupported, ///< A fixture, in a format this tool does not run. + not_a_test, ///< Not a fixture at all. +}; + +Format classify(const json::json& fixture) +{ + if (const auto info = fixture.find("_info"); info != fixture.end()) + { + if (const auto format = info->find("fixture-format"); format != info->end()) + { + if (*format == "state_test") + return Format::state_test; + if (*format == "blockchain_test") + return Format::blockchain_test; + return Format::unsupported; + } + } + // Nothing declares the format: a hand-written or pre-EEST file, or an "_info" without one. + // Each shape is named by the state it starts from and what is applied to it, never by what + // it expects, so a fixture whose expectations are missing is still a test and is run. + // Anything else is not a test at all, as EEST's shared pre-allocation is not. + if (fixture.contains("pre") && fixture.contains("blocks")) + return Format::blockchain_test; + if (fixture.contains("pre") && fixture.contains("transaction")) + return Format::state_test; + return Format::not_a_test; +} + +/// Parses the fixture file at @p path. Throws UnsupportedTestFeature for a file with no fixture +/// in it, which is nothing to run: fixture directories hold other JSON beside the fixtures. +json::json load_fixture_file(const fs::path& path) +{ + std::ifstream f{path}; + // A stream which never opened reads as EOF, which parses as a syntax error in a file which + // has none. + if (!f) + throw std::runtime_error{"cannot open the file"}; + const auto contents = json::json::parse(f); + // Not one fixture in it: EEST keeps its shared pre-allocation and an index of the fixtures + // beside them, and neither is a test. Nor is a document which is not an object at all: + // items() would walk an array by index, naming its elements "0", "1", ... + if (!contents.is_object() || std::ranges::none_of(contents.items(), [](const auto& i) { + return classify(i.value()) != Format::not_a_test; + })) + throw UnsupportedTestFeature{"not a test"}; + return contents; +} + +/// Runs one fixture of a fixture file. One this tool does not recognise is a fault in the file; +/// one in a format it does not run is skipped. +void run_fixture(const std::string& name, const json::json& fixture, const TestSettings& settings, + evmc::VM& vm, TestReport& report) +{ + report.start_case(name); // Names whatever the load itself reports. + switch (classify(fixture)) + { + case Format::state_test: + run_state_test(make_state_test(name, fixture), vm, settings.trace_summary, report); + break; + case Format::blockchain_test: + run_blockchain_test(make_blockchain_test(name, fixture), vm, report); + break; + case Format::unsupported: + throw UnsupportedTestFeature{ + "unsupported fixture format: " + fixture.at("_info").at("fixture-format").dump()}; + case Format::not_a_test: + // The rest of the file holds fixtures, so this one is broken. + report.fail("not a test"); + break; + } +} + } // namespace +Result run_one(std::string name, const std::function& run) +{ + Result result{.name = std::move(name)}; + TestReport report{[&result](const Failure& failure) { result.failures.push_back(failure); }}; + report.start_case(result.name); // Names whatever the run itself reports. + + std::string exception_reason; + try + { + run(report); + } + catch (const UnsupportedTestFeature& ex) + { + result.outcome = Outcome::skipped; + result.reason = ex.what(); + } + catch (const std::exception& ex) + { + // One unloadable fixture in a tree of thousands fails its own test, not the run. + report.fail("exception", ex.what()); + exception_reason = concat("exception: ", ex.what()); + } + catch (...) + { + report.fail("exception", "not derived from std::exception"); + exception_reason = "exception not derived from std::exception"; + } + + // A recorded failure outranks giving up afterwards, in the summary too: the exception is + // the reason only when nothing failed before it threw. + if (!result.failures.empty()) + { + result.outcome = Outcome::failed; + result.reason = result.failures.size() == 1 && !exception_reason.empty() ? + std::move(exception_reason) : + result.failures.front().what; + } + return result; +} + int run_tests(std::span cases, std::ostream& out, const RunOptions& options) { if (options.collect_only) @@ -79,68 +194,59 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio const auto started = std::chrono::steady_clock::now(); banner(out, "test session starts"); - out << "collected " << cases.size() << (cases.size() == 1 ? " test\n\n" : " tests\n\n"); + out << "collected " << cases.size() << (cases.size() == 1 ? " file\n\n" : " files\n\n"); std::vector notes; Progress row{out, cases.size()}; + size_t failed = 0; + size_t skipped = 0; + size_t passed = 0; for (const auto& test : cases) { - // Held until the run ends, as pytest holds them, so nothing interleaves. - std::vector failures; - TestReport report{[&failures](const Failure& failure) { failures.push_back(failure); }}; - report.start_case(test.name); - - auto outcome = Outcome::passed; - std::string reason; - std::string exception_reason; if (!options.progress) out << test.name << '\n'; // The only thing naming what the test prints next. out << std::flush; + + // Held until the run ends, as pytest holds them, so nothing interleaves. + std::vector results; try { - test.run(report); - } - catch (const UnsupportedTestFeature& ex) - { - outcome = Outcome::skipped; - reason = ex.what(); - } - catch (const std::exception& ex) - { - // One unloadable fixture in a tree of thousands fails its own test, not the run. - report.fail("exception", ex.what()); - exception_reason = concat("exception: ", ex.what()); + results = test.run(); } catch (...) { - report.fail("exception", "not derived from std::exception"); - exception_reason = "exception not derived from std::exception"; + // A test which throws rather than reporting is the one result which says so. + const auto error = std::current_exception(); + results.push_back( + run_one(test.name, [&error](TestReport&) { std::rethrow_exception(error); })); } // A test writes its own output, an EVM trace above all, to another stream. std::clog << std::flush; - // A recorded failure outranks giving up afterwards, in the summary too: the exception - // is the reason only when nothing failed before it threw. - if (!failures.empty()) - { + // The file counts once, for the worst its fixtures reached. It is skipped only when + // nothing in it ran at all, so one fixture running is enough to give it a verdict. + static constexpr auto is = [](Outcome outcome) { + return [outcome](const Result& result) { return result.outcome == outcome; }; + }; + auto outcome = Outcome::passed; + if (std::ranges::any_of(results, is(Outcome::failed))) outcome = Outcome::failed; - reason = failures.size() == 1 && !exception_reason.empty() ? - std::move(exception_reason) : - failures.front().what; - } - if (outcome != Outcome::passed) - notes.push_back({outcome, test.name, std::move(reason), std::move(failures)}); + else if (!results.empty() && std::ranges::none_of(results, is(Outcome::passed))) + outcome = Outcome::skipped; + + ++(outcome == Outcome::failed ? failed : outcome == Outcome::skipped ? skipped : passed); + + // Every fixture which did not pass is named, including one declined by a file which + // passed on the fixtures beside it. Otherwise it would vanish from a green run. + std::erase_if(results, is(Outcome::passed)); + if (!results.empty()) + notes.push_back({test.name, outcome, std::move(results)}); if (options.progress) row.advance(outcome); } - // Every test which did not pass left exactly one note, so the counts follow from them. - const auto failed = std::ranges::count(notes, Outcome::failed, &Note::outcome); - const auto skipped = std::ranges::count(notes, Outcome::skipped, &Note::outcome); - const auto passed = cases.size() - notes.size(); - if (failed != 0) { out << '\n'; @@ -150,8 +256,11 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio if (note.outcome != Outcome::failed) continue; banner(out, note.name, '_'); - for (const auto& failure : note.failures) - out << failure << '\n'; + for (const auto& result : note.results) + { + for (const auto& failure : result.failures) + out << failure << '\n'; + } } } @@ -161,10 +270,13 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio banner(out, "short test summary info"); for (const auto& note : notes) { - out << (note.outcome == Outcome::failed ? "FAILED " : "SKIPPED ") << note.name; - if (!note.reason.empty()) - out << " - " << note.reason; - out << '\n'; + for (const auto& result : note.results) + { + out << (result.outcome == Outcome::failed ? "FAILED " : "SKIPPED ") << result.name; + if (!result.reason.empty()) + out << " - " << result.reason; + out << '\n'; + } } } @@ -185,4 +297,28 @@ int run_tests(std::span cases, std::ostream& out, const RunOptio // case of its own still counts as passed, which this does not change. return passed == 0 ? NOTHING_VERIFIED : SUCCESS; } + +std::vector run_fixture_file( + const fs::path& path, const TestSettings& settings, evmc::VM& vm) +{ + // Named, because items() only borrows: iterating a temporary dangles. + json::json contents; + // A file which does not parse, or holds no fixture at all, never gets as far as one: it is + // itself the only result there is to report. + if (auto loaded = + run_one(path.string(), [&](TestReport&) { contents = load_fixture_file(path); }); + loaded.outcome != Outcome::passed) + return {std::move(loaded)}; + + std::vector results; + for (const auto& [name, fixture] : contents.items()) + { + if (!settings.selects(name)) + continue; + results.push_back(run_one(path.string() + "::" + name, + [&](TestReport& report) { run_fixture(name, fixture, settings, vm, report); })); + } + return results; +} + } // namespace evmone::test diff --git a/test/utils/test_driver.hpp b/test/utils/test_driver.hpp index 2694b48680..2f7790c6ba 100644 --- a/test/utils/test_driver.hpp +++ b/test/utils/test_driver.hpp @@ -3,10 +3,16 @@ // SPDX-License-Identifier: Apache-2.0 #pragma once +#include +#include #include +#include +#include namespace evmone::test { +namespace json = nlohmann; + /// Nothing failed and something passed. constexpr int SUCCESS = 0; @@ -17,15 +23,42 @@ constexpr int TESTS_FAILED = 1; /// value for the first of those; a test skipped has verified no more than a missing one. constexpr int NOTHING_VERIFIED = 5; +/// How one fixture ended, spelled as the character the progress row marks it with. +enum class Outcome : char +{ + passed = '.', + failed = 'F', + skipped = 's', +}; + +/// What running one fixture produced. +struct Result +{ + /// The fixture, as "::", or the file alone when it never got as far as one. + std::string name; + + Outcome outcome = Outcome::passed; + + /// Why it did not pass. Empty when it did. + std::string reason; + + std::vector failures; +}; + /// A single test: its name and how to run it. struct TestCase { std::string name; - /// Executes the test, recording what did not hold in the report. - std::function run; + /// Executes the test, returning what each of its fixtures produced. A test which never got + /// as far as a fixture returns the one result which says so, rather than throwing. + std::function()> run; }; +/// Runs @p run under a report of its own and says what it produced. What the run recorded +/// outranks how it ended: an exception is the reason only when nothing failed before it threw. +[[nodiscard]] Result run_one(std::string name, const std::function& run); + /// How to run and what to report. struct RunOptions { @@ -43,4 +76,29 @@ struct RunOptions /// got. [[nodiscard]] int run_tests( std::span cases, std::ostream& out, const RunOptions& options = {}); + +/// What the tests are run with. +struct TestSettings +{ + /// Run only the fixtures whose name contains this. + std::optional name_filter; + + /// Paths, relative to a test directory, not to collect tests from. + std::vector ignored; + + /// Report each test's execution summary on the trace stream. + bool trace_summary = false; + + /// Whether the name filter, if any, keeps the fixture called @p name. + [[nodiscard]] bool selects(const std::string& name) const noexcept + { + return !name_filter.has_value() || name.find(*name_filter) != std::string::npos; + } +}; + +/// Runs every selected fixture of one fixture file, which together are one test, and returns +/// what each produced. A file which holds no fixture, or does not parse, is the one result. +[[nodiscard]] std::vector run_fixture_file( + const std::filesystem::path& path, const TestSettings& settings, evmc::VM& vm); + } // namespace evmone::test diff --git a/tools/evmone/main.cpp b/tools/evmone/main.cpp index e5e6c24ece..896e06e622 100644 --- a/tools/evmone/main.cpp +++ b/tools/evmone/main.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -151,6 +152,55 @@ int exec_t8n_cmd(evmc::VM& vm, const T8nOptions& opts) evmone::tooling::t8n(vm, args); return 0; } + +/// The options of the "test" command. +struct TestOptions +{ + std::vector paths; + evmone::test::TestSettings settings; + evmone::test::RunOptions run; +}; + +const CLI::App& setup_test_cmd(CLI::App& app, TestOptions& opts) +{ + auto& cmd = *app.add_subcommand("test", "Run Ethereum tests")->fallthrough(); + cmd.add_option("path", opts.paths, + "Test file or directory. Every fixture file is one test: under a directory, each " + ".json file except index.json.") + ->required() + ->check(CLI::ExistingPath); + cmd.add_option( + "-k", opts.settings.name_filter, "Run only the test cases whose name contains this."); + cmd.add_option("--ignore", opts.settings.ignored, + "Path, relative to a test directory, not to collect tests from. May be given more than " + "once. Whole path components are matched, so --ignore bc4895 keeps bc4895-withdrawals.") + // Without this the option is variadic and swallows the positional paths after it. + ->allow_extra_args(false); + cmd.add_flag("--collect-only", opts.run.collect_only, + "List each collected test, one per line, and exit."); + cmd.add_flag("--trace-summary", opts.settings.trace_summary, + "Report each state test's execution summary, as --trace also does. Blockchain tests " + "have no summary to report."); + return cmd; +} + +int exec_test_cmd(evmc::VM& vm, TestOptions opts, bool trace, bool histogram) +{ + // main() has switched the tracer on already. Its line per instruction is worth + // unsynchronising the streams for, and anything it writes per test would run into the + // progress row, as a summary would. + if (trace) + std::ios::sync_with_stdio(false); + opts.settings.trace_summary |= trace; + opts.run.progress = !(opts.settings.trace_summary || histogram); + + std::vector cases; + for (const auto& p : opts.paths) + collect_tests(cases, p, opts.settings, vm); + + return evmone::test::run_tests(cases, std::cout, opts.run); +} + } // namespace int main(int argc, const char* const* argv) noexcept @@ -198,6 +248,9 @@ int main(int argc, const char* const* argv) noexcept T8nOptions t8n_opts; const auto& t8n_cmd = setup_t8n_cmd(app, t8n_opts); + TestOptions test_opts; + const auto& test_cmd = setup_test_cmd(app, test_opts); + try { app.parse(argc, argv); @@ -218,6 +271,9 @@ int main(int argc, const char* const* argv) noexcept if (t8n_cmd) return exec_t8n_cmd(vm, t8n_opts); + if (test_cmd) + return exec_test_cmd(vm, test_opts, trace, histogram); + return 0; } catch (const CLI::ParseError& e)