From 5eb62bef5a10418238d48291bf9699a7b6851661 Mon Sep 17 00:00:00 2001 From: adria0 Date: Fri, 26 Jun 2020 09:38:36 +0200 Subject: [PATCH] json test suite --- Cargo.lock | 2 + ethcore/Cargo.toml | 7 +- ethcore/res/ethereum/runner/full.json | 128 ++++++++ ethcore/res/ethereum/tests | 2 +- .../res/ethereum/tests-issues/currents.json | 40 --- ethcore/src/json_tests/chain.rs | 192 +----------- ethcore/src/json_tests/difficulty.rs | 76 +---- ethcore/src/json_tests/executive.rs | 26 +- ethcore/src/json_tests/macros.rs | 59 ---- ethcore/src/json_tests/mod.rs | 6 +- ethcore/src/json_tests/runner.rs | 276 ++++++++++++++++++ ethcore/src/json_tests/skip.rs | 43 --- ethcore/src/json_tests/state.rs | 159 +--------- ethcore/src/json_tests/transaction.rs | 21 +- ethcore/src/json_tests/trie.rs | 50 +--- ethcore/src/lib.rs | 14 +- json/src/test_helpers/ethspec.rs | 122 ++++++++ json/src/test_helpers/mod.rs | 2 + 18 files changed, 593 insertions(+), 632 deletions(-) create mode 100644 ethcore/res/ethereum/runner/full.json delete mode 100644 ethcore/res/ethereum/tests-issues/currents.json create mode 100644 ethcore/src/json_tests/runner.rs delete mode 100644 ethcore/src/json_tests/skip.rs create mode 100644 json/src/test_helpers/ethspec.rs diff --git a/Cargo.lock b/Cargo.lock index 7eaf5b81f24..e41497ffd1f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1179,6 +1179,7 @@ dependencies = [ "executive-state", "fetch", "futures", + "globset", "hash-db", "itertools", "journaldb", @@ -1221,6 +1222,7 @@ dependencies = [ "using_queue", "verification", "vm", + "walkdir", ] [[package]] diff --git a/ethcore/Cargo.toml b/ethcore/Cargo.toml index a7133316af6..7e1fe5face3 100644 --- a/ethcore/Cargo.toml +++ b/ethcore/Cargo.toml @@ -27,6 +27,7 @@ ethereum-types = "0.9.0" evm = { path = "evm" } executive-state = { path = "executive-state" } futures = "0.1" +globset = "0.4.5" hash-db = "0.15.0" itertools = "0.8.2" journaldb = { path = "../util/journaldb" } @@ -53,6 +54,7 @@ rustc-hex = "2.1.0" scopeguard = "1.1.0" serde = "1.0" serde_derive = "1.0" +serde_json = "1.0" snapshot = { path = "snapshot" } spec = { path = "spec" } state-db = { path = "state-db" } @@ -65,6 +67,7 @@ unexpected = { path = "../util/unexpected" } using_queue = { path = "../miner/using-queue" } verification = { path = "./verification" } vm = { path = "vm" } +walkdir = { version = "2.3.1", optional = true } [dev-dependencies] account-db = { path = "account-db" } @@ -82,10 +85,8 @@ kvdb-rocksdb = "0.7.0" lazy_static = "1.3" machine = { path = "./machine", features = ["test-helpers"] } parity-runtime = "0.1.1" -serde_json = "1.0" stats = { path = "../util/stats" } pod = { path = "pod" } -tempfile = "3.1" trie-standardmap = "0.15.0" [features] @@ -114,7 +115,7 @@ evm-debug-tests = ["evm-debug", "evm/evm-debug-tests"] # EVM debug traces are printed. slow-blocks = [] # Run JSON consensus tests. -json-tests = ["env_logger", "test-helpers", "lazy_static", "machine/test-helpers", "common-types/test-helpers"] +json-tests = ["env_logger", "test-helpers", "walkdir", "machine/test-helpers", "common-types/test-helpers"] # Run memory/cpu heavy tests. test-heavy = [] # Compile test helpers diff --git a/ethcore/res/ethereum/runner/full.json b/ethcore/res/ethereum/runner/full.json new file mode 100644 index 00000000000..d2397a60a3e --- /dev/null +++ b/ethcore/res/ethereum/runner/full.json @@ -0,0 +1,128 @@ +{ + "chain": [ + { + "path": "res/ethereum/tests/BlockchainTests", + "skip": [ + { + "reference": "Issue https://github.com/openethereum/openethereum/issues/11616", + "names": [ + "blockChainFrontierWithLargerTDvsHomesteadBlockchain2_FrontierToHomesteadAt5", + "UncleFromFrontierInHomestead_FrontierToHomesteadAt5", + "UnclePopulation_FrontierToHomesteadAt5", + "blockChainFrontierWithLargerTDvsHomesteadBlockchain_FrontierToHomesteadAt5", + "DaoTransactions_HomesteadToDaoAt5", + "DaoTransactions_UncleExtradata_HomesteadToDaoAt5", + "DaoTransactions_EmptyTransactionAndForkBlocksAhead_HomesteadToDaoAt5", + "DaoTransactions_XBlockm1_HomesteadToDaoAt5", + "HomesteadOverrideFrontier_FrontierToHomesteadAt5", + "CallContractThatCreateContractBeforeAndAfterSwitchover_FrontierToHomesteadAt5", + "ContractCreationFailsOnHomestead_FrontierToHomesteadAt5", + "EIP150Transition_HomesteadToEIP150At5" + ], + "paths": [] + }, + { + "reference": "Tests that takes a lot of time to execute, ignored by geth", + "names": [], + "paths": [ + "res/ethereum/tests/BlockchainTests/stTimeConsuming/*" + ] + } + ] + }, + { + "path": "res/ethereum/tests/LegacyTests/Constantinople/BlockchainTests", + "skip": [ + { + "reference": "Issue https://github.com/openethereum/openethereum/issues/11616", + "names": [ + "blockChainFrontierWithLargerTDvsHomesteadBlockchain2_FrontierToHomesteadAt5", + "UncleFromFrontierInHomestead_FrontierToHomesteadAt5", + "UnclePopulation_FrontierToHomesteadAt5", + "blockChainFrontierWithLargerTDvsHomesteadBlockchain_FrontierToHomesteadAt5", + "DaoTransactions_HomesteadToDaoAt5", + "DaoTransactions_UncleExtradata_HomesteadToDaoAt5", + "DaoTransactions_EmptyTransactionAndForkBlocksAhead_HomesteadToDaoAt5", + "DaoTransactions_XBlockm1_HomesteadToDaoAt5", + "HomesteadOverrideFrontier_FrontierToHomesteadAt5", + "CallContractThatCreateContractBeforeAndAfterSwitchover_FrontierToHomesteadAt5", + "ContractCreationFailsOnHomestead_FrontierToHomesteadAt5", + "EIP150Transition_HomesteadToEIP150At5" + ], + "paths": [] + }, + { + "reference": "Tests that takes a lot of time to execute, ignored by geth", + "names": [], + "paths": [ + "res/ethereum/tests/LegacyTests/Constantinople/BlockchainTests/stTimeConsuming/*" + ] + } + ] + } + ], + "state": [ + { + "path": "res/ethereum/tests/GeneralStateTests", + "skip": [ + { + "reference": "Tests that takes a lot of time to execute, ignored by geth", + "names": {}, + "paths": [ + "res/ethereum/tests/GeneralStateTests/stTimeConsuming/*", + "res/ethereum/tests/GeneralStateTests/stStaticCall/static_Call1MB1024Calldepth.json" + ] + } + ] + }, + { + "path": "res/ethereum/tests/LegacyTests/Constantinople/GeneralStateTests", + "skip": [ + { + "reference": "Tests that takes a lot of time to execute, ignored by geth", + "names": {}, + "paths": [ + "res/ethereum/tests/LegacyTests/GeneralStateTests/Constantinople/stTimeConsuming/*", + "res/ethereum/tests/LegacyTests/GeneralStateTests/Constantinople/stStaticCall/static_Call1MB1024Calldepth.json" + ] + } + ] + } + ], + "difficulty": [ + { + "path": [ + "res/ethereum/tests/BasicTests/difficulty.json", + "res/ethereum/tests/BasicTests/difficultyMainNetwork.json" + ], + "chainspec": "Foundation" + } + ], + "executive": [ + { + "path": "res/ethereum/tests/VMTests" + } + ], + "transaction": [ + { + "path": "res/ethereum/tests/TransactionTests" + } + ], + "trie": [ + { + "path": [ + "res/ethereum/tests/TrieTests/trietest.json", + "res/ethereum/tests/TrieTests/trieanyorder.json" + ], + "triespec": "Generic" + }, + { + "path": [ + "res/ethereum/tests/TrieTests/hex_encoded_securetrie_test.json", + "res/ethereum/tests/TrieTests/trietest_secureTrie.json", + "res/ethereum/tests/TrieTests/trieanyorder_secureTrie.json" + ], + "triespec": "Secure" + } + ] +} \ No newline at end of file diff --git a/ethcore/res/ethereum/tests b/ethcore/res/ethereum/tests index 16fa567686a..cf0927a499e 160000 --- a/ethcore/res/ethereum/tests +++ b/ethcore/res/ethereum/tests @@ -1 +1 @@ -Subproject commit 16fa567686a8cea578a500b0095c4d7f9f3bbe63 +Subproject commit cf0927a499e33f80b4db7d0c08ca0992911ed338 diff --git a/ethcore/res/ethereum/tests-issues/currents.json b/ethcore/res/ethereum/tests-issues/currents.json deleted file mode 100644 index b213fe4d6c5..00000000000 --- a/ethcore/res/ethereum/tests-issues/currents.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "block": [ - { - "reference": "Issue https://github.com/openethereum/openethereum/issues/11616", - "failing": "bcFrontierToHomestead", - "subtests": [ - "blockChainFrontierWithLargerTDvsHomesteadBlockchain2_FrontierToHomesteadAt5", - "UncleFromFrontierInHomestead_FrontierToHomesteadAt5", - "UnclePopulation_FrontierToHomesteadAt5", - "blockChainFrontierWithLargerTDvsHomesteadBlockchain_FrontierToHomesteadAt5" - ] - }, - { - "reference": "Issue https://github.com/openethereum/openethereum/issues/11616", - "failing": "bcHomesteadToDao", - "subtests": [ - "DaoTransactions_HomesteadToDaoAt5", - "DaoTransactions_UncleExtradata_HomesteadToDaoAt5", - "DaoTransactions_EmptyTransactionAndForkBlocksAhead_HomesteadToDaoAt5", - "DaoTransactions_XBlockm1_HomesteadToDaoAt5", - "HomesteadOverrideFrontier_FrontierToHomesteadAt5", - "CallContractThatCreateContractBeforeAndAfterSwitchover_FrontierToHomesteadAt5", - "ContractCreationFailsOnHomestead_FrontierToHomesteadAt5" - ] - }, - { - "reference": "Issue https://github.com/openethereum/openethereum/issues/11616", - "failing": "bcHomesteadToEIP150", - "subtests": [ - "EIP150Transition_HomesteadToEIP150At5" - ] - } - ], - "state": [ - ], - "legacy_block": [ - ], - "legacy_state": [ - ] -} diff --git a/ethcore/src/json_tests/chain.rs b/ethcore/src/json_tests/chain.rs index 7442ec9f2f5..9b50baf24c1 100644 --- a/ethcore/src/json_tests/chain.rs +++ b/ethcore/src/json_tests/chain.rs @@ -20,7 +20,7 @@ use client::{Client, ClientConfig}; use client_traits::{ImportBlock, ChainInfo, StateOrBlock, Balance, Nonce, BlockChainClient}; use spec::Genesis; use ethjson::{ - test_helpers::blockchain, + test_helpers::{blockchain, ethspec::ChainTests}, spec::State }; use miner::Miner; @@ -32,21 +32,10 @@ use types::{ client_types::StateResult }; use verification::{VerifierType, queue::kind::BlockLike}; -use super::{HookType, SKIP_TESTS}; +use super::HookType; use rustc_hex::ToHex; use ethereum_types::{U256, H256}; -#[allow(dead_code)] -fn skip_test(name: &String, is_legacy: bool) -> bool { - let skip_set = if is_legacy { - &SKIP_TESTS.legacy_block - } else { - &SKIP_TESTS.block - }; - skip_set.iter() - .any(|block_test| block_test.subtests.contains(name)) -} - fn check_poststate(client: &Arc, test_name: &str, post_state: State) -> bool { let mut success = true; @@ -121,16 +110,17 @@ fn check_poststate(client: &Arc, test_name: &str, post_state: State) -> success } -#[allow(dead_code)] -pub fn json_chain_test(path: &Path, json_data: &[u8], start_stop_hook: &mut H, is_legacy: bool) -> Vec { +pub fn json_chain_test(test: &ChainTests, path: &Path, json_data: &[u8], start_stop_hook: &mut H) -> Vec { let _ = ::env_logger::try_init(); let tests = blockchain::Test::load(json_data) .expect(&format!("Could not parse JSON chain test data from {}", path.display())); let mut failed = Vec::new(); for (name, blockchain) in tests.into_iter() { - if skip_test(&name, is_legacy) { - println!(" - {} | {:?}: SKIPPED", name, blockchain.network); + + let skip_test = test.skip.iter().any(|block_test| block_test.names.contains(&name)); + if skip_test { + info!(" SKIPPED {:?} {:?}", name, blockchain.network); continue; } @@ -149,8 +139,6 @@ pub fn json_chain_test(path: &Path, json_data: &[u8], } }; - flushed_write!(" - {}...", name); - let spec = { let mut spec = match EvmTestClient::fork_spec_from_json(&blockchain.network) { Some(spec) => spec, @@ -216,174 +204,14 @@ pub fn json_chain_test(path: &Path, json_data: &[u8], } } - if !fail { - flushed_writeln!("OK"); + if fail { + flushed_writeln!(" - chain: {}...FAILED", name); } else { - flushed_writeln!("FAILED"); + flushed_writeln!(" - chain: {}...OK", name); } start_stop_hook(&name, HookType::OnStop); } - if failed.len() > 0 { - println!("!!! {:?} tests failed.", failed.len()); - } failed } - -#[cfg(test)] -mod block_tests { - use std::path::Path; - - use super::json_chain_test; - use json_tests::HookType; - - fn do_json_test(path: &Path, json_data: &[u8], h: &mut H) -> Vec { - json_chain_test(path, json_data, h, false) - } - - declare_test!{BlockchainTests_InvalidBlocks_bcBlockGasLimitTest, "BlockchainTests/InvalidBlocks/bcBlockGasLimitTest/"} - declare_test!{BlockchainTests_InvalidBlocks_bcForgedTest, "BlockchainTests/InvalidBlocks/bcForgedTest/"} - declare_test!{BlockchainTests_InvalidBlocks_bcInvalidHeaderTest, "BlockchainTests/InvalidBlocks/bcInvalidHeaderTest/"} - declare_test!{BlockchainTests_InvalidBlocks_bcMultiChainTest, "BlockchainTests/InvalidBlocks/bcMultiChainTest/"} - declare_test!{BlockchainTests_InvalidBlocks_bcUncleHeaderValidity, "BlockchainTests/InvalidBlocks/bcUncleHeaderValidity/"} - declare_test!{BlockchainTests_InvalidBlocks_bcUncleSpecialTests, "BlockchainTests/InvalidBlocks/bcUncleSpecialTests/"} - declare_test!{BlockchainTests_InvalidBlocks_bcUncleTest, "BlockchainTests/InvalidBlocks/bcUncleTest/"} - - declare_test!{BlockchainTests_ValidBlocks_bcBlockGasLimitTest, "BlockchainTests/ValidBlocks/bcBlockGasLimitTest/"} - declare_test!{BlockchainTests_ValidBlocks_bcExploitTest, "BlockchainTests/ValidBlocks/bcExploitTest/"} - declare_test!{BlockchainTests_ValidBlocks_bcForkStressTest, "BlockchainTests/ValidBlocks/bcForkStressTest/"} - declare_test!{BlockchainTests_ValidBlocks_bcGasPricerTest, "BlockchainTests/ValidBlocks/bcGasPricerTest/"} - declare_test!{BlockchainTests_ValidBlocks_bcMultiChainTest, "BlockchainTests/ValidBlocks/bcMultiChainTest/"} - declare_test!{BlockchainTests_ValidBlocks_bcRandomBlockhashTest, "BlockchainTests/ValidBlocks/bcRandomBlockhashTest/"} - declare_test!{BlockchainTests_ValidBlocks_bcStateTests, "BlockchainTests/ValidBlocks/bcStateTests/"} - declare_test!{BlockchainTests_ValidBlocks_bcTotalDifficultyTest, "BlockchainTests/ValidBlocks/bcTotalDifficultyTest/"} - declare_test!{BlockchainTests_ValidBlocks_bcUncleSpecialTests, "BlockchainTests/ValidBlocks/bcUncleSpecialTests/"} - declare_test!{BlockchainTests_ValidBlocks_bcUncleTest, "BlockchainTests/ValidBlocks/bcUncleTest/"} - declare_test!{BlockchainTests_ValidBlocks_bcValidBlockTest, "BlockchainTests/ValidBlocks/bcValidBlockTest/"} - declare_test!{BlockchainTests_ValidBlocks_bcWalletTest, "BlockchainTests/ValidBlocks/bcWalletTest/"} - - declare_test!{BlockchainTests_GeneralStateTest_stArgsZeroOneBalance, "BlockchainTests/GeneralStateTests/stArgsZeroOneBalance/"} - declare_test!{BlockchainTests_GeneralStateTest_stAttackTest, "BlockchainTests/GeneralStateTests/stAttackTest/"} - declare_test!{BlockchainTests_GeneralStateTest_stBadOpcodeTest, "BlockchainTests/GeneralStateTests/stBadOpcode/"} - declare_test!{BlockchainTests_GeneralStateTest_stBugsTest, "BlockchainTests/GeneralStateTests/stBugs/"} - declare_test!{BlockchainTests_GeneralStateTest_stCallCodes, "BlockchainTests/GeneralStateTests/stCallCodes/"} - declare_test!{BlockchainTests_GeneralStateTest_stCallCreateCallCodeTest, "BlockchainTests/GeneralStateTests/stCallCreateCallCodeTest/"} - declare_test!{BlockchainTests_GeneralStateTest_stCallDelegateCodesCallCodeHomestead, "BlockchainTests/GeneralStateTests/stCallDelegateCodesCallCodeHomestead/"} - declare_test!{BlockchainTests_GeneralStateTest_stCallDelegateCodesHomestead, "BlockchainTests/GeneralStateTests/stCallDelegateCodesHomestead/"} - declare_test!{BlockchainTests_GeneralStateTest_stChangedEIP150, "BlockchainTests/GeneralStateTests/stChangedEIP150/"} - declare_test!{BlockchainTests_GeneralStateTest_stCopyCodeTest, "BlockchainTests/GeneralStateTests/stCodeCopyTest/"} - declare_test!{BlockchainTests_GeneralStateTest_stCodeSizeLimit, "BlockchainTests/GeneralStateTests/stCodeSizeLimit/"} - declare_test!{BlockchainTests_GeneralStateTest_stCreate2, "BlockchainTests/GeneralStateTests/stCreate2/"} - declare_test!{BlockchainTests_GeneralStateTest_stCreateTest, "BlockchainTests/GeneralStateTests/stCreateTest/"} - declare_test!{BlockchainTests_GeneralStateTest_stDelegatecallTestHomestead, "BlockchainTests/GeneralStateTests/stDelegatecallTestHomestead/"} - declare_test!{BlockchainTests_GeneralStateTest_stEIP150singleCodeGasPrices, "BlockchainTests/GeneralStateTests/stEIP150singleCodeGasPrices/"} - declare_test!{BlockchainTests_GeneralStateTest_stEIP150Specific, "BlockchainTests/GeneralStateTests/stEIP150Specific/"} - declare_test!{BlockchainTests_GeneralStateTest_stEIP158Specific, "BlockchainTests/GeneralStateTests/stEIP158Specific/"} - declare_test!{BlockchainTests_GeneralStateTest_stExample, "BlockchainTests/GeneralStateTests/stExample/"} - declare_test!{BlockchainTests_GeneralStateTest_stExtCodeHash, "BlockchainTests/GeneralStateTests/stExtCodeHash/"} - declare_test!{BlockchainTests_GeneralStateTest_stHomesteadSpecific, "BlockchainTests/GeneralStateTests/stHomesteadSpecific/"} - declare_test!{BlockchainTests_GeneralStateTest_stInitCodeTest, "BlockchainTests/GeneralStateTests/stInitCodeTest/"} - declare_test!{BlockchainTests_GeneralStateTest_stLogTests, "BlockchainTests/GeneralStateTests/stLogTests/"} - declare_test!{BlockchainTests_GeneralStateTest_stMemExpandingEIP150Calls, "BlockchainTests/GeneralStateTests/stMemExpandingEIP150Calls/"} - declare_test!{heavy => BlockchainTests_GeneralStateTest_stMemoryStressTest, "BlockchainTests/GeneralStateTests/stMemoryStressTest/"} - declare_test!{BlockchainTests_GeneralStateTest_stMemoryTest, "BlockchainTests/GeneralStateTests/stMemoryTest/"} - declare_test!{BlockchainTests_GeneralStateTest_stNonZeroCallsTest, "BlockchainTests/GeneralStateTests/stNonZeroCallsTest/"} - declare_test!{BlockchainTests_GeneralStateTest_stPreCompiledContracts, "BlockchainTests/GeneralStateTests/stPreCompiledContracts/"} - declare_test!{BlockchainTests_GeneralStateTest_stPreCompiledContracts2, "BlockchainTests/GeneralStateTests/stPreCompiledContracts2/"} - declare_test!{heavy => BlockchainTests_GeneralStateTest_stQuadraticComplexityTest, "BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/"} - declare_test!{BlockchainTests_GeneralStateTest_stRandom, "BlockchainTests/GeneralStateTests/stRandom/"} - declare_test!{BlockchainTests_GeneralStateTest_stRandom2, "BlockchainTests/GeneralStateTests/stRandom2/"} - declare_test!{BlockchainTests_GeneralStateTest_stRecursiveCreate, "BlockchainTests/GeneralStateTests/stRecursiveCreate/"} - declare_test!{BlockchainTests_GeneralStateTest_stRefundTest, "BlockchainTests/GeneralStateTests/stRefundTest/"} - declare_test!{BlockchainTests_GeneralStateTest_stReturnDataTest, "BlockchainTests/GeneralStateTests/stReturnDataTest/"} - declare_test!{BlockchainTests_GeneralStateTest_stRevertTest, "BlockchainTests/GeneralStateTests/stRevertTest/"} - declare_test!{BlockchainTests_GeneralStateTest_stShift, "BlockchainTests/GeneralStateTests/stShift/"} - declare_test!{BlockchainTests_GeneralStateTest_stSolidityTest, "BlockchainTests/GeneralStateTests/stSolidityTest/"} - declare_test!{BlockchainTests_GeneralStateTest_stSpecialTest, "BlockchainTests/GeneralStateTests/stSpecialTest/"} - declare_test!{BlockchainTests_GeneralStateTest_stSStoreTest, "BlockchainTests/GeneralStateTests/stSStoreTest/"} - declare_test!{BlockchainTests_GeneralStateTest_stStackTests, "BlockchainTests/GeneralStateTests/stStackTests/"} - declare_test!{BlockchainTests_GeneralStateTest_stStaticCall, "BlockchainTests/GeneralStateTests/stStaticCall/"} - declare_test!{BlockchainTests_GeneralStateTest_stSubroutine, "BlockchainTests/GeneralStateTests/stSubroutine/"} - declare_test!{BlockchainTests_GeneralStateTest_stSystemOperationsTest, "BlockchainTests/GeneralStateTests/stSystemOperationsTest/"} - declare_test!{BlockchainTests_GeneralStateTest_stTimeConsuming, "BlockchainTests/GeneralStateTests/stTimeConsuming/"} - declare_test!{BlockchainTests_GeneralStateTest_stTransactionTest, "BlockchainTests/GeneralStateTests/stTransactionTest/"} - declare_test!{BlockchainTests_GeneralStateTest_stTransitionTest, "BlockchainTests/GeneralStateTests/stTransitionTest/"} - declare_test!{BlockchainTests_GeneralStateTest_stWalletTest, "BlockchainTests/GeneralStateTests/stWalletTest/"} - declare_test!{BlockchainTests_GeneralStateTest_stZeroCallsRevert, "BlockchainTests/GeneralStateTests/stZeroCallsRevert/"} - declare_test!{BlockchainTests_GeneralStateTest_stZeroCallsTest, "BlockchainTests/GeneralStateTests/stZeroCallsTest/"} - declare_test!{BlockchainTests_GeneralStateTest_stZeroKnowledge, "BlockchainTests/GeneralStateTests/stZeroKnowledge/"} - declare_test!{BlockchainTests_GeneralStateTest_stZeroKnowledge2, "BlockchainTests/GeneralStateTests/stZeroKnowledge2/"} - - declare_test!{BlockchainTests_TransitionTests_bcEIP158ToByzantium, "BlockchainTests/TransitionTests/bcEIP158ToByzantium/"} - declare_test!{BlockchainTests_TransitionTests_bcFrontierToHomestead, "BlockchainTests/TransitionTests/bcFrontierToHomestead/"} - declare_test!{BlockchainTests_TransitionTests_bcHomesteadToDao, "BlockchainTests/TransitionTests/bcHomesteadToDao/"} - declare_test!{BlockchainTests_TransitionTests_bcHomesteadToEIP150, "BlockchainTests/TransitionTests/bcHomesteadToEIP150/"} - declare_test!{BlockchainTests_TransitionTests_bcByzantiumToConstantinopleFix, "BlockchainTests/TransitionTests/bcByzantiumToConstantinopleFix/"} - - declare_test!{BlockchainTests_RandomStateTest391, "BlockchainTests/randomStatetest391.json"} -} - -/// Legacy tests, still keeping it to check if there is any regression in blocks < Instambul HF -#[cfg(test)] -mod block_tests_legacy { - use std::path::Path; - - use super::json_chain_test; - use json_tests::HookType; - - fn do_json_test(path: &Path, json_data: &[u8], h: &mut H) -> Vec { - json_chain_test(path, json_data, h, true) - } - - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stArgsZeroOneBalance, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stArgsZeroOneBalance/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stAttackTest, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stAttackTest/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stBadOpcode, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stBadOpcode/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stBugs, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stBugs/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stCallCodes, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCallCodes/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stCallCreateCallCodeTest, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCallCreateCallCodeTest/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stCallDelegateCodesCallCodeHomestead, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCallDelegateCodesCallCodeHomestead/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stCallDelegateCodesHomestead, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCallDelegateCodesHomestead/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stChangedEIP150, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stChangedEIP150/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stCodeCopyTest, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCodeCopyTest/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stCodeSizeLimit, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCodeSizeLimit/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stCreate2, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreate2/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stCreateTest, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stCreateTest/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stDelegatecallTestHomestead, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stDelegatecallTestHomestead/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stEIP150singleCodeGasPrices, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stEIP150singleCodeGasPrices/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stEIP150Specific, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stEIP150Specific/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stEIP158Specific, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stEIP158Specific/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stExample, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stExample/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stExtCodeHash, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stExtCodeHash/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stHomesteadSpecific, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stHomesteadSpecific/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stInitCodeTest, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stInitCodeTest/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stLogTests, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stLogTests/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stMemExpandingEIP150Calls, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stMemExpandingEIP150Calls/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stMemoryStressTest, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stMemoryStressTest/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stMemoryTest, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stMemoryTest/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stNonZeroCallsTest, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stNonZeroCallsTest/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stPreCompiledContracts, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stPreCompiledContracts/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stPreCompiledContracts2, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stPreCompiledContracts2/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stQuadraticComplexityTest, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stQuadraticComplexityTest/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stRandom, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stRandom/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stRandom2, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stRandom2/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stRecursiveCreate, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stRecursiveCreate/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stRefundTest, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stRefundTest/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stReturnDataTest, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stReturnDataTest/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stRevertTest, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stRevertTest/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stShift, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stShift/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stSolidityTest, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stSolidityTest/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stSpecialTest, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stSpecialTest/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stSStoreTest, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stSStoreTest/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stStackTests, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStackTests/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stStaticCall, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stStaticCall/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stSystemOperationsTest, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stSystemOperationsTest/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stTimeConsuming, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stTimeConsuming/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stTransactionTest, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stTransactionTest/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stTransitionTest, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stTransitionTest/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stWalletTest, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stWalletTest/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stZeroCallsRevert, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stZeroCallsRevert/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stZeroCallsTest, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stZeroCallsTest/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stZeroKnowledge, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stZeroKnowledge/"} - declare_test!{Constantinople_BlockchainTests_GeneralStateTests_stZeroKnowledge2, "LegacyTests/Constantinople/BlockchainTests/GeneralStateTests/stZeroKnowledge2/"} -} diff --git a/ethcore/src/json_tests/difficulty.rs b/ethcore/src/json_tests/difficulty.rs index cf0c380b69e..d2db5c8cb53 100644 --- a/ethcore/src/json_tests/difficulty.rs +++ b/ethcore/src/json_tests/difficulty.rs @@ -29,6 +29,7 @@ pub fn json_difficulty_test( spec: Spec, start_stop_hook: &mut H ) -> Vec { + let mut ret = Vec::new(); let _ = env_logger::try_init(); let tests = DifficultyTest::load(json_data) .expect(&format!("Could not parse JSON difficulty test data from {}", path.display())); @@ -37,8 +38,6 @@ pub fn json_difficulty_test( for (name, test) in tests.into_iter() { start_stop_hook(&name, HookType::OnStart); - flushed_writeln!(" - {}...", name); - let mut parent_header = Header::new(); let block_number: u64 = test.current_block_number.into(); parent_header.set_number(block_number - 1); @@ -51,71 +50,14 @@ pub fn json_difficulty_test( header.set_timestamp(test.current_timestamp.into()); engine.populate_from_parent(&mut header, &parent_header); let expected_difficulty: U256 = test.current_difficulty.into(); - assert_eq!(header.difficulty(), &expected_difficulty); - flushed_writeln!("OK"); + if header.difficulty() == &expected_difficulty { + flushed_writeln!(" - difficulty: {}...OK",name); + } else { + flushed_writeln!(" - difficulty: {}...FAILED",name); + ret.push(format!("{}:{}",path.to_string_lossy(),name)); + } start_stop_hook(&name, HookType::OnStop); } - vec![] -} - -macro_rules! difficulty_json_test { - ( $spec:ident ) => { - - use std::path::Path; - use super::json_difficulty_test; - use tempfile::TempDir; - use json_tests::HookType; - - fn do_json_test(path: &Path, json_data: &[u8], h: &mut H) -> Vec { - let tempdir = TempDir::new().unwrap(); - json_difficulty_test(path, json_data, crate::spec::$spec(&tempdir.path()), h) - } - - } -} - -macro_rules! difficulty_json_test_nopath { - ( $spec:ident ) => { - use std::path::Path; - - use super::json_difficulty_test; - use json_tests::HookType; - - fn do_json_test(path: &Path, json_data: &[u8], h: &mut H) -> Vec { - json_difficulty_test(path, json_data, crate::spec::$spec(), h) - } - - } -} - -mod difficulty_test { - difficulty_json_test!(new_foundation); - declare_test!{DifficultyTests_difficulty, "BasicTests/difficulty.json"} -} - -mod difficulty_test_byzantium { - difficulty_json_test_nopath!(new_byzantium_test); - declare_test!{DifficultyTests_difficultyByzantium, "BasicTests/difficultyByzantium.json"} -} - -mod difficulty_test_foundation { - difficulty_json_test!(new_foundation); - declare_test!{DifficultyTests_difficultyMainNetwork, "BasicTests/difficultyMainNetwork.json"} -} - -// Disabling Ropsten diff tests; waiting for upstream ethereum/tests Constantinople update -//mod difficulty_test_ropsten { -// difficulty_json_test_nopath!(new_ropsten_test); -// declare_test!{DifficultyTests_difficultyRopsten, "BasicTests/difficultyRopsten.json"} -//} - -mod difficulty_test_frontier { - difficulty_json_test_nopath!(new_frontier_test); - declare_test!{DifficultyTests_difficultyFrontier, "BasicTests/difficultyFrontier.json"} -} - -mod difficulty_test_homestead { - difficulty_json_test_nopath!(new_homestead_test); - declare_test!{DifficultyTests_difficultyHomestead, "BasicTests/difficultyHomestead.json"} -} + ret +} \ No newline at end of file diff --git a/ethcore/src/json_tests/executive.rs b/ethcore/src/json_tests/executive.rs index 93620254d71..b5110d17119 100644 --- a/ethcore/src/json_tests/executive.rs +++ b/ethcore/src/json_tests/executive.rs @@ -235,7 +235,7 @@ impl<'a, T: 'a, V: 'a, B: 'a> Ext for TestExt<'a, T, V, B> } } -fn do_json_test( +pub fn do_json_test( path: &Path, json_data: &[u8], start_stop_hook: &mut H @@ -247,7 +247,6 @@ fn do_json_test( for (name, vm) in tests.into_iter() { start_stop_hook(&format!("{}", name), HookType::OnStart); - info!(target: "jsontests", "name: {:?}", name); let mut fail = false; let mut fail_unless = |cond: bool, s: &str | if !cond && !fail { @@ -354,26 +353,15 @@ fn do_json_test( } }; - start_stop_hook(&format!("{}", name), HookType::OnStop); - } + if fail { + println!(" - vm: {:?}...FAILED", name); + } else { + println!(" - vm: {:?}...OK", name); + } - for f in &failed { - error!("FAILED: {:?}", f); + start_stop_hook(&format!("{}", name), HookType::OnStop); } failed } -declare_test!{ExecutiveTests_vmArithmeticTest, "VMTests/vmArithmeticTest"} -declare_test!{ExecutiveTests_vmBitwiseLogicOperationTest, "VMTests/vmBitwiseLogicOperation"} -declare_test!{ExecutiveTests_vmBlockInfoTest, "VMTests/vmBlockInfoTest"} - // TODO [todr] Fails with Signal 11 when using JIT -declare_test!{ExecutiveTests_vmEnvironmentalInfoTest, "VMTests/vmEnvironmentalInfo"} -declare_test!{ExecutiveTests_vmIOandFlowOperationsTest, "VMTests/vmIOandFlowOperations"} -declare_test!{ExecutiveTests_vmLogTest, "VMTests/vmLogTest"} -declare_test!{heavy => ExecutiveTests_vmPerformance, "VMTests/vmPerformance"} -declare_test!{ExecutiveTests_vmPushDupSwapTest, "VMTests/vmPushDupSwapTest"} -declare_test!{ExecutiveTests_vmRandomTest, "VMTests/vmRandomTest"} -declare_test!{ExecutiveTests_vmSha3Test, "VMTests/vmSha3Test"} -declare_test!{ExecutiveTests_vmSystemOperationsTest, "VMTests/vmSystemOperations"} -declare_test!{ExecutiveTests_vmTests, "VMTests/vmTests"} diff --git a/ethcore/src/json_tests/macros.rs b/ethcore/src/json_tests/macros.rs index 4fdbeb0be89..0c23b8e9e6d 100644 --- a/ethcore/src/json_tests/macros.rs +++ b/ethcore/src/json_tests/macros.rs @@ -16,65 +16,6 @@ //! Helper macros for running the `JSON tests` -/// Declares a test: -/// -/// declare_test!(test_name, "path/to/folder/with/tests"); -/// -/// Declares a test but skip the named test files inside the folder (no extension): -/// -/// declare_test!(skip => ["a-test-file", "other-test-file"], test_name, "path/to/folder/with/tests"); -/// -/// NOTE: a skipped test is considered a passing test as far as `cargo test` is concerned. Normally -/// one test corresponds to a folder full of test files, each of which may contain many tests. -#[macro_export] -macro_rules! declare_test { - (skip => $arr: expr, $id: ident, $name: expr) => { - #[cfg(test)] - #[test] - #[allow(non_snake_case)] - fn $id() { - test!($name, $arr); - } - }; - (ignore => $id: ident, $name: expr) => { - #[cfg(test)] - #[ignore] - #[test] - #[allow(non_snake_case)] - fn $id() { - test!($name, []); - } - }; - (heavy => $id: ident, $name: expr) => { - #[cfg(test)] - #[cfg(feature = "test-heavy")] - #[test] - #[allow(non_snake_case)] - fn $id() { - test!($name, []); - } - }; - ($id: ident, $name: expr) => { - #[cfg(test)] - #[test] - #[allow(non_snake_case)] - fn $id() { - test!($name, []); - } - } -} - -#[cfg(test)] -macro_rules! test { - ($name: expr, $skip: expr) => { - $crate::json_tests::test_common::run_test_path( - std::path::Path::new(concat!("res/ethereum/tests/", $name)), - &$skip, - do_json_test, - &mut |_, _| () - ); - } -} /// Similar to `print!` but flushes stdout in order to ensure the output is emitted immediately. #[macro_export] diff --git a/ethcore/src/json_tests/mod.rs b/ethcore/src/json_tests/mod.rs index 023dc6e2c53..ceb49d1331c 100644 --- a/ethcore/src/json_tests/mod.rs +++ b/ethcore/src/json_tests/mod.rs @@ -21,17 +21,13 @@ mod macros; mod chain; mod executive; -mod skip; mod state; mod test_common; mod transaction; mod trie; - -#[cfg(test)] mod difficulty; +mod runner; pub use self::executive::run_test_path as run_executive_test_path; pub use self::executive::run_test_file as run_executive_test_file; pub use self::test_common::HookType; - -use self::skip::SKIP_TESTS; diff --git a/ethcore/src/json_tests/runner.rs b/ethcore/src/json_tests/runner.rs new file mode 100644 index 00000000000..b4904c600ff --- /dev/null +++ b/ethcore/src/json_tests/runner.rs @@ -0,0 +1,276 @@ +// Copyright 2015-2020 Parity Technologies (UK) Ltd. +// This file is part of Open Ethereum. + +// Open Ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Open Ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Open Ethereum. If not, see . +#![allow(dead_code)] + +use ethjson::test_helpers::ethspec::{ + ChainTests, DifficultyTests, EthereumTestSuite, ExecutiveTests, StateTests, TestChainSpec, + TestTrieSpec, TransactionTests, TrieTests, +}; +use globset::Glob; +use log::info; +use rayon::prelude::*; +use std::path::{Path, PathBuf}; +use tempfile::tempdir; +use trie::TrieSpec; +use walkdir::{DirEntry, WalkDir}; + +/// Result of tests execution +pub struct TestResult { + /// Number of success execution + pub success: usize, + /// Number of success execution + pub failed: Vec, +} + +impl TestResult { + /// Creates a new TestResult without results + pub fn zero() -> Self { + TestResult { + success: 0, + failed: Vec::new(), + } + } + /// Creates a new success TestResult + pub fn success() -> Self { + TestResult { + success: 1, + failed: Vec::new(), + } + } + /// Creates a new failed TestResult + pub fn failed(name: &str) -> Self { + TestResult { + success: 0, + failed: vec![name.to_string()], + } + } +} + +impl std::ops::Add for TestResult { + type Output = Self; + + fn add(self, other: Self) -> Self { + let mut mself = self; + mself.success += other.success; + mself.failed.extend_from_slice(&other.failed); + mself + } +} + +impl std::ops::AddAssign for TestResult { + fn add_assign(&mut self, other: Self) { + self.success += other.success; + self.failed.extend_from_slice(&other.failed); + } +} + +pub struct TestRunner(EthereumTestSuite); + +impl TestRunner { + /// Loads a new JSON Test suite + pub fn load(reader: R) -> Result + where + R: std::io::Read, + { + Ok(TestRunner(serde_json::from_reader(reader)?)) + } + + /// Run the tests with one thread + pub fn run_without_par(&self) -> TestResult { + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(1) + .build() + .unwrap(); + pool.install(|| self.run()) + } + + /// Run the tests + pub fn run(&self) -> TestResult { + let mut res = TestResult::zero(); + for t in &self.0.chain { + res += Self::run_chain_tests(&t); + } + for t in &self.0.state { + res += Self::run_state_tests(&t); + } + for t in &self.0.difficulty { + res += Self::run_difficuly_tests(&t); + } + for t in &self.0.executive { + res += Self::run_executive_tests(&t); + } + for t in &self.0.transaction { + res += Self::run_transaction_tests(&t); + } + for t in &self.0.trie { + res += Self::run_trie_tests(&t); + } + res + } + + fn find_json_files_recursive(path: &str) -> Vec { + WalkDir::new(path) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|e| e.file_name().to_string_lossy().ends_with(".json")) + .map(DirEntry::into_path) + .collect::>() + } + + fn run1(test: &T, base_path: &str, f: F) -> TestResult + where + T: Send + Sync, + F: Fn(&T, &Path, &[u8]) -> Vec + Send + Sync, + { + let result = Self::find_json_files_recursive(&base_path) + .into_par_iter() + .map(|path| { + info!("{:?}", path); + let json = std::fs::read(&path).unwrap(); + let faileds = f(test, &path, &json); + if faileds.len() > 0 { + TestResult::failed(&faileds.join(",")) + } else { + TestResult::success() + } + }) + .reduce(TestResult::zero, |a, b| a + b); + + if result.success + result.failed.len() == 0 { + panic!("There is no tests in the specified path {}", base_path); + } + result + } + + fn in_set(path: &Path, exprs: &[String]) -> bool { + for pathexp in exprs { + let glob = Glob::new(&pathexp) + .expect(&format!("cannot parse expression {}", pathexp)) + .compile_matcher(); + if glob.is_match(path) { + return true; + } + } + false + } + + fn run_chain_tests(test: &ChainTests) -> TestResult { + Self::run1( + test, + &test.path, + |test: &ChainTests, path: &Path, json: &[u8]| { + for skip in &test.skip { + if Self::in_set(&path, &skip.paths) { + println!(" - {} ..SKIPPED", path.to_string_lossy()); + return Vec::new(); + } + } + super::chain::json_chain_test(&test, &path, &json, &mut |_, _| {}) + }, + ) + } + + fn run_state_tests(test: &StateTests) -> TestResult { + Self::run1( + test, + &test.path, + |test: &StateTests, path: &Path, json: &[u8]| { + for skip in &test.skip { + if Self::in_set(&path, &skip.paths) { + println!(" - {} ..SKIPPED", path.to_string_lossy()); + return Vec::new(); + } + } + super::state::json_chain_test(&test, &path, &json, &mut |_, _| {}) + }, + ) + } + + fn run_difficuly_tests(test: &DifficultyTests) -> TestResult { + let mut acc = TestResult::zero(); + for path in &test.path { + acc += Self::run1( + test, + &path, + |test: &DifficultyTests, path: &Path, json: &[u8]| { + let spec = match &test.chainspec { + TestChainSpec::Foundation => { + crate::spec::new_foundation(&tempdir().unwrap().path()) + } + TestChainSpec::ByzantiumTest => crate::spec::new_byzantium_test(), + TestChainSpec::FrontierTest => crate::spec::new_frontier_test(), + TestChainSpec::HomesteadTest => crate::spec::new_homestead_test(), + }; + super::difficulty::json_difficulty_test(&path, &json, spec, &mut |_, _| {}) + }, + ) + } + acc + } + + fn run_executive_tests(test: &ExecutiveTests) -> TestResult { + Self::run1( + test, + &test.path, + |_: &ExecutiveTests, path: &Path, json: &[u8]| { + super::executive::do_json_test(&path, &json, &mut |_, _| {}) + }, + ) + } + + fn run_transaction_tests(test: &TransactionTests) -> TestResult { + Self::run1( + test, + &test.path, + |_: &TransactionTests, path: &Path, json: &[u8]| { + super::transaction::do_json_test(&path, &json, &mut |_, _| {}) + }, + ) + } + + fn run_trie_tests(test: &TrieTests) -> TestResult { + let mut acc = TestResult::zero(); + for path in &test.path { + acc += Self::run1(test, &path, |test: &TrieTests, path: &Path, json: &[u8]| { + let spec = match &test.triespec { + TestTrieSpec::Generic => TrieSpec::Generic, + TestTrieSpec::Secure => TrieSpec::Secure, + }; + super::trie::test_trie(&path, &json, spec, &mut |_, _| {}) + }); + } + acc + } +} + +#[cfg(test)] +mod test { + use super::TestRunner; + #[test] + fn test_ethereum_json_tests() { + let content = std::fs::read("res/ethereum/runner/full.json") + .expect("cannot open ethreum tests spec file"); + let runner = TestRunner::load(content.as_slice()).expect("cannot load content"); + let result = runner.run(); + println!( + "SUCCESS: {} FAILED: {} {:?}", + result.success, + result.failed.len(), + result.failed + ); + assert!(result.failed.len() == 0); + } +} diff --git a/ethcore/src/json_tests/skip.rs b/ethcore/src/json_tests/skip.rs deleted file mode 100644 index cd9b7e388f8..00000000000 --- a/ethcore/src/json_tests/skip.rs +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2015-2020 Parity Technologies (UK) Ltd. -// This file is part of Open Ethereum. - -// Open Ethereum is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// Open Ethereum is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with Open Ethereum. If not, see . - -//! State or blockchain tests to skip. -//! -//! Looks in the `ethereum/tests/test-issues/currents.json` file. This file contains two -//! collections, `block` and `state`, each with a different format to specify single tests to skip. -//! -//! To skip a blockchain test, add a JSON object to the `block` array, where `failing` names the -//! leaf folder with the tests to skip. The `subtests` array contains the names of the tests to skip. -//! Note that this does not handle duplicate folder names, e.g. `ValidBlocks/funTests/` and -//! `Something/funTests` would both be matched when `failing` is set to `funTests`. -//! -//! To skip a state test, add a JSON object to the `state` array. The `failing` works like for block -//! tests, but the `subtests` key is an object on the form: -//! "testName": {"subnumbers": [INDEX_OF_SKIPPED_SUBTESTS | "*"], "chain": "Blockchain name (informational)"}` -//! -//! Use the `reference` key to point to the github issue tracking to solution to the problem. -//! -//! Note: the `declare_test!` macro can also be use to skip tests, but skips entire files rather -//! than single tests. - -use ethjson::test_helpers::skip::SkipTests; - -lazy_static! { - pub static ref SKIP_TESTS: SkipTests = { - let skip_data = include_bytes!("../../res/ethereum/tests-issues/currents.json"); - SkipTests::load(&skip_data[..]).expect("JSON from disk is valid") - }; -} diff --git a/ethcore/src/json_tests/state.rs b/ethcore/src/json_tests/state.rs index 9ec3628f982..4bb22223380 100644 --- a/ethcore/src/json_tests/state.rs +++ b/ethcore/src/json_tests/state.rs @@ -20,19 +20,13 @@ use pod::PodState; use test_helpers::{EvmTestClient, EvmTestError, TransactErr, TransactSuccess}; use types::transaction::SignedTransaction; use vm::EnvInfo; -use super::SKIP_TESTS; +use ethjson::test_helpers::ethspec::StateTests; -#[allow(dead_code)] -fn skip_test(subname: &str, chain: &String, number: usize, is_legacy: bool) -> bool { - trace!(target: "json-tests", "[state, skip_test] subname: '{}', chain: '{}', number: {} legacy:{}", subname, chain, number, is_legacy); - let skip_set = if is_legacy { - &SKIP_TESTS.legacy_state - } else { - &SKIP_TESTS.state - }; - skip_set.iter().any(|state_test|{ - if let Some(subtest) = state_test.subtests.get(subname) { - trace!(target: "json-tests", "[state, skip_test] Maybe skipping {:?} (legacy:{})", subtest, is_legacy); +fn skip_test(test: &StateTests, subname: &str, chain: &String, number: usize) -> bool { + trace!(target: "json-tests", "[state, skip_test] subname: '{}', chain: '{}', number: {}", subname, chain, number); + test.skip.iter().any(|state_test|{ + if let Some(subtest) = state_test.names.get(subname) { + trace!(target: "json-tests", "[state, skip_test] Maybe skipping {:?}", subtest); chain == &subtest.chain && ( subtest.subnumbers[0] == "*" || @@ -44,8 +38,7 @@ fn skip_test(subname: &str, chain: &String, number: usize, is_legacy: bool) -> b }) } -#[allow(dead_code)] -pub fn json_chain_test(path: &Path, json_data: &[u8], start_stop_hook: &mut H, is_legacy: bool) -> Vec { +pub fn json_chain_test(state_test: &StateTests, path: &Path, json_data: &[u8], start_stop_hook: &mut H) -> Vec { let _ = ::env_logger::try_init(); let tests = ethjson::test_helpers::state::Test::load(json_data) .expect(&format!("Could not parse JSON state test data from {}", path.display())); @@ -69,8 +62,8 @@ pub fn json_chain_test(path: &Path, json_data: &[u8], }; for (i, state) in states.into_iter().enumerate() { - let info = format!(" - {} | {:?} ({}/{}) ...", name, spec_name, i + 1, total); - if skip_test(&name, &spec.name, i + 1, is_legacy) { + let info = format!(" - state: {} | {:?} ({}/{}) ...", name, spec_name, i + 1, total); + if skip_test(&state_test, &name, &spec.name, i + 1) { println!("{}: SKIPPED", info); continue; } @@ -113,140 +106,6 @@ pub fn json_chain_test(path: &Path, json_data: &[u8], start_stop_hook(&name, HookType::OnStop); } - if !failed.is_empty() { - println!("!!! {:?} tests failed.", failed.len()); - } failed } -#[cfg(test)] -mod state_tests { - use std::path::Path; - - use super::json_chain_test; - use json_tests::HookType; - - fn do_json_test(path: &Path, json_data: &[u8], h: &mut H) -> Vec { - json_chain_test(path, json_data, h, false) - } - - declare_test!{GeneralStateTest_stArgsZeroOneBalance, "GeneralStateTests/stArgsZeroOneBalance/"} - declare_test!{GeneralStateTest_stAttackTest, "GeneralStateTests/stAttackTest/"} - declare_test!{GeneralStateTest_stBadOpcodeTest, "GeneralStateTests/stBadOpcode/"} - declare_test!{GeneralStateTest_stBugs, "GeneralStateTests/stBugs/"} - declare_test!{GeneralStateTest_stCallCodes, "GeneralStateTests/stCallCodes/"} - declare_test!{GeneralStateTest_stCallCreateCallCodeTest, "GeneralStateTests/stCallCreateCallCodeTest/"} - declare_test!{GeneralStateTest_stCallDelegateCodesCallCodeHomestead, "GeneralStateTests/stCallDelegateCodesCallCodeHomestead/"} - declare_test!{GeneralStateTest_stCallDelegateCodesHomestead, "GeneralStateTests/stCallDelegateCodesHomestead/"} - declare_test!{GeneralStateTest_stChainId, "GeneralStateTests/stChainId/"} - declare_test!{GeneralStateTest_stChangedEIP150, "GeneralStateTests/stChangedEIP150/"} - declare_test!{GeneralStateTest_stCodeCopyTest, "GeneralStateTests/stCodeCopyTest/"} - declare_test!{GeneralStateTest_stCodeSizeLimit, "GeneralStateTests/stCodeSizeLimit/"} - declare_test!{GeneralStateTest_stCreate2Test, "GeneralStateTests/stCreate2/"} - declare_test!{GeneralStateTest_stCreateTest, "GeneralStateTests/stCreateTest/"} - declare_test!{GeneralStateTest_stDelegatecallTestHomestead, "GeneralStateTests/stDelegatecallTestHomestead/"} - declare_test!{GeneralStateTest_stEIP150singleCodeGasPrices, "GeneralStateTests/stEIP150singleCodeGasPrices/"} - declare_test!{GeneralStateTest_stEIP150Specific, "GeneralStateTests/stEIP150Specific/"} - declare_test!{GeneralStateTest_stEIP158Specific, "GeneralStateTests/stEIP158Specific/"} - declare_test!{GeneralStateTest_stEWASMTests, "GeneralStateTests/stEWASMTests/"} - declare_test!{GeneralStateTest_stExample, "GeneralStateTests/stExample/"} - declare_test!{GeneralStateTest_stExtCodeHash, "GeneralStateTests/stExtCodeHash/"} - declare_test!{GeneralStateTest_stHomesteadSpecific, "GeneralStateTests/stHomesteadSpecific/"} - declare_test!{GeneralStateTest_stInitCodeTest, "GeneralStateTests/stInitCodeTest/"} - declare_test!{GeneralStateTest_stLogTests, "GeneralStateTests/stLogTests/"} - declare_test!{GeneralStateTest_stMemExpandingEIP150Calls, "GeneralStateTests/stMemExpandingEIP150Calls/"} - declare_test!{heavy => GeneralStateTest_stMemoryStressTest, "GeneralStateTests/stMemoryStressTest/"} - declare_test!{GeneralStateTest_stMemoryTest, "GeneralStateTests/stMemoryTest/"} - declare_test!{GeneralStateTest_stNonZeroCallsTest, "GeneralStateTests/stNonZeroCallsTest/"} - declare_test!{GeneralStateTest_stPreCompiledContracts, "GeneralStateTests/stPreCompiledContracts/"} - declare_test!{GeneralStateTest_stPreCompiledContracts2, "GeneralStateTests/stPreCompiledContracts2/"} - declare_test!{heavy => GeneralStateTest_stQuadraticComplexityTest, "GeneralStateTests/stQuadraticComplexityTest/"} - declare_test!{GeneralStateTest_stRandom, "GeneralStateTests/stRandom/"} - declare_test!{GeneralStateTest_stRandom2, "GeneralStateTests/stRandom2/"} - declare_test!{GeneralStateTest_stRecursiveCreate, "GeneralStateTests/stRecursiveCreate/"} - declare_test!{GeneralStateTest_stRefundTest, "GeneralStateTests/stRefundTest/"} - declare_test!{GeneralStateTest_stReturnDataTest, "GeneralStateTests/stReturnDataTest/"} - declare_test!{GeneralStateTest_stRevertTest, "GeneralStateTests/stRevertTest/"} - declare_test!{GeneralStateTest_stSelfBalance, "GeneralStateTests/stSelfBalance/"} - declare_test!{GeneralStateTest_stShift, "GeneralStateTests/stShift/"} - declare_test!{GeneralStateTest_stSLoadTest, "GeneralStateTests/stSLoadTest/"} - declare_test!{GeneralStateTest_stSolidityTest, "GeneralStateTests/stSolidityTest/"} - declare_test!{GeneralStateTest_stSpecialTest, "GeneralStateTests/stSpecialTest/"} - declare_test!{GeneralStateTest_stSStoreTest, "GeneralStateTests/stSStoreTest/"} - declare_test!{GeneralStateTest_stStackTests, "GeneralStateTests/stStackTests/"} - declare_test!{GeneralStateTest_stStaticCall, "GeneralStateTests/stStaticCall/"} - declare_test!{GeneralStateTest_stSubroutine, "GeneralStateTests/stSubroutine/"} - declare_test!{GeneralStateTest_stSystemOperationsTest, "GeneralStateTests/stSystemOperationsTest/"} - declare_test!{GeneralStateTest_stTimeConsuming, "GeneralStateTests/stTimeConsuming/"} - declare_test!{GeneralStateTest_stTransactionTest, "GeneralStateTests/stTransactionTest/"} - declare_test!{GeneralStateTest_stTransitionTest, "GeneralStateTests/stTransitionTest/"} - declare_test!{GeneralStateTest_stWalletTest, "GeneralStateTests/stWalletTest/"} - declare_test!{GeneralStateTest_stZeroCallsRevert, "GeneralStateTests/stZeroCallsRevert/"} - declare_test!{GeneralStateTest_stZeroCallsTest, "GeneralStateTests/stZeroCallsTest/"} - declare_test!{GeneralStateTest_stZeroKnowledge, "GeneralStateTests/stZeroKnowledge/"} - declare_test!{GeneralStateTest_stZeroKnowledge2, "GeneralStateTests/stZeroKnowledge2/"} -} - -/// Legacy tests, still keeping it to check if there is any regression in blocks < Instambul HF -#[cfg(test)] -mod legacy_state_tests { - use std::path::Path; - - use super::json_chain_test; - use json_tests::HookType; - - fn do_json_test(path: &Path, json_data: &[u8], h: &mut H) -> Vec { - json_chain_test(path, json_data, h, true) - } - declare_test!{Constantinople_GeneralStateTests_stArgsZeroOneBalance,"LegacyTests/Constantinople/GeneralStateTests/stArgsZeroOneBalance/"} - declare_test!{Constantinople_GeneralStateTests_stAttackTest,"LegacyTests/Constantinople/GeneralStateTests/stAttackTest/"} - declare_test!{Constantinople_GeneralStateTests_stBadOpcode,"LegacyTests/Constantinople/GeneralStateTests/stBadOpcode/"} - declare_test!{Constantinople_GeneralStateTests_stBugs,"LegacyTests/Constantinople/GeneralStateTests/stBugs/"} - declare_test!{Constantinople_GeneralStateTests_stCallCodes,"LegacyTests/Constantinople/GeneralStateTests/stCallCodes/"} - declare_test!{Constantinople_GeneralStateTests_stCallCreateCallCodeTest,"LegacyTests/Constantinople/GeneralStateTests/stCallCreateCallCodeTest/"} - declare_test!{Constantinople_GeneralStateTests_stCallDelegateCodesCallCodeHomestead,"LegacyTests/Constantinople/GeneralStateTests/stCallDelegateCodesCallCodeHomestead/"} - declare_test!{Constantinople_GeneralStateTests_stCallDelegateCodesHomestead,"LegacyTests/Constantinople/GeneralStateTests/stCallDelegateCodesHomestead/"} - declare_test!{Constantinople_GeneralStateTests_stChangedEIP150,"LegacyTests/Constantinople/GeneralStateTests/stChangedEIP150/"} - declare_test!{Constantinople_GeneralStateTests_stCodeCopyTest,"LegacyTests/Constantinople/GeneralStateTests/stCodeCopyTest/"} - declare_test!{Constantinople_GeneralStateTests_stCodeSizeLimit,"LegacyTests/Constantinople/GeneralStateTests/stCodeSizeLimit/"} - declare_test!{Constantinople_GeneralStateTests_stCreate2,"LegacyTests/Constantinople/GeneralStateTests/stCreate2/"} - declare_test!{Constantinople_GeneralStateTests_stCreateTest,"LegacyTests/Constantinople/GeneralStateTests/stCreateTest/"} - declare_test!{Constantinople_GeneralStateTests_stDelegatecallTestHomestead,"LegacyTests/Constantinople/GeneralStateTests/stDelegatecallTestHomestead/"} - declare_test!{Constantinople_GeneralStateTests_stEIP150singleCodeGasPrices,"LegacyTests/Constantinople/GeneralStateTests/stEIP150singleCodeGasPrices/"} - declare_test!{Constantinople_GeneralStateTests_stEIP150Specific,"LegacyTests/Constantinople/GeneralStateTests/stEIP150Specific/"} - declare_test!{Constantinople_GeneralStateTests_stEIP158Specific,"LegacyTests/Constantinople/GeneralStateTests/stEIP158Specific/"} - declare_test!{Constantinople_GeneralStateTests_stEWASMTests,"LegacyTests/Constantinople/GeneralStateTests/stEWASMTests/"} - declare_test!{Constantinople_GeneralStateTests_stExample,"LegacyTests/Constantinople/GeneralStateTests/stExample/"} - declare_test!{Constantinople_GeneralStateTests_stExtCodeHash,"LegacyTests/Constantinople/GeneralStateTests/stExtCodeHash/"} - declare_test!{Constantinople_GeneralStateTests_stHomesteadSpecific,"LegacyTests/Constantinople/GeneralStateTests/stHomesteadSpecific/"} - declare_test!{Constantinople_GeneralStateTests_stInitCodeTest,"LegacyTests/Constantinople/GeneralStateTests/stInitCodeTest/"} - declare_test!{Constantinople_GeneralStateTests_stLogTests,"LegacyTests/Constantinople/GeneralStateTests/stLogTests/"} - declare_test!{Constantinople_GeneralStateTests_stMemExpandingEIP150Calls,"LegacyTests/Constantinople/GeneralStateTests/stMemExpandingEIP150Calls/"} - declare_test!{Constantinople_GeneralStateTests_stMemoryStressTest,"LegacyTests/Constantinople/GeneralStateTests/stMemoryStressTest/"} - declare_test!{Constantinople_GeneralStateTests_stMemoryTest,"LegacyTests/Constantinople/GeneralStateTests/stMemoryTest/"} - declare_test!{Constantinople_GeneralStateTests_stNonZeroCallsTest,"LegacyTests/Constantinople/GeneralStateTests/stNonZeroCallsTest/"} - declare_test!{Constantinople_GeneralStateTests_stPreCompiledContracts,"LegacyTests/Constantinople/GeneralStateTests/stPreCompiledContracts/"} - declare_test!{Constantinople_GeneralStateTests_stPreCompiledContracts2,"LegacyTests/Constantinople/GeneralStateTests/stPreCompiledContracts2/"} - declare_test!{Constantinople_GeneralStateTests_stQuadraticComplexityTest,"LegacyTests/Constantinople/GeneralStateTests/stQuadraticComplexityTest/"} - declare_test!{Constantinople_GeneralStateTests_stRandom,"LegacyTests/Constantinople/GeneralStateTests/stRandom/"} - declare_test!{Constantinople_GeneralStateTests_stRandom2,"LegacyTests/Constantinople/GeneralStateTests/stRandom2/"} - declare_test!{Constantinople_GeneralStateTests_stRecursiveCreate,"LegacyTests/Constantinople/GeneralStateTests/stRecursiveCreate/"} - declare_test!{Constantinople_GeneralStateTests_stRefundTest,"LegacyTests/Constantinople/GeneralStateTests/stRefundTest/"} - declare_test!{Constantinople_GeneralStateTests_stReturnDataTest,"LegacyTests/Constantinople/GeneralStateTests/stReturnDataTest/"} - declare_test!{Constantinople_GeneralStateTests_stRevertTest,"LegacyTests/Constantinople/GeneralStateTests/stRevertTest/"} - declare_test!{Constantinople_GeneralStateTests_stShift,"LegacyTests/Constantinople/GeneralStateTests/stShift/"} - declare_test!{Constantinople_GeneralStateTests_stSolidityTest,"LegacyTests/Constantinople/GeneralStateTests/stSolidityTest/"} - declare_test!{Constantinople_GeneralStateTests_stSpecialTest,"LegacyTests/Constantinople/GeneralStateTests/stSpecialTest/"} - declare_test!{Constantinople_GeneralStateTests_stSStoreTest,"LegacyTests/Constantinople/GeneralStateTests/stSStoreTest/"} - declare_test!{Constantinople_GeneralStateTests_stStackTests,"LegacyTests/Constantinople/GeneralStateTests/stStackTests/"} - declare_test!{Constantinople_GeneralStateTests_stStaticCall,"LegacyTests/Constantinople/GeneralStateTests/stStaticCall/"} - declare_test!{Constantinople_GeneralStateTests_stSystemOperationsTest,"LegacyTests/Constantinople/GeneralStateTests/stSystemOperationsTest/"} - declare_test!{Constantinople_GeneralStateTests_stTimeConsuming,"LegacyTests/Constantinople/GeneralStateTests/stTimeConsuming/"} - declare_test!{Constantinople_GeneralStateTests_stTransactionTest,"LegacyTests/Constantinople/GeneralStateTests/stTransactionTest/"} - declare_test!{Constantinople_GeneralStateTests_stTransitionTest,"LegacyTests/Constantinople/GeneralStateTests/stTransitionTest/"} - declare_test!{Constantinople_GeneralStateTests_stWalletTest,"LegacyTests/Constantinople/GeneralStateTests/stWalletTest/"} - declare_test!{Constantinople_GeneralStateTests_stZeroCallsRevert,"LegacyTests/Constantinople/GeneralStateTests/stZeroCallsRevert/"} - declare_test!{Constantinople_GeneralStateTests_stZeroCallsTest,"LegacyTests/Constantinople/GeneralStateTests/stZeroCallsTest/"} - declare_test!{Constantinople_GeneralStateTests_stZeroKnowledge,"LegacyTests/Constantinople/GeneralStateTests/stZeroKnowledge/"} - declare_test!{Constantinople_GeneralStateTests_stZeroKnowledge2,"LegacyTests/Constantinople/GeneralStateTests/stZeroKnowledge2/"} -} \ No newline at end of file diff --git a/ethcore/src/json_tests/transaction.rs b/ethcore/src/json_tests/transaction.rs index 97a4f48b23b..b348ee0f92e 100644 --- a/ethcore/src/json_tests/transaction.rs +++ b/ethcore/src/json_tests/transaction.rs @@ -26,8 +26,7 @@ use types::{ }; use machine::transaction_ext::Transaction; -#[allow(dead_code)] -fn do_json_test(path: &Path, json_data: &[u8], start_stop_hook: &mut H) -> Vec { +pub fn do_json_test(path: &Path, json_data: &[u8], start_stop_hook: &mut H) -> Vec { // Block number used to run the tests. // Make sure that all the specified features are activated. const BLOCK_NUMBER: u64 = 0x6ffffffffffffe; @@ -38,12 +37,13 @@ fn do_json_test(path: &Path, json_data: &[u8], start_s for (name, test) in tests.into_iter() { start_stop_hook(&name, HookType::OnStart); + println!(" - tx: {} ", name); + for (spec_name, result) in test.post_state { let spec = match EvmTestClient::fork_spec_from_json(&spec_name) { Some(spec) => spec, None => { - println!(" - {} | {:?} Ignoring tests because of missing spec", name, spec_name); - continue; + failed.push(format!("{}-{:?} (missing spec)", name, spec_name)); continue; } }; @@ -93,15 +93,4 @@ fn do_json_test(path: &Path, json_data: &[u8], start_s println!("FAILED: {:?}", f); } failed -} - -declare_test!{TransactionTests_ttAddress, "TransactionTests/ttAddress"} -declare_test!{TransactionTests_ttData, "TransactionTests/ttData"} -declare_test!{TransactionTests_ttGasLimit, "TransactionTests/ttGasLimit"} -declare_test!{TransactionTests_ttGasPrice, "TransactionTests/ttGasPrice"} -declare_test!{TransactionTests_ttNonce, "TransactionTests/ttNonce"} -declare_test!{TransactionTests_ttRSValue, "TransactionTests/ttRSValue"} -declare_test!{TransactionTests_ttSignature, "TransactionTests/ttSignature"} -declare_test!{TransactionTests_ttValue, "TransactionTests/ttValue"} -declare_test!{TransactionTests_ttVValue, "TransactionTests/ttVValue"} -declare_test!{TransactionTests_ttWrongRLP, "TransactionTests/ttWrongRLP"} +} \ No newline at end of file diff --git a/ethcore/src/json_tests/trie.rs b/ethcore/src/json_tests/trie.rs index f671f34de22..c737e45f841 100644 --- a/ethcore/src/json_tests/trie.rs +++ b/ethcore/src/json_tests/trie.rs @@ -22,12 +22,11 @@ use ethereum_types::H256; use super::HookType; -#[allow(dead_code)] -fn test_trie(path: &Path, json: &[u8], trie: TrieSpec, start_stop_hook: &mut H) -> Vec { +pub fn test_trie(path: &Path, json: &[u8], trie: TrieSpec, start_stop_hook: &mut H) -> Vec { let tests = ethjson::test_helpers::trie::Test::load(json) .expect(&format!("Could not parse JSON trie test data from {}", path.display())); let factory = TrieFactory::new(trie, ethtrie::Layout); - let mut result = vec![]; + let mut failed = vec![]; for (name, test) in tests.into_iter() { start_stop_hook(&name, HookType::OnStart); @@ -43,47 +42,14 @@ fn test_trie(path: &Path, json: &[u8], trie: TrieSpec, .expect(&format!("Trie test '{:?}' failed due to internal error", name)); } - if *t.root() != test.root.into() { - result.push(format!("Trie test '{:?}' failed.", name)); + if *t.root() == test.root.into() { + println!(" - trie: {}...OK", name); + } else { + println!(" - trie: {}...FAILED ({:?})",name,path); + failed.push(format!("{}", name)); } - start_stop_hook(&name, HookType::OnStop); } - for i in &result { - println!("FAILED: {}", i); - } - - result -} - -mod generic { - use std::path::Path; - use trie::TrieSpec; - - use super::HookType; - - #[allow(dead_code)] - fn do_json_test(path: &Path, json: &[u8], h: &mut H) -> Vec { - super::test_trie(path, json, TrieSpec::Generic, h) - } - - declare_test!{TrieTests_trietest, "TrieTests/trietest.json"} - declare_test!{TrieTests_trieanyorder, "TrieTests/trieanyorder.json"} -} - -mod secure { - use std::path::Path; - use trie::TrieSpec; - - use super::HookType; - - #[allow(dead_code)] - fn do_json_test(path: &Path, json: &[u8], h: &mut H) -> Vec { - super::test_trie(path, json, TrieSpec::Secure, h) - } - - declare_test!{TrieTests_hex_encoded_secure, "TrieTests/hex_encoded_securetrie_test.json"} - declare_test!{TrieTests_trietest_secure, "TrieTests/trietest_secureTrie.json"} - declare_test!{TrieTests_trieanyorder_secure, "TrieTests/trieanyorder_secureTrie.json"} + failed } diff --git a/ethcore/src/lib.rs b/ethcore/src/lib.rs index b21cd959b5f..d1014d6d468 100644 --- a/ethcore/src/lib.rs +++ b/ethcore/src/lib.rs @@ -59,6 +59,15 @@ extern crate using_queue; extern crate verification; extern crate vm; +#[cfg(feature = "json-tests")] +extern crate globset; + +#[cfg(feature = "json-tests")] +extern crate tempfile; + +#[cfg(feature = "json-tests")] +extern crate walkdir; + #[cfg(test)] extern crate account_db; #[cfg(test)] @@ -78,9 +87,6 @@ extern crate ethjson; extern crate kvdb_memorydb; #[cfg(any(test, feature = "kvdb-rocksdb"))] extern crate kvdb_rocksdb; -#[cfg(feature = "json-tests")] -#[macro_use] -extern crate lazy_static; #[cfg(any(test, feature = "test-helpers"))] extern crate pod; #[cfg(any(test, feature = "blooms-db"))] @@ -89,8 +95,6 @@ extern crate blooms_db; extern crate env_logger; #[cfg(test)] extern crate serde_json; -#[cfg(any(test, feature = "tempdir"))] -extern crate tempfile; #[macro_use] extern crate log; diff --git a/json/src/test_helpers/ethspec.rs b/json/src/test_helpers/ethspec.rs new file mode 100644 index 00000000000..85bfc88c822 --- /dev/null +++ b/json/src/test_helpers/ethspec.rs @@ -0,0 +1,122 @@ +use serde::Deserialize; +use std::collections::BTreeMap; + +/// Describes a github.com/ethereum/tests suite +#[derive(Debug, PartialEq, Deserialize)] +pub struct EthereumTestSuite { + /// Blockchain tests + pub chain: Vec, + /// State tests + pub state: Vec, + /// Difficulty tests + pub difficulty: Vec, + /// Executive tests + pub executive: Vec, + /// Transaction tests + pub transaction: Vec, + /// Trie tests + pub trie: Vec, +} + +/// Chain spec used in tests +#[derive(Debug, PartialEq, Deserialize)] +pub enum TestChainSpec { + /// Foundation + Foundation, + /// ByzantiumTest + ByzantiumTest, + /// FrontierTest + FrontierTest, + /// HomesteadTest + HomesteadTest, +} + +/// Kind of trie used in test +#[derive(Debug, PartialEq, Deserialize)] +pub enum TestTrieSpec { + /// Generic + Generic, + /// Secure + Secure, +} + +/// A set of blockchain tests +#[derive(Debug, PartialEq, Deserialize)] +pub struct ChainTests { + /// Path of the json tests + pub path: String, + /// Tests to skip + pub skip: Vec, +} + +/// Tests to skip in chain tests +#[derive(Debug, PartialEq, Deserialize)] +pub struct ChainTestSkip { + /// Issue reference. + pub reference: String, + /// Test names to skip + pub names: Vec, + /// Test paths to skip + pub paths: Vec, +} + +/// A set of state tests +#[derive(Debug, PartialEq, Deserialize)] +pub struct StateTests { + /// Path of the json tests + pub path: String, + /// Tests to skip + pub skip: Vec, +} + +/// State test to skip +#[derive(Debug, PartialEq, Deserialize)] +pub struct StateTestSkip{ + /// Issue reference. + pub reference: String, + /// Paths to skip + pub paths: Vec, + /// Test names to skip + pub names: BTreeMap +} + +/// State subtest to skip. +#[derive(Debug, PartialEq, Deserialize)] +pub struct StateSkipSubStates { + /// State test number of this item. Or '*' for all state. + pub subnumbers: Vec, + /// Chain for this items. + pub chain: String, +} + +/// A set of difficulty tests +#[derive(Debug, PartialEq, Deserialize)] +pub struct DifficultyTests { + /// Path of the json tests + pub path: Vec, + /// Chain spec to use + pub chainspec : TestChainSpec, +} + +/// A set of executive tests +#[derive(Debug, PartialEq, Deserialize)] +pub struct ExecutiveTests { + /// Path of the json tests + pub path: String, +} + +/// A set of transaction tests +#[derive(Debug, PartialEq, Deserialize)] +pub struct TransactionTests { + /// Path of the json tests + pub path: String, +} + +/// A set of trie tests +#[derive(Debug, PartialEq, Deserialize)] +pub struct TrieTests { + /// Path of the json tests + pub path: Vec, + /// Trie spec to use + pub triespec: TestTrieSpec, +} diff --git a/json/src/test_helpers/mod.rs b/json/src/test_helpers/mod.rs index 70a3b1e93e3..db0cc2cfc28 100644 --- a/json/src/test_helpers/mod.rs +++ b/json/src/test_helpers/mod.rs @@ -16,6 +16,8 @@ //! Test structures for JSON deserialization. +/// Ethereum core tests execution instruction +pub mod ethspec; /// Blockchain test helpers pub mod blockchain; /// Difficulty test helpers