From 69ef4fc159d32e257604b9c31a93b97377c28c78 Mon Sep 17 00:00:00 2001 From: Mario Rugiero Date: Tue, 25 Mar 2025 19:19:52 -0300 Subject: [PATCH 1/2] feat: initial migration to async APIs --- CHANGELOG.md | 4 + Cargo.lock | 5 + Cargo.toml | 1 + bench/criterion_benchmark.rs | 6 +- cmd/ef_tests/blockchain/Cargo.toml | 1 + cmd/ef_tests/blockchain/test_runner.rs | 11 +- cmd/ef_tests/blockchain/tests/cancun.rs | 6 +- cmd/ef_tests/blockchain/tests/prague.rs | 3 +- cmd/ef_tests/blockchain/tests/shanghai.rs | 3 +- cmd/ef_tests/state/Cargo.toml | 1 + cmd/ef_tests/state/runner/levm_runner.rs | 19 +- cmd/ef_tests/state/runner/mod.rs | 20 +- cmd/ef_tests/state/runner/revm_runner.rs | 37 +- cmd/ef_tests/state/tests/all.rs | 5 +- cmd/ef_tests/state/utils.rs | 8 +- cmd/ethrex/cli.rs | 8 +- cmd/ethrex/ethrex.rs | 2 +- cmd/ethrex/initializers.rs | 3 +- cmd/ethrex/l2.rs | 2 +- cmd/ethrex_l2/src/commands/stack.rs | 16 +- crates/blockchain/Cargo.toml | 1 + crates/blockchain/blockchain.rs | 44 ++- crates/blockchain/fork_choice.rs | 18 +- crates/blockchain/mempool.rs | 20 +- crates/blockchain/payload.rs | 15 +- crates/blockchain/smoke_test.rs | 112 +++--- crates/common/trie/db.rs | 2 +- crates/l2/prover/tests/perf_zkvm.rs | 4 +- .../l2/prover/zkvm/interface/sp1/Cargo.lock | 12 + crates/l2/sequencer/block_producer.rs | 15 +- crates/l2/utils/prover/save_state.rs | 8 +- crates/l2/utils/test_data_io.rs | 6 +- crates/networking/p2p/rlpx/connection.rs | 2 +- crates/networking/p2p/rlpx/eth/backend.rs | 5 +- .../networking/p2p/rlpx/eth/transactions.rs | 2 +- crates/networking/p2p/sync.rs | 45 +-- .../networking/p2p/sync/bytecode_fetcher.rs | 2 +- crates/networking/p2p/sync/state_healing.rs | 2 +- crates/networking/p2p/sync/state_sync.rs | 8 +- crates/networking/p2p/sync/storage_fetcher.rs | 12 +- crates/networking/p2p/sync/storage_healing.rs | 4 +- crates/networking/p2p/sync/trie_rebuild.rs | 6 +- .../rpc/engine/exchange_transition_config.rs | 2 +- crates/networking/rpc/engine/fork_choice.rs | 28 +- crates/networking/rpc/engine/mod.rs | 2 +- crates/networking/rpc/engine/payload.rs | 60 +-- crates/networking/rpc/eth/account.rs | 10 +- crates/networking/rpc/eth/block.rs | 18 +- crates/networking/rpc/eth/client.rs | 4 +- crates/networking/rpc/eth/fee_calculator.rs | 30 +- crates/networking/rpc/eth/fee_market.rs | 2 +- crates/networking/rpc/eth/filter.rs | 1 + crates/networking/rpc/eth/gas_price.rs | 58 +-- crates/networking/rpc/eth/logs.rs | 2 +- crates/networking/rpc/eth/max_priority_fee.rs | 58 +-- crates/networking/rpc/eth/mod.rs | 23 +- crates/networking/rpc/eth/transaction.rs | 20 +- crates/networking/rpc/l2/transaction.rs | 9 +- crates/networking/rpc/rpc.rs | 128 ++++--- crates/networking/rpc/utils.rs | 1 + crates/storage/Cargo.toml | 8 +- crates/storage/api.rs | 101 +++-- crates/storage/store.rs | 310 ++++++++++------ crates/storage/store_db/in_memory.rs | 106 ++++-- crates/storage/store_db/libmdbx.rs | 350 +++++++++++------- crates/storage/store_db/redb.rs | 326 ++++++++++------ crates/vm/levm/src/db/mod.rs | 2 +- crates/vm/levm/src/vm.rs | 2 +- 68 files changed, 1296 insertions(+), 871 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73c5f98af1..e13abfa7b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Perf +#### 2025-04-01 + +- Asyncify DB write APIs, as well as its users [#2336](https://github.com/lambdaclass/ethrex/pull/2336) + #### 2025-03-30 - Faster block import, use a slice instead of copy diff --git a/Cargo.lock b/Cargo.lock index 007afa123e..5924ae1b95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2742,6 +2742,7 @@ dependencies = [ "lazy_static", "serde", "serde_json", + "tokio", ] [[package]] @@ -2766,6 +2767,7 @@ dependencies = [ "serde_json", "spinoff", "thiserror 2.0.12", + "tokio", ] [[package]] @@ -3052,6 +3054,7 @@ dependencies = [ "serde_json", "sha3", "thiserror 2.0.12", + "tokio", "tracing", ] @@ -3323,6 +3326,7 @@ name = "ethrex-storage" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", "bytes", "ethereum-types", "ethrex-common", @@ -3337,6 +3341,7 @@ dependencies = [ "sha3", "tempdir", "thiserror 2.0.12", + "tokio", "tracing", ] diff --git a/Cargo.toml b/Cargo.toml index 905fa1801d..6341ad0be6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,7 @@ ethrex-prover = { path = "./crates/l2/prover" } tracing = { version = "0.1", features = ["log"] } tracing-subscriber = { version = "0.3.0", features = ["env-filter"] } +async-trait = "0.1.88" ethereum-types = { version = "0.15.1", features = ["serialize"] } serde = { version = "1.0.203", features = ["derive"] } serde_json = "1.0.117" diff --git a/bench/criterion_benchmark.rs b/bench/criterion_benchmark.rs index 17f7a4998c..4aa6f8445e 100644 --- a/bench/criterion_benchmark.rs +++ b/bench/criterion_benchmark.rs @@ -15,19 +15,19 @@ use tracing_subscriber::{filter::Directive, EnvFilter, FmtSubscriber}; fn block_import() { let data_dir = DEFAULT_DATADIR; set_datadir(data_dir); - remove_db(data_dir); let evm_engine = "revm".to_owned().try_into().unwrap(); let network = "../../test_data/genesis-l2-ci.json"; - import_blocks( + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(import_blocks( "../../test_data/l2-1k-erc20.rlp", data_dir, network, evm_engine, - ); + )); } pub fn criterion_benchmark(c: &mut Criterion) { diff --git a/cmd/ef_tests/blockchain/Cargo.toml b/cmd/ef_tests/blockchain/Cargo.toml index d259af2a80..868e976113 100644 --- a/cmd/ef_tests/blockchain/Cargo.toml +++ b/cmd/ef_tests/blockchain/Cargo.toml @@ -15,6 +15,7 @@ serde_json.workspace = true bytes.workspace = true hex.workspace = true lazy_static.workspace = true +tokio.workspace = true [dev-dependencies] datatest-stable = "0.2.9" diff --git a/cmd/ef_tests/blockchain/test_runner.rs b/cmd/ef_tests/blockchain/test_runner.rs index 63333b36ff..e3a5dd1a34 100644 --- a/cmd/ef_tests/blockchain/test_runner.rs +++ b/cmd/ef_tests/blockchain/test_runner.rs @@ -8,14 +8,14 @@ use ethrex_common::types::{ use ethrex_rlp::decode::RLPDecode; use ethrex_storage::{EngineType, Store}; -pub fn run_ef_test(test_key: &str, test: &TestUnit) { +pub async fn run_ef_test(test_key: &str, test: &TestUnit) { // check that the decoded genesis block header matches the deserialized one let genesis_rlp = test.genesis_rlp.clone(); let decoded_block = CoreBlock::decode(&genesis_rlp).unwrap(); let genesis_block_header = CoreBlockHeader::from(test.genesis_block_header.clone()); assert_eq!(decoded_block.header, genesis_block_header); - let store = build_store_for_test(test); + let store = build_store_for_test(test).await; // Check world_state check_prestate_against_db(test_key, test, &store); @@ -33,7 +33,7 @@ pub fn run_ef_test(test_key: &str, test: &TestUnit) { let hash = block.hash(); // Attempt to add the block as the head of the chain - let chain_result = blockchain.add_block(block); + let chain_result = blockchain.add_block(block).await; match chain_result { Err(error) => { assert!( @@ -50,7 +50,7 @@ pub fn run_ef_test(test_key: &str, test: &TestUnit) { test_key, block_fixture.expect_exception.clone().unwrap() ); - apply_fork_choice(&store, hash, hash, hash).unwrap(); + apply_fork_choice(&store, hash, hash, hash).await.unwrap(); } } } @@ -82,12 +82,13 @@ pub fn parse_test_file(path: &Path) -> HashMap { } /// Creats a new in-memory store and adds the genesis state -pub fn build_store_for_test(test: &TestUnit) -> Store { +pub async fn build_store_for_test(test: &TestUnit) -> Store { let store = Store::new("store.db", EngineType::InMemory).expect("Failed to build DB for testing"); let genesis = test.get_genesis(); store .add_initial_state(genesis) + .await .expect("Failed to add genesis state"); store } diff --git a/cmd/ef_tests/blockchain/tests/cancun.rs b/cmd/ef_tests/blockchain/tests/cancun.rs index 29017235b2..9b4feeac17 100644 --- a/cmd/ef_tests/blockchain/tests/cancun.rs +++ b/cmd/ef_tests/blockchain/tests/cancun.rs @@ -14,6 +14,7 @@ use std::path::Path; // test or set of tests fn parse_and_execute_until_cancun(path: &Path) -> datatest_stable::Result<()> { + let rt = tokio::runtime::Runtime::new().unwrap(); let tests = parse_test_file(path); for (test_key, test) in tests { @@ -22,7 +23,7 @@ fn parse_and_execute_until_cancun(path: &Path) -> datatest_stable::Result<()> { // them. This produces false positives continue; } - run_ef_test(&test_key, &test); + rt.block_on(run_ef_test(&test_key, &test)); } Ok(()) @@ -30,6 +31,7 @@ fn parse_and_execute_until_cancun(path: &Path) -> datatest_stable::Result<()> { #[allow(dead_code)] fn parse_and_execute_all(path: &Path) -> datatest_stable::Result<()> { + let rt = tokio::runtime::Runtime::new().unwrap(); let tests = parse_test_file(path); for (test_key, test) in tests { @@ -37,7 +39,7 @@ fn parse_and_execute_all(path: &Path) -> datatest_stable::Result<()> { // These tests fall into the not supported forks. This produces false positives continue; } - run_ef_test(&test_key, &test); + rt.block_on(run_ef_test(&test_key, &test)); } Ok(()) diff --git a/cmd/ef_tests/blockchain/tests/prague.rs b/cmd/ef_tests/blockchain/tests/prague.rs index 4d387cf759..0662121b5f 100644 --- a/cmd/ef_tests/blockchain/tests/prague.rs +++ b/cmd/ef_tests/blockchain/tests/prague.rs @@ -13,6 +13,7 @@ const SKIPPED_TEST: [&str; 2] = [ #[allow(dead_code)] fn parse_and_execute(path: &Path) -> datatest_stable::Result<()> { + let rt = tokio::runtime::Runtime::new().unwrap(); let tests = parse_test_file(path); for (test_key, test) in tests { @@ -21,7 +22,7 @@ fn parse_and_execute(path: &Path) -> datatest_stable::Result<()> { continue; } - run_ef_test(&test_key, &test); + rt.block_on(run_ef_test(&test_key, &test)); } Ok(()) } diff --git a/cmd/ef_tests/blockchain/tests/shanghai.rs b/cmd/ef_tests/blockchain/tests/shanghai.rs index 8b1624a86e..2eea5fbc85 100644 --- a/cmd/ef_tests/blockchain/tests/shanghai.rs +++ b/cmd/ef_tests/blockchain/tests/shanghai.rs @@ -6,6 +6,7 @@ use ef_tests_blockchain::{ }; fn parse_and_execute(path: &Path) -> datatest_stable::Result<()> { + let rt = tokio::runtime::Runtime::new().unwrap(); let tests = parse_test_file(path); for (test_key, test) in tests { @@ -14,7 +15,7 @@ fn parse_and_execute(path: &Path) -> datatest_stable::Result<()> { continue; } - run_ef_test(&test_key, &test); + rt.block_on(run_ef_test(&test_key, &test)); } Ok(()) } diff --git a/cmd/ef_tests/state/Cargo.toml b/cmd/ef_tests/state/Cargo.toml index 8aa67dd478..79866bb290 100644 --- a/cmd/ef_tests/state/Cargo.toml +++ b/cmd/ef_tests/state/Cargo.toml @@ -29,6 +29,7 @@ revm = { version = "19.0.0", features = [ "optional_no_base_fee", "optional_block_gas_limit", ], default-features = false } +tokio.workspace = true [dev-dependencies] hex = "0.4.3" diff --git a/cmd/ef_tests/state/runner/levm_runner.rs b/cmd/ef_tests/state/runner/levm_runner.rs index dc7829ae08..13e2dd786b 100644 --- a/cmd/ef_tests/state/runner/levm_runner.rs +++ b/cmd/ef_tests/state/runner/levm_runner.rs @@ -18,7 +18,7 @@ use ethrex_vm::backends; use keccak_hash::keccak; use std::collections::HashMap; -pub fn run_ef_test(test: &EFTest) -> Result { +pub async fn run_ef_test(test: &EFTest) -> Result { // There are some tests that don't have a hash, unwrap will panic let hash = test ._info @@ -35,7 +35,7 @@ pub fn run_ef_test(test: &EFTest) -> Result { if !test.post.has_vector_for_fork(vector, *fork) { continue; } - match run_ef_test_tx(vector, test, fork) { + match run_ef_test_tx(vector, test, fork).await { Ok(_) => continue, Err(EFTestRunnerError::VMInitializationFailed(reason)) => { ef_test_report_fork.register_vm_initialization_failure(reason, *vector); @@ -78,16 +78,16 @@ pub fn run_ef_test(test: &EFTest) -> Result { Ok(ef_test_report) } -pub fn run_ef_test_tx( +pub async fn run_ef_test_tx( vector: &TestVector, test: &EFTest, fork: &Fork, ) -> Result<(), EFTestRunnerError> { - let mut db = utils::load_initial_state_levm(test); + let mut db = utils::load_initial_state_levm(test).await; let mut levm = prepare_vm_for_tx(vector, test, fork, &mut db)?; ensure_pre_state(&levm, test)?; let levm_execution_result = levm.execute(); - ensure_post_state(&levm_execution_result, vector, test, fork, &mut db)?; + ensure_post_state(&levm_execution_result, vector, test, fork, &mut db).await?; Ok(()) } @@ -282,7 +282,7 @@ fn exception_is_expected( }) } -pub fn ensure_post_state( +pub async fn ensure_post_state( levm_execution_result: &Result, vector: &TestVector, test: &EFTest, @@ -326,7 +326,7 @@ pub fn ensure_post_state( .to_owned(), ) })?; - let pos_state_root = post_state_root(&levm_account_updates, test); + let pos_state_root = post_state_root(&levm_account_updates, test).await; let expected_post_state_root_hash = test.post.vector_post_value(vector, *fork).hash; if expected_post_state_root_hash != pos_state_root { @@ -381,12 +381,13 @@ pub fn ensure_post_state( Ok(()) } -pub fn post_state_root(account_updates: &[AccountUpdate], test: &EFTest) -> H256 { - let (initial_state, block_hash) = utils::load_initial_state(test); +pub async fn post_state_root(account_updates: &[AccountUpdate], test: &EFTest) -> H256 { + let (initial_state, block_hash) = utils::load_initial_state(test).await; initial_state .database() .unwrap() .apply_account_updates(block_hash, account_updates) + .await .unwrap() .unwrap() } diff --git a/cmd/ef_tests/state/runner/mod.rs b/cmd/ef_tests/state/runner/mod.rs index b6bf12d440..1d4e9cbc3e 100644 --- a/cmd/ef_tests/state/runner/mod.rs +++ b/cmd/ef_tests/state/runner/mod.rs @@ -70,27 +70,27 @@ pub struct EFTestRunnerOptions { pub revm: bool, } -pub fn run_ef_tests( +pub async fn run_ef_tests( ef_tests: Vec, opts: &EFTestRunnerOptions, ) -> Result<(), EFTestRunnerError> { let mut reports = report::load()?; if reports.is_empty() { if opts.revm { - run_with_revm(&mut reports, &ef_tests, opts)?; + run_with_revm(&mut reports, &ef_tests, opts).await?; return Ok(()); } else { - run_with_levm(&mut reports, &ef_tests, opts)?; + run_with_levm(&mut reports, &ef_tests, opts).await?; } } if opts.summary { return Ok(()); } - re_run_with_revm(&mut reports, &ef_tests, opts)?; + re_run_with_revm(&mut reports, &ef_tests, opts).await?; write_report(&reports) } -fn run_with_levm( +async fn run_with_levm( reports: &mut Vec, ef_tests: &[EFTest], opts: &EFTestRunnerOptions, @@ -113,7 +113,7 @@ fn run_with_levm( if !opts.spinner && opts.verbose { println!("Running test: {:?}", test.name); } - let ef_test_report = match levm_runner::run_ef_test(test) { + let ef_test_report = match levm_runner::run_ef_test(test).await { Ok(ef_test_report) => ef_test_report, Err(EFTestRunnerError::Internal(err)) => return Err(EFTestRunnerError::Internal(err)), non_internal_errors => { @@ -154,7 +154,7 @@ fn run_with_levm( } /// ### Runs all tests with REVM -fn run_with_revm( +async fn run_with_revm( reports: &mut Vec, ef_tests: &[EFTest], opts: &EFTestRunnerOptions, @@ -183,7 +183,7 @@ fn run_with_revm( ), opts.spinner, ); - let ef_test_report = match revm_runner::_run_ef_test_revm(test) { + let ef_test_report = match revm_runner::_run_ef_test_revm(test).await { Ok(ef_test_report) => ef_test_report, Err(EFTestRunnerError::Internal(err)) => return Err(EFTestRunnerError::Internal(err)), non_internal_errors => { @@ -210,7 +210,7 @@ fn run_with_revm( Ok(()) } -fn re_run_with_revm( +async fn re_run_with_revm( reports: &mut [EFTestReport], ef_tests: &[EFTest], opts: &EFTestRunnerOptions, @@ -262,7 +262,7 @@ fn re_run_with_revm( }) .unwrap(), failed_test_report, - ) { + ).await { Ok(re_run_report) => { failed_test_report.register_re_run_report(re_run_report.clone()); } diff --git a/cmd/ef_tests/state/runner/revm_runner.rs b/cmd/ef_tests/state/runner/revm_runner.rs index 0bbd978f52..af49744c8d 100644 --- a/cmd/ef_tests/state/runner/revm_runner.rs +++ b/cmd/ef_tests/state/runner/revm_runner.rs @@ -32,7 +32,7 @@ use revm::{ }; use std::collections::{HashMap, HashSet}; -pub fn re_run_failed_ef_test( +pub async fn re_run_failed_ef_test( test: &EFTest, failed_test_report: &EFTestReport, ) -> Result { @@ -43,7 +43,7 @@ pub fn re_run_failed_ef_test( match vector_failure { // We only want to re-run tests that failed in the post-state validation. EFTestRunnerError::FailedToEnsurePostState(transaction_report, _, levm_cache) => { - match re_run_failed_ef_test_tx(levm_cache.clone(), vector, test, transaction_report, &mut re_run_report, fork) { + match re_run_failed_ef_test_tx(levm_cache.clone(), vector, test, transaction_report, &mut re_run_report, fork).await { Ok(_) => continue, Err(EFTestRunnerError::VMInitializationFailed(reason)) => { return Err(EFTestRunnerError::Internal(InternalError::ReRunInternal( @@ -75,7 +75,7 @@ pub fn re_run_failed_ef_test( Ok(re_run_report) } -pub fn re_run_failed_ef_test_tx( +pub async fn re_run_failed_ef_test_tx( levm_cache: HashMap, vector: &TestVector, test: &EFTest, @@ -83,7 +83,7 @@ pub fn re_run_failed_ef_test_tx( re_run_report: &mut TestReRunReport, fork: &Fork, ) -> Result<(), EFTestRunnerError> { - let (mut state, _block_hash) = load_initial_state(test); + let (mut state, _block_hash) = load_initial_state(test).await; let mut revm = prepare_revm_for_tx(&mut state, vector, test, fork)?; if !test.post.has_vector_for_fork(vector, *fork) { return Ok(()); @@ -97,7 +97,7 @@ pub fn re_run_failed_ef_test_tx( re_run_report, fork, )?; - ensure_post_state(levm_cache, vector, &mut state, test, re_run_report, fork)?; + ensure_post_state(levm_cache, vector, &mut state, test, re_run_report, fork).await?; Ok(()) } @@ -310,7 +310,7 @@ pub fn compare_levm_revm_execution_results( Ok(()) } -pub fn ensure_post_state( +pub async fn ensure_post_state( levm_cache: HashMap, vector: &TestVector, revm_state: &mut EvmState, @@ -322,7 +322,7 @@ pub fn ensure_post_state( Some(_expected_exception) => {} // We only want to compare account updates when no exception is expected. None => { - let mut db = load_initial_state_levm(test); + let mut db = load_initial_state_levm(test).await; db.cache = levm_cache; let levm_account_updates = backends::levm::LEVM::get_state_transitions(&mut db, *fork) .map_err(|_| { @@ -335,7 +335,8 @@ pub fn ensure_post_state( fork, &levm_account_updates, &revm_account_updates, - ); + ) + .await; re_run_report.register_account_updates_report(*vector, account_updates_report, *fork); } } @@ -343,15 +344,15 @@ pub fn ensure_post_state( Ok(()) } -pub fn compare_levm_revm_account_updates( +pub async fn compare_levm_revm_account_updates( vector: &TestVector, test: &EFTest, fork: &Fork, levm_account_updates: &[AccountUpdate], revm_account_updates: &[AccountUpdate], ) -> ComparisonReport { - let levm_post_state_root = post_state_root(levm_account_updates, test); - let revm_post_state_root = post_state_root(revm_account_updates, test); + let levm_post_state_root = post_state_root(levm_account_updates, test).await; + let revm_post_state_root = post_state_root(revm_account_updates, test).await; let mut initial_accounts: HashMap = test .pre .0 @@ -411,7 +412,7 @@ pub fn compare_levm_revm_account_updates( } } -pub fn _run_ef_test_revm(test: &EFTest) -> Result { +pub async fn _run_ef_test_revm(test: &EFTest) -> Result { let hash = test ._info .generated_test_hash @@ -426,7 +427,7 @@ pub fn _run_ef_test_revm(test: &EFTest) -> Result continue, Err(EFTestRunnerError::VMInitializationFailed(reason)) => { ef_test_report_fork.register_vm_initialization_failure(reason, *vector); @@ -471,22 +472,22 @@ pub fn _run_ef_test_revm(test: &EFTest) -> Result Result<(), EFTestRunnerError> { - let (mut state, _block_hash) = load_initial_state(test); + let (mut state, _block_hash) = load_initial_state(test).await; let mut revm = prepare_revm_for_tx(&mut state, vector, test, fork)?; let revm_execution_result = revm.transact_commit(); drop(revm); // Need to drop the state mutable reference. - _ensure_post_state_revm(revm_execution_result, vector, test, &mut state, fork)?; + _ensure_post_state_revm(revm_execution_result, vector, test, &mut state, fork).await?; Ok(()) } -pub fn _ensure_post_state_revm( +pub async fn _ensure_post_state_revm( revm_execution_result: Result>, vector: &TestVector, test: &EFTest, @@ -516,7 +517,7 @@ pub fn _ensure_post_state_revm( None => { let revm_account_updates = backends::revm::REVM::get_state_transitions(revm_state); - let pos_state_root = post_state_root(&revm_account_updates, test); + let pos_state_root = post_state_root(&revm_account_updates, test).await; let expected_post_state_root_hash = test.post.vector_post_value(vector, *fork).hash; if expected_post_state_root_hash != pos_state_root { diff --git a/cmd/ef_tests/state/tests/all.rs b/cmd/ef_tests/state/tests/all.rs index 25af6323b6..9c09e709bb 100644 --- a/cmd/ef_tests/state/tests/all.rs +++ b/cmd/ef_tests/state/tests/all.rs @@ -5,9 +5,10 @@ use ef_tests_state::{ }; use std::error::Error; -fn main() -> Result<(), Box> { +#[tokio::main] +async fn main() -> Result<(), Box> { let opts = EFTestRunnerOptions::parse(); let ef_tests = parser::parse_ef_tests(&opts)?; - runner::run_ef_tests(ef_tests, &opts)?; + runner::run_ef_tests(ef_tests, &opts).await?; Ok(()) } diff --git a/cmd/ef_tests/state/utils.rs b/cmd/ef_tests/state/utils.rs index 3f520c3fd2..c968047700 100644 --- a/cmd/ef_tests/state/utils.rs +++ b/cmd/ef_tests/state/utils.rs @@ -14,11 +14,11 @@ use ethrex_vm::{ use spinoff::Spinner; /// Loads initial state, used for REVM as it contains EvmState. -pub fn load_initial_state(test: &EFTest) -> (EvmState, H256) { +pub async fn load_initial_state(test: &EFTest) -> (EvmState, H256) { let genesis = Genesis::from(test); let storage = Store::new("./temp", EngineType::InMemory).expect("Failed to create Store"); - storage.add_initial_state(genesis.clone()).unwrap(); + storage.add_initial_state(genesis.clone()).await.unwrap(); ( evm_state( @@ -30,11 +30,11 @@ pub fn load_initial_state(test: &EFTest) -> (EvmState, H256) { } /// Loads initial state, function for LEVM as it does not require EvmState -pub fn load_initial_state_levm(test: &EFTest) -> GeneralizedDatabase { +pub async fn load_initial_state_levm(test: &EFTest) -> GeneralizedDatabase { let genesis = Genesis::from(test); let storage = Store::new("./temp", EngineType::InMemory).expect("Failed to create Store"); - storage.add_initial_state(genesis.clone()).unwrap(); + storage.add_initial_state(genesis.clone()).await.unwrap(); let block_hash = genesis.get_block().header.compute_block_hash(); diff --git a/cmd/ethrex/cli.rs b/cmd/ethrex/cli.rs index 4d82f87ac1..5f1d938ba4 100644 --- a/cmd/ethrex/cli.rs +++ b/cmd/ethrex/cli.rs @@ -233,7 +233,7 @@ impl Subcommand { .as_ref() .expect("--network is required and it was not provided"); - import_blocks(&path, &opts.datadir, network, opts.evm); + import_blocks(&path, &opts.datadir, network, opts.evm).await; } #[cfg(any(feature = "l2", feature = "based"))] Subcommand::L2(command) => command.run().await?, @@ -255,10 +255,10 @@ pub fn remove_db(datadir: &str) { } } -pub fn import_blocks(path: &str, data_dir: &str, network: &str, evm: EvmEngine) { +pub async fn import_blocks(path: &str, data_dir: &str, network: &str, evm: EvmEngine) { let data_dir = set_datadir(data_dir); - let store = init_store(&data_dir, network); + let store = init_store(&data_dir, network).await; let blockchain = init_blockchain(evm, store); @@ -279,5 +279,5 @@ pub fn import_blocks(path: &str, data_dir: &str, network: &str, evm: EvmEngine) info!("Importing blocks from chain file: {path}"); utils::read_chain_file(path) }; - blockchain.import_blocks(&blocks); + blockchain.import_blocks(&blocks).await; } diff --git a/cmd/ethrex/ethrex.rs b/cmd/ethrex/ethrex.rs index 9580e4c7e5..d5100f1377 100644 --- a/cmd/ethrex/ethrex.rs +++ b/cmd/ethrex/ethrex.rs @@ -29,7 +29,7 @@ async fn main() -> eyre::Result<()> { let network = get_network(&opts); - let store = init_store(&data_dir, &network); + let store = init_store(&data_dir, &network).await; let blockchain = init_blockchain(opts.evm, store.clone()); diff --git a/cmd/ethrex/initializers.rs b/cmd/ethrex/initializers.rs index 77201306bd..942c48aa96 100644 --- a/cmd/ethrex/initializers.rs +++ b/cmd/ethrex/initializers.rs @@ -63,7 +63,7 @@ pub fn init_metrics(opts: &Options, tracker: TaskTracker) { tracker.spawn(metrics_api); } -pub fn init_store(data_dir: &str, network: &str) -> Store { +pub async fn init_store(data_dir: &str, network: &str) -> Store { let path = PathBuf::from(data_dir); let store = if path.ends_with("memory") { Store::new(data_dir, EngineType::InMemory).expect("Failed to create Store") @@ -84,6 +84,7 @@ pub fn init_store(data_dir: &str, network: &str) -> Store { let genesis = read_genesis_file(network); store .add_initial_state(genesis.clone()) + .await .expect("Failed to create genesis block"); store } diff --git a/cmd/ethrex/l2.rs b/cmd/ethrex/l2.rs index 0c7a50825d..6ce3c87aa8 100644 --- a/cmd/ethrex/l2.rs +++ b/cmd/ethrex/l2.rs @@ -99,7 +99,7 @@ impl Command { let network = get_network(&opts.node_opts); - let store = init_store(&data_dir, &network); + let store = init_store(&data_dir, &network).await; let blockchain = init_blockchain(opts.node_opts.evm, store.clone()); diff --git a/cmd/ethrex_l2/src/commands/stack.rs b/cmd/ethrex_l2/src/commands/stack.rs index 92dcfe7018..bb736e1c8f 100644 --- a/cmd/ethrex_l2/src/commands/stack.rs +++ b/cmd/ethrex_l2/src/commands/stack.rs @@ -303,7 +303,8 @@ impl Command { store_path.to_str().expect("Invalid store path"), EngineType::Libmdbx, genesis.to_str().expect("Invalid genesis path"), - )?; + ) + .await?; let genesis_header = store.get_block_header(0)?.expect("Genesis block not found"); let genesis_block_hash = genesis_header.compute_block_hash(); @@ -328,6 +329,7 @@ impl Command { new_trie = store .apply_account_updates_from_trie(new_trie, &account_updates) + .await .expect("Error applying account updates"); let new_block = BlockHeader { @@ -339,15 +341,19 @@ impl Command { }; let new_block_hash = new_block.compute_block_hash(); - store.add_block_header(new_block_hash, new_block)?; - store.add_block_number(new_block_hash, last_number + 1)?; - store.set_canonical_block(last_number + 1, new_block_hash)?; + store.add_block_header(new_block_hash, new_block).await?; + store + .add_block_number(new_block_hash, last_number + 1) + .await?; + store + .set_canonical_block(last_number + 1, new_block_hash) + .await?; last_number += 1; last_hash = new_block_hash; } - store.update_latest_block_number(last_number)?; + store.update_latest_block_number(last_number).await?; } } Ok(()) diff --git a/crates/blockchain/Cargo.toml b/crates/blockchain/Cargo.toml index 9660f02519..9b7c24027b 100644 --- a/crates/blockchain/Cargo.toml +++ b/crates/blockchain/Cargo.toml @@ -24,6 +24,7 @@ ethrex-metrics = { path = "./metrics", default-features = false } [dev-dependencies] serde_json.workspace = true hex = "0.4.3" +tokio.workspace = true [lib] path = "./blockchain.rs" diff --git a/crates/blockchain/blockchain.rs b/crates/blockchain/blockchain.rs index 9f77dce405..e29b747b83 100644 --- a/crates/blockchain/blockchain.rs +++ b/crates/blockchain/blockchain.rs @@ -63,11 +63,11 @@ impl Blockchain { } /// Executes a block withing a new vm instance and state - fn execute_block(&self, block: &Block) -> Result { + async fn execute_block(&self, block: &Block) -> Result { // Validate if it can be the new head and find the parent let Ok(parent_header) = find_parent_header(&block.header, &self.storage) else { // If the parent is not present, we store it as pending. - self.storage.add_pending_block(block.clone())?; + self.storage.add_pending_block(block.clone()).await?; return Err(ChainError::ParentNotFound); }; let chain_config = self.storage.get_chain_config()?; @@ -111,7 +111,7 @@ impl Blockchain { Ok(execution_result) } - pub fn store_block( + pub async fn store_block( &self, block: &Block, execution_result: BlockExecutionResult, @@ -119,7 +119,8 @@ impl Blockchain { // Apply the account updates over the last block's state and compute the new state root let new_state_root = self .storage - .apply_account_updates(block.header.parent_hash, &execution_result.account_updates)? + .apply_account_updates(block.header.parent_hash, &execution_result.account_updates) + .await? .ok_or(ChainError::ParentStateNotFound)?; // Check state root matches the one in block header @@ -127,18 +128,25 @@ impl Blockchain { self.storage .add_block(block.clone()) + .await .map_err(ChainError::StoreError)?; self.storage .add_receipts(block.hash(), execution_result.receipts) + .await .map_err(ChainError::StoreError) } - pub fn add_block(&self, block: &Block) -> Result<(), ChainError> { + pub async fn add_block(&self, block: &Block) -> Result<(), ChainError> { let since = Instant::now(); + // Easiest way to operate on the result of `execute_block` without + // having to add too much control flow or return early + // Async doesn't play well with `.and_then` + let inner = || async { + let res = self.execute_block(block).await?; + self.store_block(block, res).await + }; - let result = self - .execute_block(block) - .and_then(|res| self.store_block(block, res)); + let result = inner().await; let interval = Instant::now().duration_since(since).as_millis(); if interval != 0 { @@ -146,7 +154,6 @@ impl Blockchain { let throughput = (as_gigas) / (interval as f64) * 1000_f64; info!("[METRIC] BLOCK EXECUTION THROUGHPUT: {throughput} Gigagas/s TIME SPENT: {interval} msecs"); } - result } @@ -157,9 +164,9 @@ impl Blockchain { /// - [`BatchProcessingFailure`] (if the error was caused by block processing). /// /// Note: only the last block's state trie is stored in the db - pub fn add_blocks_in_batch( + pub async fn add_blocks_in_batch( &self, - blocks: &[Block], + blocks: Vec, ) -> Result<(), (ChainError, Option)> { let mut last_valid_hash = H256::default(); @@ -257,6 +264,7 @@ impl Blockchain { first_block_header.parent_hash, &all_account_updates.into_values().collect::>(), ) + .await .map_err(|e| (e.into(), None))? .ok_or((ChainError::ParentStateNotFound, None))?; @@ -265,9 +273,11 @@ impl Blockchain { self.storage .add_blocks(blocks) + .await .map_err(|e| (e.into(), None))?; self.storage .add_receipts_for_blocks(all_receipts) + .await .map_err(|e| (e.into(), None))?; let elapsed_total = interval.elapsed().as_millis(); @@ -286,7 +296,7 @@ impl Blockchain { } //TODO: Forkchoice Update shouldn't be part of this function - pub fn import_blocks(&self, blocks: &Vec) { + pub async fn import_blocks(&self, blocks: &[Block]) { let size = blocks.len(); for block in blocks { let hash = block.hash(); @@ -294,7 +304,7 @@ impl Blockchain { "Adding block {} with hash {:#x}.", block.header.number, hash ); - if let Err(error) = self.add_block(block) { + if let Err(error) = self.add_block(block).await { warn!( "Failed to add block {} with hash {:#x}: {}.", block.header.number, hash, error @@ -303,6 +313,7 @@ impl Blockchain { if self .storage .update_latest_block_number(block.header.number) + .await .is_err() { error!("Fatal: added block {} but could not update the block number -- aborting block import", block.header.number); @@ -311,6 +322,7 @@ impl Blockchain { if self .storage .set_canonical_block(block.header.number, hash) + .await .is_err() { error!( @@ -325,10 +337,12 @@ impl Blockchain { match self.evm_engine { EvmEngine::LEVM => { // We are allowing this not to unwrap so that tests can run even if block execution results in the wrong root hash with LEVM. - let _ = apply_fork_choice(&self.storage, hash, hash, hash); + let _ = apply_fork_choice(&self.storage, hash, hash, hash).await; } EvmEngine::REVM => { - apply_fork_choice(&self.storage, hash, hash, hash).unwrap(); + apply_fork_choice(&self.storage, hash, hash, hash) + .await + .unwrap(); } } } diff --git a/crates/blockchain/fork_choice.rs b/crates/blockchain/fork_choice.rs index 01a23fe121..f1013b922c 100644 --- a/crates/blockchain/fork_choice.rs +++ b/crates/blockchain/fork_choice.rs @@ -18,7 +18,7 @@ use crate::{ /// and itself are made canonical. /// /// If the fork choice state is applied correctly, the head block header is returned. -pub fn apply_fork_choice( +pub async fn apply_fork_choice( store: &Store, head_hash: H256, safe_hash: H256, @@ -104,24 +104,26 @@ pub fn apply_fork_choice( // Make all ancestors to head canonical. for (number, hash) in new_canonical_blocks { - store.set_canonical_block(number, hash)?; + store.set_canonical_block(number, hash).await?; } // Remove anything after the head from the canonical chain. for number in (head.number + 1)..(latest + 1) { - store.unset_canonical_block(number)?; + store.unset_canonical_block(number).await?; } // Make head canonical and label all special blocks correctly. - store.set_canonical_block(head.number, head_hash)?; + store.set_canonical_block(head.number, head_hash).await?; if let Some(finalized) = finalized_res { - store.update_finalized_block_number(finalized.number)?; + store + .update_finalized_block_number(finalized.number) + .await?; } if let Some(safe) = safe_res { - store.update_safe_block_number(safe.number)?; + store.update_safe_block_number(safe.number).await?; } - store.update_latest_block_number(head.number)?; - store.update_sync_status(true)?; + store.update_latest_block_number(head.number).await?; + store.update_sync_status(true).await?; Ok(head) } diff --git a/crates/blockchain/mempool.rs b/crates/blockchain/mempool.rs index 66d0f5426f..b214553373 100644 --- a/crates/blockchain/mempool.rs +++ b/crates/blockchain/mempool.rs @@ -291,15 +291,17 @@ mod tests { use ethrex_storage::{error::StoreError, Store}; fn setup_storage(config: ChainConfig, header: BlockHeader) -> Result { - let store = Store::new("test", EngineType::InMemory)?; - let block_number = header.number; - let block_hash = header.compute_block_hash(); - store.add_block_header(block_hash, header)?; - store.set_canonical_block(block_number, block_hash)?; - store.update_latest_block_number(block_number)?; - store.set_chain_config(&config)?; - - Ok(store) + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + let store = Store::new("test", EngineType::InMemory)?; + let block_number = header.number; + let block_hash = header.compute_block_hash(); + store.add_block_header(block_hash, header).await?; + store.set_canonical_block(block_number, block_hash).await?; + store.update_latest_block_number(block_number).await?; + store.set_chain_config(&config).await?; + Ok(store) + }) } fn build_basic_config_and_header( diff --git a/crates/blockchain/payload.rs b/crates/blockchain/payload.rs index 55fbea30b4..0734b9dc9d 100644 --- a/crates/blockchain/payload.rs +++ b/crates/blockchain/payload.rs @@ -254,7 +254,10 @@ impl<'a> From> for PayloadBuildResult { impl Blockchain { /// Completes the payload building process, return the block value - pub fn build_payload(&self, payload: &mut Block) -> Result { + pub async fn build_payload( + &self, + payload: &mut Block, + ) -> Result { let since = Instant::now(); let gas_limit = payload.header.gas_limit; @@ -266,7 +269,7 @@ impl Blockchain { self.apply_withdrawals(&mut context)?; self.fill_transactions(&mut context)?; self.extract_requests(&mut context)?; - self.finalize_payload(&mut context)?; + self.finalize_payload(&mut context).await?; let interval = Instant::now().duration_since(since).as_millis(); tracing::info!("[METRIC] BUILDING PAYLOAD TOOK: {interval} ms"); @@ -523,14 +526,18 @@ impl Blockchain { Ok(()) } - fn finalize_payload(&self, context: &mut PayloadBuildContext) -> Result<(), ChainError> { + async fn finalize_payload( + &self, + context: &mut PayloadBuildContext<'_>, + ) -> Result<(), ChainError> { let chain_config = &context.store.get_chain_config()?; let fork = chain_config.fork(context.payload.header.timestamp); let account_updates = context.vm.get_state_transitions(fork)?; context.payload.header.state_root = context .store - .apply_account_updates(context.parent_hash(), &account_updates)? + .apply_account_updates(context.parent_hash(), &account_updates) + .await? .unwrap_or_default(); context.payload.header.transactions_root = compute_transactions_root(&context.payload.body.transactions); diff --git a/crates/blockchain/smoke_test.rs b/crates/blockchain/smoke_test.rs index 0d85ab7ea2..e2ab55d3ae 100644 --- a/crates/blockchain/smoke_test.rs +++ b/crates/blockchain/smoke_test.rs @@ -16,10 +16,10 @@ mod blockchain_integration_test { }; use ethrex_storage::{EngineType, Store}; - #[test] - fn test_small_to_long_reorg() { + #[tokio::test] + async fn test_small_to_long_reorg() { // Store and genesis - let store = test_store(); + let store = test_store().await; let genesis_header = store.get_block_header(0).unwrap().unwrap(); let genesis_hash = genesis_header.compute_block_hash(); @@ -27,20 +27,21 @@ mod blockchain_integration_test { let blockchain = Blockchain::default_with_store(store.clone()); // Add first block. We'll make it canonical. - let block_1a = new_block(&store, &genesis_header); + let block_1a = new_block(&store, &genesis_header).await; let hash_1a = block_1a.hash(); - blockchain.add_block(&block_1a).unwrap(); - store.set_canonical_block(1, hash_1a).unwrap(); + blockchain.add_block(&block_1a).await.unwrap(); + store.set_canonical_block(1, hash_1a).await.unwrap(); let retrieved_1a = store.get_block_header(1).unwrap().unwrap(); assert_eq!(retrieved_1a, block_1a.header); assert!(is_canonical(&store, 1, hash_1a).unwrap()); // Add second block at height 1. Will not be canonical. - let block_1b = new_block(&store, &genesis_header); + let block_1b = new_block(&store, &genesis_header).await; let hash_1b = block_1b.hash(); blockchain .add_block(&block_1b) + .await .expect("Could not add block 1b."); let retrieved_1b = store.get_block_header_by_hash(hash_1b).unwrap().unwrap(); @@ -48,10 +49,11 @@ mod blockchain_integration_test { assert!(!is_canonical(&store, 1, hash_1b).unwrap()); // Add a third block at height 2, child to the non canonical block. - let block_2 = new_block(&store, &block_1b.header); + let block_2 = new_block(&store, &block_1b.header).await; let hash_2 = block_2.hash(); blockchain .add_block(&block_2) + .await .expect("Could not add block 2."); let retrieved_2 = store.get_block_header_by_hash(hash_2).unwrap(); @@ -65,6 +67,7 @@ mod blockchain_integration_test { genesis_header.compute_block_hash(), genesis_header.compute_block_hash(), ) + .await .unwrap(); // Check that canonical blocks changed to the new branch. @@ -74,41 +77,43 @@ mod blockchain_integration_test { assert!(!is_canonical(&store, 1, hash_1a).unwrap()); } - #[test] - fn test_sync_not_supported_yet() { - let store = test_store(); + #[tokio::test] + async fn test_sync_not_supported_yet() { + let store = test_store().await; let genesis_header = store.get_block_header(0).unwrap().unwrap(); // Create blockchain let blockchain = Blockchain::default_with_store(store.clone()); // Build a single valid block. - let block_1 = new_block(&store, &genesis_header); + let block_1 = new_block(&store, &genesis_header).await; let hash_1 = block_1.header.compute_block_hash(); - blockchain.add_block(&block_1).unwrap(); - apply_fork_choice(&store, hash_1, H256::zero(), H256::zero()).unwrap(); + blockchain.add_block(&block_1).await.unwrap(); + apply_fork_choice(&store, hash_1, H256::zero(), H256::zero()) + .await + .unwrap(); // Build a child, then change its parent, making it effectively a pending block. - let mut block_2 = new_block(&store, &block_1.header); + let mut block_2 = new_block(&store, &block_1.header).await; block_2.header.parent_hash = H256::random(); let hash_2 = block_2.header.compute_block_hash(); - let result = blockchain.add_block(&block_2); + let result = blockchain.add_block(&block_2).await; assert!(matches!(result, Err(ChainError::ParentNotFound))); // block 2 should now be pending. assert!(store.get_pending_block(hash_2).unwrap().is_some()); - let fc_result = apply_fork_choice(&store, hash_2, H256::zero(), H256::zero()); + let fc_result = apply_fork_choice(&store, hash_2, H256::zero(), H256::zero()).await; assert!(matches!(fc_result, Err(InvalidForkChoice::Syncing))); // block 2 should still be pending. assert!(store.get_pending_block(hash_2).unwrap().is_some()); } - #[test] - fn test_reorg_from_long_to_short_chain() { + #[tokio::test] + async fn test_reorg_from_long_to_short_chain() { // Store and genesis - let store = test_store(); + let store = test_store().await; let genesis_header = store.get_block_header(0).unwrap().unwrap(); let genesis_hash = genesis_header.compute_block_hash(); @@ -116,20 +121,23 @@ mod blockchain_integration_test { let blockchain = Blockchain::default_with_store(store.clone()); // Add first block. Not canonical. - let block_1a = new_block(&store, &genesis_header); + let block_1a = new_block(&store, &genesis_header).await; let hash_1a = block_1a.hash(); - blockchain.add_block(&block_1a).unwrap(); + blockchain.add_block(&block_1a).await.unwrap(); let retrieved_1a = store.get_block_header_by_hash(hash_1a).unwrap().unwrap(); assert!(!is_canonical(&store, 1, hash_1a).unwrap()); // Add second block at height 1. Canonical. - let block_1b = new_block(&store, &genesis_header); + let block_1b = new_block(&store, &genesis_header).await; let hash_1b = block_1b.hash(); blockchain .add_block(&block_1b) + .await .expect("Could not add block 1b."); - apply_fork_choice(&store, hash_1b, genesis_hash, genesis_hash).unwrap(); + apply_fork_choice(&store, hash_1b, genesis_hash, genesis_hash) + .await + .unwrap(); let retrieved_1b = store.get_block_header(1).unwrap().unwrap(); assert_ne!(retrieved_1a, retrieved_1b); @@ -138,12 +146,15 @@ mod blockchain_integration_test { assert_eq!(latest_canonical_block_hash(&store).unwrap(), hash_1b); // Add a third block at height 2, child to the canonical one. - let block_2 = new_block(&store, &block_1b.header); + let block_2 = new_block(&store, &block_1b.header).await; let hash_2 = block_2.hash(); blockchain .add_block(&block_2) + .await .expect("Could not add block 2."); - apply_fork_choice(&store, hash_2, genesis_hash, genesis_hash).unwrap(); + apply_fork_choice(&store, hash_2, genesis_hash, genesis_hash) + .await + .unwrap(); let retrieved_2 = store.get_block_header_by_hash(hash_2).unwrap(); assert_eq!(latest_canonical_block_hash(&store).unwrap(), hash_2); @@ -158,6 +169,7 @@ mod blockchain_integration_test { genesis_header.compute_block_hash(), genesis_header.compute_block_hash(), ) + .await .unwrap(); // Check that canonical blocks changed to the new branch. @@ -167,10 +179,10 @@ mod blockchain_integration_test { assert!(!is_canonical(&store, 2, hash_2).unwrap()); } - #[test] - fn new_head_with_canonical_ancestor_should_skip() { + #[tokio::test] + async fn new_head_with_canonical_ancestor_should_skip() { // Store and genesis - let store = test_store(); + let store = test_store().await; let genesis_header = store.get_block_header(0).unwrap().unwrap(); let genesis_hash = genesis_header.compute_block_hash(); @@ -178,29 +190,33 @@ mod blockchain_integration_test { let blockchain = Blockchain::default_with_store(store.clone()); // Add block at height 1. - let block_1 = new_block(&store, &genesis_header); + let block_1 = new_block(&store, &genesis_header).await; let hash_1 = block_1.hash(); blockchain .add_block(&block_1) + .await .expect("Could not add block 1b."); // Add child at height 2. - let block_2 = new_block(&store, &block_1.header); + let block_2 = new_block(&store, &block_1.header).await; let hash_2 = block_2.hash(); blockchain .add_block(&block_2) + .await .expect("Could not add block 2."); assert!(!is_canonical(&store, 1, hash_1).unwrap()); assert!(!is_canonical(&store, 2, hash_2).unwrap()); // Make that chain the canonical one. - apply_fork_choice(&store, hash_2, genesis_hash, genesis_hash).unwrap(); + apply_fork_choice(&store, hash_2, genesis_hash, genesis_hash) + .await + .unwrap(); assert!(is_canonical(&store, 1, hash_1).unwrap()); assert!(is_canonical(&store, 2, hash_2).unwrap()); - let result = apply_fork_choice(&store, hash_1, hash_1, hash_1); + let result = apply_fork_choice(&store, hash_1, hash_1, hash_1).await; assert!(matches!( result, @@ -213,13 +229,13 @@ mod blockchain_integration_test { assert!(store.get_latest_block_number().unwrap() == 2); } - #[test] - fn latest_block_number_should_always_be_the_canonical_head() { + #[tokio::test] + async fn latest_block_number_should_always_be_the_canonical_head() { // Goal: put a, b in the same branch, both canonical. // Then add one in a different branch. Check that the last one is still the same. // Store and genesis - let store = test_store(); + let store = test_store().await; let genesis_header = store.get_block_header(0).unwrap().unwrap(); let genesis_hash = genesis_header.compute_block_hash(); @@ -227,43 +243,50 @@ mod blockchain_integration_test { let blockchain = Blockchain::default_with_store(store.clone()); // Add block at height 1. - let block_1 = new_block(&store, &genesis_header); + let block_1 = new_block(&store, &genesis_header).await; blockchain .add_block(&block_1) + .await .expect("Could not add block 1b."); // Add child at height 2. - let block_2 = new_block(&store, &block_1.header); + let block_2 = new_block(&store, &block_1.header).await; let hash_2 = block_2.hash(); blockchain .add_block(&block_2) + .await .expect("Could not add block 2."); assert_eq!(latest_canonical_block_hash(&store).unwrap(), genesis_hash); // Make that chain the canonical one. - apply_fork_choice(&store, hash_2, genesis_hash, genesis_hash).unwrap(); + apply_fork_choice(&store, hash_2, genesis_hash, genesis_hash) + .await + .unwrap(); assert_eq!(latest_canonical_block_hash(&store).unwrap(), hash_2); // Add a new, non canonical block, starting from genesis. - let block_1b = new_block(&store, &genesis_header); + let block_1b = new_block(&store, &genesis_header).await; let hash_b = block_1b.hash(); blockchain .add_block(&block_1b) + .await .expect("Could not add block b."); // The latest block should be the same. assert_eq!(latest_canonical_block_hash(&store).unwrap(), hash_2); // if we apply fork choice to the new one, then we should - apply_fork_choice(&store, hash_b, genesis_hash, genesis_hash).unwrap(); + apply_fork_choice(&store, hash_b, genesis_hash, genesis_hash) + .await + .unwrap(); // The latest block should now be the new head. assert_eq!(latest_canonical_block_hash(&store).unwrap(), hash_b); } - fn new_block(store: &Store, parent: &BlockHeader) -> Block { + async fn new_block(store: &Store, parent: &BlockHeader) -> Block { let args = BuildPayloadArgs { parent: parent.compute_block_hash(), timestamp: parent.timestamp + 12, @@ -278,11 +301,11 @@ mod blockchain_integration_test { let blockchain = Blockchain::default_with_store(store.clone().clone()); let mut block = create_payload(&args, store).unwrap(); - blockchain.build_payload(&mut block).unwrap(); + blockchain.build_payload(&mut block).await.unwrap(); block } - fn test_store() -> Store { + async fn test_store() -> Store { // Get genesis let file = File::open("../../test_data/genesis-execution-api.json") .expect("Failed to open genesis file"); @@ -295,6 +318,7 @@ mod blockchain_integration_test { store .add_initial_state(genesis) + .await .expect("Failed to add genesis state"); store diff --git a/crates/common/trie/db.rs b/crates/common/trie/db.rs index f139926661..9cc3dbcd40 100644 --- a/crates/common/trie/db.rs +++ b/crates/common/trie/db.rs @@ -4,7 +4,7 @@ use std::{ sync::{Arc, Mutex}, }; -pub trait TrieDB { +pub trait TrieDB: Send + Sync { fn get(&self, key: Vec) -> Result>, TrieError>; fn put(&self, key: Vec, value: Vec) -> Result<(), TrieError>; // fn put_batch(&self, key: Vec, value: Vec) -> Result<(), TrieError>; diff --git a/crates/l2/prover/tests/perf_zkvm.rs b/crates/l2/prover/tests/perf_zkvm.rs index b9a43520b0..c5260fc67c 100644 --- a/crates/l2/prover/tests/perf_zkvm.rs +++ b/crates/l2/prover/tests/perf_zkvm.rs @@ -44,7 +44,7 @@ async fn setup() -> (ProgramInput, Block) { let genesis = ethrex_l2::utils::test_data_io::read_genesis_file(genesis_file_path.to_str().unwrap()); - store.add_initial_state(genesis.clone()).unwrap(); + store.add_initial_state(genesis.clone()).await.unwrap(); let blocks = ethrex_l2::utils::test_data_io::read_chain_file(chain_file_path.to_str().unwrap()); info!("Number of blocks to insert: {}", blocks.len()); @@ -56,7 +56,7 @@ async fn setup() -> (ProgramInput, Block) { block.body.transactions.len(), block.header.number ); - blockchain.add_block(block).unwrap(); + blockchain.add_block(block).await.unwrap(); } let block_to_prove = blocks.get(3).unwrap(); diff --git a/crates/l2/prover/zkvm/interface/sp1/Cargo.lock b/crates/l2/prover/zkvm/interface/sp1/Cargo.lock index 2a7d2a9838..425dcc15ad 100644 --- a/crates/l2/prover/zkvm/interface/sp1/Cargo.lock +++ b/crates/l2/prover/zkvm/interface/sp1/Cargo.lock @@ -538,6 +538,17 @@ dependencies = [ "serde", ] +[[package]] +name = "async-trait" +version = "0.1.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "aurora-engine-modexp" version = "1.2.0" @@ -1298,6 +1309,7 @@ name = "ethrex-storage" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", "bytes", "ethereum-types", "ethrex-common", diff --git a/crates/l2/sequencer/block_producer.rs b/crates/l2/sequencer/block_producer.rs index cd12cfdcd5..67590845de 100644 --- a/crates/l2/sequencer/block_producer.rs +++ b/crates/l2/sequencer/block_producer.rs @@ -57,8 +57,9 @@ impl BlockProducer { execution_cache: Arc, ) { loop { - if let Err(err) = - self.main_logic(store.clone(), blockchain.clone(), execution_cache.clone()) + if let Err(err) = self + .main_logic(store.clone(), blockchain.clone(), execution_cache.clone()) + .await { error!("Block Producer Error: {}", err); } @@ -67,7 +68,7 @@ impl BlockProducer { } } - pub fn main_logic( + pub async fn main_logic( &self, store: Store, blockchain: Arc, @@ -102,7 +103,7 @@ impl BlockProducer { let mut payload = create_payload(&args, &store)?; // Blockchain builds the payload from mempool txs and executes them - let payload_build_result = blockchain.build_payload(&mut payload)?; + let payload_build_result = blockchain.build_payload(&mut payload).await?; info!("Built payload for new block {}", payload.header.number); // Blockchain stores block @@ -116,7 +117,9 @@ impl BlockProducer { requests: Vec::new(), }; - blockchain.store_block(&block, execution_result.clone())?; + blockchain + .store_block(&block, execution_result.clone()) + .await?; info!("Stored new block {:x}", block.hash()); // WARN: We're not storing the payload into the Store because there's no use to it by the L2 for now. @@ -124,7 +127,7 @@ impl BlockProducer { execution_cache.push(block.hash(), execution_result.account_updates)?; // Make the new head be part of the canonical chain - apply_fork_choice(&store, block.hash(), block.hash(), block.hash())?; + apply_fork_choice(&store, block.hash(), block.hash(), block.hash()).await?; Ok(()) } diff --git a/crates/l2/utils/prover/save_state.rs b/crates/l2/utils/prover/save_state.rs index 7db715afc7..a6bceb2d5f 100644 --- a/crates/l2/utils/prover/save_state.rs +++ b/crates/l2/utils/prover/save_state.rs @@ -394,8 +394,8 @@ mod tests { use test_casing::test_casing; #[test_casing(2, [EvmEngine::LEVM, EvmEngine::REVM])] - #[test] - fn test_state_file_integration( + #[tokio::test] + async fn test_state_file_integration( _evm_engine: EvmEngine, ) -> Result<(), Box> { if let Err(e) = fs::remove_dir_all(default_datadir()?) { @@ -413,13 +413,13 @@ mod tests { let store = Store::new("memory", EngineType::InMemory).expect("Failed to create Store"); let genesis = test_data_io::read_genesis_file(genesis_file_path.to_str().unwrap()); - store.add_initial_state(genesis.clone()).unwrap(); + store.add_initial_state(genesis.clone()).await.unwrap(); let blocks = test_data_io::read_chain_file(chain_file_path.to_str().unwrap()); // create blockchain let blockchain = Blockchain::default_with_store(store.clone()); for block in &blocks { - blockchain.add_block(block).unwrap(); + blockchain.add_block(block).await.unwrap(); } let mut account_updates_vec: Vec> = Vec::new(); diff --git a/crates/l2/utils/test_data_io.rs b/crates/l2/utils/test_data_io.rs index 99f509c5ba..e4150056bf 100644 --- a/crates/l2/utils/test_data_io.rs +++ b/crates/l2/utils/test_data_io.rs @@ -63,6 +63,8 @@ pub fn generate_program_input( chain: Vec, block_number: usize, ) -> Result { + let rt = tokio::runtime::Runtime::new().unwrap(); + let block = chain .get(block_number) .ok_or(ProverInputError::InvalidBlockNumber(block_number))? @@ -70,11 +72,11 @@ pub fn generate_program_input( // create store let store = Store::new("memory", EngineType::InMemory)?; - store.add_initial_state(genesis)?; + rt.block_on(store.add_initial_state(genesis))?; // create blockchain let blockchain = Blockchain::default_with_store(store.clone()); for block in chain { - blockchain.add_block(&block)?; + rt.block_on(blockchain.add_block(&block))?; } let parent_hash = block.header.parent_hash; diff --git a/crates/networking/p2p/rlpx/connection.rs b/crates/networking/p2p/rlpx/connection.rs index 078f5a69a3..52ec860f49 100644 --- a/crates/networking/p2p/rlpx/connection.rs +++ b/crates/networking/p2p/rlpx/connection.rs @@ -505,7 +505,7 @@ impl RLPxConnection { } Message::PooledTransactions(msg) if peer_supports_eth => { if is_synced { - msg.handle(&self.node, &self.blockchain)?; + msg.handle(&self.node, &self.blockchain).await?; } } Message::GetStorageRanges(req) => { diff --git a/crates/networking/p2p/rlpx/eth/backend.rs b/crates/networking/p2p/rlpx/eth/backend.rs index 274c0e8f9c..5e812fe733 100644 --- a/crates/networking/p2p/rlpx/eth/backend.rs +++ b/crates/networking/p2p/rlpx/eth/backend.rs @@ -103,9 +103,9 @@ mod tests { use ethrex_storage::{EngineType, Store}; use std::{fs::File, io::BufReader}; - #[test] + #[tokio::test] // TODO add tests for failing validations - fn test_validate_status() { + async fn test_validate_status() { // Setup // TODO we should have this setup exported to some test_utils module and use from there let storage = @@ -117,6 +117,7 @@ mod tests { serde_json::from_reader(reader).expect("Failed to deserialize genesis file"); storage .add_initial_state(genesis.clone()) + .await .expect("Failed to add genesis block to DB"); let config = genesis.config; let total_difficulty = U256::from(config.terminal_total_difficulty.unwrap_or_default()); diff --git a/crates/networking/p2p/rlpx/eth/transactions.rs b/crates/networking/p2p/rlpx/eth/transactions.rs index 66c346dc2f..691236cd87 100644 --- a/crates/networking/p2p/rlpx/eth/transactions.rs +++ b/crates/networking/p2p/rlpx/eth/transactions.rs @@ -272,7 +272,7 @@ impl PooledTransactions { /// Saves every incoming pooled transaction to the mempool. - pub fn handle(self, node: &Node, blockchain: &Blockchain) -> Result<(), MempoolError> { + pub async fn handle(self, node: &Node, blockchain: &Blockchain) -> Result<(), MempoolError> { for tx in self.pooled_transactions { if let P2PTransaction::EIP4844TransactionWithBlobs(itx) = tx { if let Err(e) = blockchain.add_blob_transaction_to_pool(itx.tx, itx.blobs_bundle) { diff --git a/crates/networking/p2p/sync.rs b/crates/networking/p2p/sync.rs index 1204e2332b..4182b3921a 100644 --- a/crates/networking/p2p/sync.rs +++ b/crates/networking/p2p/sync.rs @@ -270,7 +270,7 @@ impl Syncer { search_head = last_block_hash; current_head = last_block_hash; if sync_mode == SyncMode::Snap { - store.set_header_download_checkpoint(current_head)?; + store.set_header_download_checkpoint(current_head).await?; } } @@ -281,7 +281,7 @@ impl Syncer { < MIN_FULL_BLOCKS as u64 { // Too few blocks for a snap sync, switching to full sync - store.clear_snap_state()?; + store.clear_snap_state().await?; sync_mode = SyncMode::Full; self.snap_enabled.store(false, Ordering::Relaxed); } @@ -293,7 +293,9 @@ impl Syncer { // Store headers and save hashes for full block retrieval all_block_hashes.extend_from_slice(&block_hashes[..]); // This step is necessary for full sync because some opcodes depend on previous blocks during execution. - store.add_block_headers(block_hashes.clone(), block_headers.clone())?; + store + .add_block_headers(block_hashes.clone(), block_headers.clone()) + .await?; if sync_mode == SyncMode::Full { let last_block_hash = self @@ -351,13 +353,13 @@ impl Syncer { .get_block_by_hash(*hash)? .ok_or(SyncError::CorruptDB)?; let block_number = block.header.number; - self.blockchain.add_block(&block)?; - store.set_canonical_block(block_number, *hash)?; - store.update_latest_block_number(block_number)?; + self.blockchain.add_block(&block).await?; + store.set_canonical_block(block_number, *hash).await?; + store.update_latest_block_number(block_number).await?; } self.last_snap_pivot = pivot_header.number; // Finished a sync cycle without aborting halfway, clear current checkpoint - store.clear_snap_state()?; + store.clear_snap_state().await?; // Next sync will be full-sync self.snap_enabled.store(false, Ordering::Relaxed); } @@ -454,18 +456,15 @@ impl Syncer { // For more details, refer to the `get_block_hash` function in [`LevmDatabase`] and the [`revm::Database`]. store .mark_chain_as_canonical(&blocks) + .await .map_err(SyncError::Store)?; // Executing blocks is a CPU heavy operation // Spawn a blocking task to not block the tokio runtime - let res: Result<(), (ChainError, Option)> = { + let res = { let blockchain = self.blockchain.clone(); - tokio::task::spawn_blocking(move || { - Self::add_blocks(blockchain, &blocks, sync_head_found) - }) - .await - .map_err(SyncError::JoinHandle) - }?; + Self::add_blocks(blockchain, blocks, sync_head_found).await + }; if let Err((error, failure)) = res { warn!("Failed to add block during FullSync: {error}"); @@ -486,7 +485,9 @@ impl Syncer { return Err(error.into()); } - store.update_latest_block_number(last_block.header.number)?; + store + .update_latest_block_number(last_block.header.number) + .await?; let elapsed_secs: f64 = since.elapsed().as_millis() as f64 / 1000.0; let blocks_per_second = blocks_len as f64 / elapsed_secs; @@ -508,16 +509,16 @@ impl Syncer { Ok(Some(last_block.hash())) } - fn add_blocks( + async fn add_blocks( blockchain: Arc, - blocks: &[Block], + blocks: Vec, sync_head_found: bool, ) -> Result<(), (ChainError, Option)> { // If we found the sync head, run the blocks sequentially to store all the blocks's state if sync_head_found { let mut last_valid_hash = H256::default(); for block in blocks { - blockchain.add_block(block).map_err(|e| { + blockchain.add_block(&block).await.map_err(|e| { ( e, Some(BatchBlockProcessingFailure { @@ -530,7 +531,7 @@ impl Syncer { } Ok(()) } else { - blockchain.add_blocks_in_batch(blocks) + blockchain.add_blocks_in_batch(blocks).await } } } @@ -549,7 +550,7 @@ async fn store_block_bodies( let current_block_hashes = block_hashes.drain(..block_bodies.len()); // Add bodies to storage for (hash, body) in current_block_hashes.zip(block_bodies.into_iter()) { - store.add_block_body(hash, body)?; + store.add_block_body(hash, body).await?; } // Check if we need to ask for another batch @@ -575,7 +576,7 @@ async fn store_receipts( debug!(" Received {} Receipts", receipts.len()); // Track which blocks we have already fetched receipts for for (block_hash, receipts) in block_hashes.drain(0..receipts.len()).zip(receipts) { - store.add_receipts(block_hash, receipts)?; + store.add_receipts(block_hash, receipts).await?; } // Check if we need to ask for another batch if block_hashes.is_empty() { @@ -654,7 +655,7 @@ impl Syncer { rebuild_start.elapsed().as_secs() ); // Clear snapshot - store.clear_snapshot()?; + store.clear_snapshot().await?; // Perform Healing let state_heal_complete = heal_state_trie( diff --git a/crates/networking/p2p/sync/bytecode_fetcher.rs b/crates/networking/p2p/sync/bytecode_fetcher.rs index 4d4e43072b..1ca160fc87 100644 --- a/crates/networking/p2p/sync/bytecode_fetcher.rs +++ b/crates/networking/p2p/sync/bytecode_fetcher.rs @@ -56,7 +56,7 @@ async fn fetch_bytecode_batch( debug!("Received {} bytecodes", bytecodes.len()); // Store the bytecodes for code in bytecodes.into_iter() { - store.add_account_code(batch.remove(0), code)?; + store.add_account_code(batch.remove(0), code).await?; } } // Return remaining code hashes in the batch if we couldn't fetch all of them diff --git a/crates/networking/p2p/sync/state_healing.rs b/crates/networking/p2p/sync/state_healing.rs index e207cd8940..ce452adea9 100644 --- a/crates/networking/p2p/sync/state_healing.rs +++ b/crates/networking/p2p/sync/state_healing.rs @@ -82,7 +82,7 @@ pub(crate) async fn heal_state_trie( // Save paths for the next cycle if !paths.is_empty() { debug!("Caching {} paths for the next cycle", paths.len()); - store.set_state_heal_paths(paths.clone())?; + store.set_state_heal_paths(paths.clone()).await?; } // Send empty batch to signal that no more batches are incoming bytecode_sender.send(vec![]).await?; diff --git a/crates/networking/p2p/sync/state_sync.rs b/crates/networking/p2p/sync/state_sync.rs index d0e866d5bc..d4efb62444 100644 --- a/crates/networking/p2p/sync/state_sync.rs +++ b/crates/networking/p2p/sync/state_sync.rs @@ -71,7 +71,9 @@ pub(crate) async fn state_sync( state_trie_checkpoint[index] = last_key; } // Update state trie checkpoint - store.set_state_trie_key_checkpoint(state_trie_checkpoint)?; + store + .set_state_trie_key_checkpoint(state_trie_checkpoint) + .await?; Ok(stale_pivot) } @@ -180,7 +182,9 @@ async fn state_sync_segment( .await?; } // Update Snapshot - store.write_snapshot_account_batch(account_hashes, accounts)?; + store + .write_snapshot_account_batch(account_hashes, accounts) + .await?; // As we are downloading the state trie in segments the `should_continue` flag will mean that there // are more accounts to be fetched but these accounts may belong to the next segment if !should_continue || start_account_hash >= STATE_TRIE_SEGMENTS_END[segment_number] { diff --git a/crates/networking/p2p/sync/storage_fetcher.rs b/crates/networking/p2p/sync/storage_fetcher.rs index 22de111701..c716bf7287 100644 --- a/crates/networking/p2p/sync/storage_fetcher.rs +++ b/crates/networking/p2p/sync/storage_fetcher.rs @@ -158,7 +158,9 @@ async fn fetch_storage_batch( let (account_hash, storage_root) = batch.remove(0); let last_key = *last_keys.last().unwrap(); // Store downloaded range - store.write_snapshot_storage_batch(account_hash, last_keys, last_values)?; + store + .write_snapshot_storage_batch(account_hash, last_keys, last_values) + .await?; // Delegate the rest of the trie to the large trie fetcher large_storage_sender .send(vec![LargeStorageRequest { @@ -174,7 +176,9 @@ async fn fetch_storage_batch( // Store the storage ranges & rebuild the storage trie for each account let filled_storages: Vec<(H256, H256)> = batch.drain(..values.len()).collect(); let account_hashes: Vec = filled_storages.iter().map(|(hash, _)| *hash).collect(); - store.write_snapshot_storage_batches(account_hashes, keys, values)?; + store + .write_snapshot_storage_batches(account_hashes, keys, values) + .await?; // Send complete storages to the rebuilder storage_trie_rebuilder_sender.send(filled_storages).await?; // Return remaining code hashes in the batch if we couldn't fetch all of them @@ -295,7 +299,9 @@ async fn fetch_large_storage( // Update next batch's start request.last_key = *keys.last().unwrap(); // Write storage range to snapshot - store.write_snapshot_storage_batch(request.account_hash, keys, values)?; + store + .write_snapshot_storage_batch(request.account_hash, keys, values) + .await?; if incomplete { Ok((Some(request), false)) } else { diff --git a/crates/networking/p2p/sync/storage_healing.rs b/crates/networking/p2p/sync/storage_healing.rs index 81f51c765f..63ba58d11a 100644 --- a/crates/networking/p2p/sync/storage_healing.rs +++ b/crates/networking/p2p/sync/storage_healing.rs @@ -92,7 +92,9 @@ pub(crate) async fn storage_healer( } let healing_complete = pending_paths.is_empty(); // Store pending paths - store.set_storage_heal_paths(pending_paths.into_iter().collect())?; + store + .set_storage_heal_paths(pending_paths.into_iter().collect()) + .await?; Ok(healing_complete) } diff --git a/crates/networking/p2p/sync/trie_rebuild.rs b/crates/networking/p2p/sync/trie_rebuild.rs index 1dd736750a..eb0209f746 100644 --- a/crates/networking/p2p/sync/trie_rebuild.rs +++ b/crates/networking/p2p/sync/trie_rebuild.rs @@ -138,7 +138,7 @@ async fn rebuild_state_trie_in_backgound( } // Update DB checkpoint let checkpoint = (root, rebuild_status.clone().map(|st| st.current)); - store.set_state_trie_rebuild_checkpoint(checkpoint)?; + store.set_state_trie_rebuild_checkpoint(checkpoint).await?; // Move on to the next segment current_segment = (current_segment + 1) % STATE_TRIE_SEGMENTS } @@ -235,7 +235,9 @@ async fn rebuild_storage_trie_in_background( res?; } } - store.set_storage_trie_rebuild_pending(pending_storages)?; + store + .set_storage_trie_rebuild_pending(pending_storages) + .await?; Ok(()) } diff --git a/crates/networking/rpc/engine/exchange_transition_config.rs b/crates/networking/rpc/engine/exchange_transition_config.rs index 66a28f1492..7c55632bb9 100644 --- a/crates/networking/rpc/engine/exchange_transition_config.rs +++ b/crates/networking/rpc/engine/exchange_transition_config.rs @@ -47,7 +47,7 @@ impl RpcHandler for ExchangeTransitionConfigV1Req { Ok(ExchangeTransitionConfigV1Req { payload }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { info!("Received new engine request: {self}"); let payload = &self.payload; diff --git a/crates/networking/rpc/engine/fork_choice.rs b/crates/networking/rpc/engine/fork_choice.rs index 7ea07db062..7b7be6efa6 100644 --- a/crates/networking/rpc/engine/fork_choice.rs +++ b/crates/networking/rpc/engine/fork_choice.rs @@ -34,9 +34,9 @@ impl RpcHandler for ForkChoiceUpdatedV1 { }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let (head_block_opt, mut response) = - handle_forkchoice(&self.fork_choice_state, context.clone(), 1)?; + handle_forkchoice(&self.fork_choice_state, context.clone(), 1).await?; if let (Some(head_block), Some(attributes)) = (head_block_opt, &self.payload_attributes) { let chain_config = context.storage.get_chain_config()?; if chain_config.is_cancun_activated(attributes.timestamp) { @@ -45,7 +45,7 @@ impl RpcHandler for ForkChoiceUpdatedV1 { )); } validate_attributes_v1(attributes, &head_block)?; - let payload_id = build_payload(attributes, context, &self.fork_choice_state, 1)?; + let payload_id = build_payload(attributes, context, &self.fork_choice_state, 1).await?; response.set_id(payload_id); } serde_json::to_value(response).map_err(|error| RpcErr::Internal(error.to_string())) @@ -67,9 +67,9 @@ impl RpcHandler for ForkChoiceUpdatedV2 { }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let (head_block_opt, mut response) = - handle_forkchoice(&self.fork_choice_state, context.clone(), 2)?; + handle_forkchoice(&self.fork_choice_state, context.clone(), 2).await?; if let (Some(head_block), Some(attributes)) = (head_block_opt, &self.payload_attributes) { let chain_config = context.storage.get_chain_config()?; if chain_config.is_cancun_activated(attributes.timestamp) { @@ -82,7 +82,7 @@ impl RpcHandler for ForkChoiceUpdatedV2 { // Behave as a v1 validate_attributes_v1(attributes, &head_block)?; } - let payload_id = build_payload(attributes, context, &self.fork_choice_state, 2)?; + let payload_id = build_payload(attributes, context, &self.fork_choice_state, 2).await?; response.set_id(payload_id); } serde_json::to_value(response).map_err(|error| RpcErr::Internal(error.to_string())) @@ -133,7 +133,7 @@ impl RpcHandler for ForkChoiceUpdatedV3 { // Parse it again as it was consumed for gateway_response and it is the same as cloning it. let request = Self::parse(&req.params)?; - let client_response = request.handle(context); + let client_response = request.handle(context).await; let gateway_response = gateway_request .await @@ -155,12 +155,12 @@ impl RpcHandler for ForkChoiceUpdatedV3 { gateway_response.or(client_response) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let (head_block_opt, mut response) = - handle_forkchoice(&self.fork_choice_state, context.clone(), 3)?; + handle_forkchoice(&self.fork_choice_state, context.clone(), 3).await?; if let (Some(head_block), Some(attributes)) = (head_block_opt, &self.payload_attributes) { validate_attributes_v3(attributes, &head_block, &context)?; - let payload_id = build_payload(attributes, context, &self.fork_choice_state, 3)?; + let payload_id = build_payload(attributes, context, &self.fork_choice_state, 3).await?; response.set_id(payload_id); } serde_json::to_value(response).map_err(|error| RpcErr::Internal(error.to_string())) @@ -198,7 +198,7 @@ fn parse( Ok((forkchoice_state, payload_attributes)) } -fn handle_forkchoice( +async fn handle_forkchoice( fork_choice_state: &ForkChoiceState, context: RpcApiContext, version: usize, @@ -252,6 +252,7 @@ fn handle_forkchoice( fork_choice_state.safe_block_hash, fork_choice_state.finalized_block_hash, ) + .await } } } @@ -302,6 +303,7 @@ fn handle_forkchoice( context .storage .update_sync_status(false) + .await .map_err(|e| RpcErr::Internal(e.to_string()))?; context.syncer.start_sync(); ForkChoiceResponse::from(PayloadStatus::syncing()) @@ -392,7 +394,7 @@ fn validate_timestamp( Ok(()) } -fn build_payload( +async fn build_payload( attributes: &PayloadAttributesV3, context: RpcApiContext, fork_choice_state: &ForkChoiceState, @@ -416,7 +418,7 @@ fn build_payload( // so the only errors that may be returned are internal storage errors Err(error) => return Err(RpcErr::Internal(error.to_string())), }; - context.storage.add_payload(payload_id, payload)?; + context.storage.add_payload(payload_id, payload).await?; Ok(payload_id) } diff --git a/crates/networking/rpc/engine/mod.rs b/crates/networking/rpc/engine/mod.rs index ed2be76a59..aa123a32a3 100644 --- a/crates/networking/rpc/engine/mod.rs +++ b/crates/networking/rpc/engine/mod.rs @@ -53,7 +53,7 @@ impl RpcHandler for ExchangeCapabilitiesRequest { }) } - fn handle(&self, _context: RpcApiContext) -> Result { + async fn handle(&self, _context: RpcApiContext) -> Result { Ok(json!(CAPABILITIES)) } } diff --git a/crates/networking/rpc/engine/payload.rs b/crates/networking/rpc/engine/payload.rs index 5e9a764786..f1323bfb3e 100644 --- a/crates/networking/rpc/engine/payload.rs +++ b/crates/networking/rpc/engine/payload.rs @@ -32,9 +32,9 @@ impl RpcHandler for NewPayloadV1Request { }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { validate_execution_payload_v1(&self.payload)?; - handle_new_payload_v1_v2(&self.payload, context) + handle_new_payload_v1_v2(&self.payload, context).await } } @@ -49,7 +49,7 @@ impl RpcHandler for NewPayloadV2Request { }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let chain_config = &context.storage.get_chain_config()?; if chain_config.is_shanghai_activated(self.payload.timestamp) { validate_execution_payload_v2(&self.payload)?; @@ -58,7 +58,7 @@ impl RpcHandler for NewPayloadV2Request { validate_execution_payload_v1(&self.payload)?; } - handle_new_payload_v1_v2(&self.payload, context) + handle_new_payload_v1_v2(&self.payload, context).await } } @@ -117,7 +117,7 @@ impl RpcHandler for NewPayloadV3Request { request.parent_beacon_block_root, ); - let client_response = Self::call(req, context); + let client_response = Self::call(req, context).await; let gateway_response = gateway_request .await @@ -139,7 +139,7 @@ impl RpcHandler for NewPayloadV3Request { gateway_response.or(client_response) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let block = get_block_from_payload(&self.payload, Some(self.parent_beacon_block_root), None)?; validate_fork(&block, Fork::Cancun, &context)?; @@ -149,7 +149,8 @@ impl RpcHandler for NewPayloadV3Request { context, block, self.expected_blob_versioned_hashes.clone(), - )?; + ) + .await?; serde_json::to_value(payload_status).map_err(|error| RpcErr::Internal(error.to_string())) } } @@ -180,7 +181,7 @@ impl RpcHandler for NewPayloadV4Request { }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { // validate the received requests validate_execution_requests(&self.execution_requests)?; @@ -206,7 +207,8 @@ impl RpcHandler for NewPayloadV4Request { context, block, self.expected_blob_versioned_hashes.clone(), - )?; + ) + .await?; serde_json::to_value(payload_status).map_err(|error| RpcErr::Internal(error.to_string())) } } @@ -222,12 +224,12 @@ impl RpcHandler for GetPayloadV1Request { Ok(Self { payload_id }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let payload = get_payload(self.payload_id, &context)?; // NOTE: This validation is actually not required to run Hive tests. Not sure if it's // necessary validate_payload_v1_v2(&payload.block, &context)?; - let payload_bundle = build_payload_if_necessary(self.payload_id, payload, context)?; + let payload_bundle = build_payload_if_necessary(self.payload_id, payload, context).await?; let response = ExecutionPayload::from_block(payload_bundle.block); @@ -245,10 +247,10 @@ impl RpcHandler for GetPayloadV2Request { Ok(Self { payload_id }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let payload = get_payload(self.payload_id, &context)?; validate_payload_v1_v2(&payload.block, &context)?; - let payload_bundle = build_payload_if_necessary(self.payload_id, payload, context)?; + let payload_bundle = build_payload_if_necessary(self.payload_id, payload, context).await?; let response = ExecutionPayloadResponse { execution_payload: ExecutionPayload::from_block(payload_bundle.block), @@ -295,7 +297,7 @@ impl RpcHandler for GetPayloadV3Request { let gateway_request = gateway_auth_client.engine_get_payload_v3(request.payload_id); - let client_response = Self::call(req, context); + let client_response = Self::call(req, context).await; let gateway_response = gateway_request .await @@ -317,10 +319,10 @@ impl RpcHandler for GetPayloadV3Request { gateway_response.or(client_response) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let payload = get_payload(self.payload_id, &context)?; validate_fork(&payload.block, Fork::Cancun, &context)?; - let payload_bundle = build_payload_if_necessary(self.payload_id, payload, context)?; + let payload_bundle = build_payload_if_necessary(self.payload_id, payload, context).await?; let response = ExecutionPayloadResponse { execution_payload: ExecutionPayload::from_block(payload_bundle.block), @@ -354,7 +356,7 @@ impl RpcHandler for GetPayloadV4Request { Ok(Self { payload_id }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let payload = get_payload(self.payload_id, &context)?; let chain_config = &context.storage.get_chain_config()?; @@ -365,7 +367,7 @@ impl RpcHandler for GetPayloadV4Request { ))); } - let payload_bundle = build_payload_if_necessary(self.payload_id, payload, context)?; + let payload_bundle = build_payload_if_necessary(self.payload_id, payload, context).await?; let response = ExecutionPayloadResponse { execution_payload: ExecutionPayload::from_block(payload_bundle.block), @@ -403,7 +405,7 @@ impl RpcHandler for GetPayloadBodiesByHashV1Request { }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { if self.hashes.len() >= GET_PAYLOAD_BODIES_REQUEST_MAX_SIZE { return Err(RpcErr::TooLargeRequest); } @@ -440,7 +442,7 @@ impl RpcHandler for GetPayloadBodiesByRangeV1Request { Ok(GetPayloadBodiesByRangeV1Request { start, count }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { if self.count as usize >= GET_PAYLOAD_BODIES_REQUEST_MAX_SIZE { return Err(RpcErr::TooLargeRequest); } @@ -556,7 +558,7 @@ fn validate_ancestors( } // TODO: We need to check why we return a Result here instead of a Result as in v3. -fn handle_new_payload_v1_v2( +async fn handle_new_payload_v1_v2( payload: &ExecutionPayload, context: RpcApiContext, ) -> Result { @@ -583,11 +585,11 @@ fn handle_new_payload_v1_v2( } // All checks passed, execute payload - let payload_status = execute_payload(&block, &context)?; + let payload_status = execute_payload(&block, &context).await?; serde_json::to_value(payload_status).map_err(|error| RpcErr::Internal(error.to_string())) } -fn handle_new_payload_v3( +async fn handle_new_payload_v3( payload: &ExecutionPayload, context: RpcApiContext, block: Block, @@ -625,7 +627,7 @@ fn handle_new_payload_v3( } // All checks passed, execute payload - execute_payload(&block, &context) + execute_payload(&block, &context).await } // Elements of the list MUST be ordered by request_type in ascending order. @@ -671,7 +673,7 @@ fn validate_block_hash(payload: &ExecutionPayload, block: &Block) -> Result<(), Ok(()) } -fn execute_payload(block: &Block, context: &RpcApiContext) -> Result { +async fn execute_payload(block: &Block, context: &RpcApiContext) -> Result { let block_hash = block.hash(); let storage = &context.storage; // Return the valid message directly if we have it. @@ -699,7 +701,7 @@ fn execute_payload(block: &Block, context: &RpcApiContext) -> Result Ok(PayloadStatus::syncing()), // Under the current implementation this is not possible: we always calculate the state // transition of any new payload as long as the parent is present. If we received the @@ -786,7 +788,7 @@ fn validate_fork(block: &Block, fork: Fork, context: &RpcApiContext) -> Result<( Ok(()) } -fn build_payload_if_necessary( +async fn build_payload_if_necessary( payload_id: u64, mut payload: PayloadBundle, context: RpcApiContext, @@ -803,6 +805,7 @@ fn build_payload_if_necessary( } = context .blockchain .build_payload(&mut payload.block) + .await .map_err(|err| RpcErr::Internal(err.to_string()))?; (blobs_bundle, requests, block_value) }; @@ -817,7 +820,8 @@ fn build_payload_if_necessary( context .storage - .update_payload(payload_id, new_payload.clone())?; + .update_payload(payload_id, new_payload.clone()) + .await?; Ok(new_payload) } diff --git a/crates/networking/rpc/eth/account.rs b/crates/networking/rpc/eth/account.rs index 9ae67a7d9a..2ac08cd0ef 100644 --- a/crates/networking/rpc/eth/account.rs +++ b/crates/networking/rpc/eth/account.rs @@ -47,7 +47,7 @@ impl RpcHandler for GetBalanceRequest { block: BlockIdentifierOrHash::parse(params[1].clone(), 1)?, }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { info!( "Requested balance of account {} at block {}", self.address, self.block @@ -82,7 +82,7 @@ impl RpcHandler for GetCodeRequest { block: BlockIdentifierOrHash::parse(params[1].clone(), 1)?, }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { info!( "Requested code of account {} at block {}", self.address, self.block @@ -118,7 +118,7 @@ impl RpcHandler for GetStorageAtRequest { block: BlockIdentifierOrHash::parse(params[2].clone(), 2)?, }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { info!( "Requested storage sot {} of account {} at block {}", self.storage_slot, self.address, self.block @@ -153,7 +153,7 @@ impl RpcHandler for GetTransactionCountRequest { block: BlockIdentifierOrHash::parse(params[1].clone(), 1)?, }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { info!( "Requested nonce of account {} at block {}", self.address, self.block @@ -203,7 +203,7 @@ impl RpcHandler for GetProofRequest { }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let storage = &context.storage; info!( "Requested proof for account {} at block {} with storage keys: {:?}", diff --git a/crates/networking/rpc/eth/block.rs b/crates/networking/rpc/eth/block.rs index e3b7899177..109a4e8da7 100644 --- a/crates/networking/rpc/eth/block.rs +++ b/crates/networking/rpc/eth/block.rs @@ -64,7 +64,7 @@ impl RpcHandler for GetBlockByNumberRequest { hydrated: serde_json::from_value(params[1].clone())?, }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let storage = &context.storage; info!("Requested block with number: {}", self.block); let block_number = match self.block.resolve_block_number(storage)? { @@ -98,7 +98,7 @@ impl RpcHandler for GetBlockByHashRequest { hydrated: serde_json::from_value(params[1].clone())?, }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let storage = &context.storage; info!("Requested block with hash: {:#x}", self.block); let block_number = match storage.get_block_number(self.block)? { @@ -131,7 +131,7 @@ impl RpcHandler for GetBlockTransactionCountRequest { }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { info!( "Requested transaction count for block with number: {}", self.block @@ -164,7 +164,7 @@ impl RpcHandler for GetBlockReceiptsRequest { }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let storage = &context.storage; info!("Requested receipts for block with number: {}", self.block); let block_number = match self.block.resolve_block_number(storage)? { @@ -197,7 +197,7 @@ impl RpcHandler for GetRawHeaderRequest { }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { info!( "Requested raw header for block with identifier: {}", self.block @@ -230,7 +230,7 @@ impl RpcHandler for GetRawBlockRequest { }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { info!("Requested raw block: {}", self.block); let block_number = match self.block.resolve_block_number(&context.storage)? { Some(block_number) => block_number, @@ -263,7 +263,7 @@ impl RpcHandler for GetRawReceipts { }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let storage = &context.storage; let block_number = match self.block.resolve_block_number(storage)? { Some(block_number) => block_number, @@ -288,7 +288,7 @@ impl RpcHandler for BlockNumberRequest { Ok(Self {}) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { info!("Requested latest block number"); serde_json::to_value(format!("{:#x}", context.storage.get_latest_block_number()?)) .map_err(|error| RpcErr::Internal(error.to_string())) @@ -300,7 +300,7 @@ impl RpcHandler for GetBlobBaseFee { Ok(Self {}) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { info!("Requested blob gas price"); let block_number = context.storage.get_latest_block_number()?; let header = match context.storage.get_block_header(block_number)? { diff --git a/crates/networking/rpc/eth/client.rs b/crates/networking/rpc/eth/client.rs index 273986a1f6..615c43918e 100644 --- a/crates/networking/rpc/eth/client.rs +++ b/crates/networking/rpc/eth/client.rs @@ -12,7 +12,7 @@ impl RpcHandler for ChainId { Ok(Self {}) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { info!("Requested chain id"); let chain_spec = context .storage @@ -29,7 +29,7 @@ impl RpcHandler for Syncing { Ok(Self {}) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let is_synced = context.storage.is_synced()?; Ok(Value::Bool(!is_synced)) } diff --git a/crates/networking/rpc/eth/fee_calculator.rs b/crates/networking/rpc/eth/fee_calculator.rs index d5655dee49..54e07c12ce 100644 --- a/crates/networking/rpc/eth/fee_calculator.rs +++ b/crates/networking/rpc/eth/fee_calculator.rs @@ -87,33 +87,33 @@ mod tests { BASE_PRICE_IN_WEI, }; - #[test] - fn test_for_legacy_txs() { - let storage = setup_store(); - add_legacy_tx_blocks(&storage, 20, 10); + #[tokio::test] + async fn test_for_legacy_txs() { + let storage = setup_store().await; + add_legacy_tx_blocks(&storage, 20, 10).await; let gas_tip = estimate_gas_tip(&storage).unwrap().unwrap(); assert_eq!(gas_tip, BASE_PRICE_IN_WEI); } - #[test] - fn test_for_eip1559_txs() { - let storage = setup_store(); - add_eip1559_tx_blocks(&storage, 20, 10); + #[tokio::test] + async fn test_for_eip1559_txs() { + let storage = setup_store().await; + add_eip1559_tx_blocks(&storage, 20, 10).await; let gas_tip = estimate_gas_tip(&storage).unwrap().unwrap(); assert_eq!(gas_tip, BASE_PRICE_IN_WEI); } - #[test] - fn test_for_mixed_txs() { - let storage = setup_store(); - add_mixed_tx_blocks(&storage, 20, 10); + #[tokio::test] + async fn test_for_mixed_txs() { + let storage = setup_store().await; + add_mixed_tx_blocks(&storage, 20, 10).await; let gas_tip = estimate_gas_tip(&storage).unwrap().unwrap(); assert_eq!(gas_tip, BASE_PRICE_IN_WEI); } - #[test] - fn test_for_empty_blocks() { - let storage = setup_store(); + #[tokio::test] + async fn test_for_empty_blocks() { + let storage = setup_store().await; let gas_tip = estimate_gas_tip(&storage).unwrap(); assert_eq!(gas_tip, None); } diff --git a/crates/networking/rpc/eth/fee_market.rs b/crates/networking/rpc/eth/fee_market.rs index 87a6650afd..0443d5c61c 100644 --- a/crates/networking/rpc/eth/fee_market.rs +++ b/crates/networking/rpc/eth/fee_market.rs @@ -84,7 +84,7 @@ impl RpcHandler for FeeHistoryRequest { }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let storage = &context.storage; let config = storage.get_chain_config()?; info!( diff --git a/crates/networking/rpc/eth/filter.rs b/crates/networking/rpc/eth/filter.rs index 412f6e79c7..026dddadf4 100644 --- a/crates/networking/rpc/eth/filter.rs +++ b/crates/networking/rpc/eth/filter.rs @@ -465,6 +465,7 @@ mod tests { context .storage .add_initial_state(genesis_config) + .await .expect("Fatal: could not add test genesis in test"); let response = map_http_requests(&request, context) .await diff --git a/crates/networking/rpc/eth/gas_price.rs b/crates/networking/rpc/eth/gas_price.rs index 740dbb3048..e576c7ddbd 100644 --- a/crates/networking/rpc/eth/gas_price.rs +++ b/crates/networking/rpc/eth/gas_price.rs @@ -17,7 +17,7 @@ impl RpcHandler for GasPrice { Ok(GasPrice {}) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let latest_block_number = context.storage.get_latest_block_number()?; let estimated_gas_tip = estimate_gas_tip(&context.storage)?; @@ -71,8 +71,8 @@ mod tests { use serde_json::json; use std::sync::Arc; - fn default_context() -> RpcApiContext { - let storage = setup_store(); + async fn default_context() -> RpcApiContext { + let storage = setup_store().await; let blockchain = Arc::new(Blockchain::default_with_store(storage.clone())); RpcApiContext { storage, @@ -95,58 +95,58 @@ mod tests { } } - #[test] - fn test_for_legacy_txs() { - let context = default_context(); + #[tokio::test] + async fn test_for_legacy_txs() { + let context = default_context().await; - add_legacy_tx_blocks(&context.storage, 100, 10); + add_legacy_tx_blocks(&context.storage, 100, 10).await; let gas_price = GasPrice {}; - let response = gas_price.handle(context).unwrap(); + let response = gas_price.handle(context).await.unwrap(); let parsed_result = parse_json_hex(&response).unwrap(); assert_eq!(parsed_result, 2 * BASE_PRICE_IN_WEI); } - #[test] - fn test_for_eip_1559_txs() { - let context = default_context(); + #[tokio::test] + async fn test_for_eip_1559_txs() { + let context = default_context().await; - add_eip1559_tx_blocks(&context.storage, 100, 10); + add_eip1559_tx_blocks(&context.storage, 100, 10).await; let gas_price = GasPrice {}; - let response = gas_price.handle(context).unwrap(); + let response = gas_price.handle(context).await.unwrap(); let parsed_result = parse_json_hex(&response).unwrap(); assert_eq!(parsed_result, 2 * BASE_PRICE_IN_WEI); } - #[test] - fn test_with_mixed_transactions() { - let context = default_context(); + #[tokio::test] + async fn test_with_mixed_transactions() { + let context = default_context().await; - add_mixed_tx_blocks(&context.storage, 100, 10); + add_mixed_tx_blocks(&context.storage, 100, 10).await; let gas_price = GasPrice {}; - let response = gas_price.handle(context).unwrap(); + let response = gas_price.handle(context).await.unwrap(); let parsed_result = parse_json_hex(&response).unwrap(); assert_eq!(parsed_result, 2 * BASE_PRICE_IN_WEI); } - #[test] - fn test_with_not_enough_blocks_or_transactions() { - let context = default_context(); + #[tokio::test] + async fn test_with_not_enough_blocks_or_transactions() { + let context = default_context().await; - add_mixed_tx_blocks(&context.storage, 100, 0); + add_mixed_tx_blocks(&context.storage, 100, 0).await; let gas_price = GasPrice {}; - let response = gas_price.handle(context).unwrap(); + let response = gas_price.handle(context).await.unwrap(); let parsed_result = parse_json_hex(&response).unwrap(); assert_eq!(parsed_result, BASE_PRICE_IN_WEI); } - #[test] - fn test_with_no_blocks_but_genesis() { - let context = default_context(); + #[tokio::test] + async fn test_with_no_blocks_but_genesis() { + let context = default_context().await; let gas_price = GasPrice {}; // genesis base fee is = BASE_PRICE_IN_WEI let expected_gas_price = BASE_PRICE_IN_WEI; - let response = gas_price.handle(context).unwrap(); + let response = gas_price.handle(context).await.unwrap(); let parsed_result = parse_json_hex(&response).unwrap(); assert_eq!(parsed_result, expected_gas_price); } @@ -160,10 +160,10 @@ mod tests { }); let expected_response = json!("0x3b9aca00"); let request: RpcRequest = serde_json::from_value(raw_json).expect("Test json is not valid"); - let mut context = default_context(); + let mut context = default_context().await; context.local_p2p_node = example_p2p_node(); - add_legacy_tx_blocks(&context.storage, 100, 1); + add_legacy_tx_blocks(&context.storage, 100, 1).await; let response = map_http_requests(&request, context).await.unwrap(); assert_eq!(response, expected_response) diff --git a/crates/networking/rpc/eth/logs.rs b/crates/networking/rpc/eth/logs.rs index bf0c73c3f8..1fae0a0b51 100644 --- a/crates/networking/rpc/eth/logs.rs +++ b/crates/networking/rpc/eth/logs.rs @@ -85,7 +85,7 @@ impl RpcHandler for LogsFilter { )), } } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let filtered_logs = fetch_logs_with_filter(self, context.storage)?; serde_json::to_value(filtered_logs).map_err(|error| { tracing::error!("Log filtering request failed with: {error}"); diff --git a/crates/networking/rpc/eth/max_priority_fee.rs b/crates/networking/rpc/eth/max_priority_fee.rs index e053a59c10..93c677b196 100644 --- a/crates/networking/rpc/eth/max_priority_fee.rs +++ b/crates/networking/rpc/eth/max_priority_fee.rs @@ -17,7 +17,7 @@ impl RpcHandler for MaxPriorityFee { Ok(MaxPriorityFee {}) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let estimated_gas_tip = estimate_gas_tip(&context.storage)?; let gas_tip = match estimated_gas_tip { @@ -54,8 +54,8 @@ mod tests { use serde_json::{json, Value}; use std::sync::Arc; - fn default_context() -> RpcApiContext { - let storage = setup_store(); + async fn default_context() -> RpcApiContext { + let storage = setup_store().await; let blockchain = Arc::new(Blockchain::default_with_store(storage.clone())); RpcApiContext { storage, @@ -78,56 +78,56 @@ mod tests { } } - #[test] - fn test_for_legacy_txs() { - let context = default_context(); + #[tokio::test] + async fn test_for_legacy_txs() { + let context = default_context().await; - add_legacy_tx_blocks(&context.storage, 100, 10); + add_legacy_tx_blocks(&context.storage, 100, 10).await; let gas_price = MaxPriorityFee {}; - let response = gas_price.handle(context).unwrap(); + let response = gas_price.handle(context).await.unwrap(); let parsed_result = parse_json_hex(&response).unwrap(); assert_eq!(parsed_result, BASE_PRICE_IN_WEI); } - #[test] - fn test_for_eip_1559_txs() { - let context = default_context(); + #[tokio::test] + async fn test_for_eip_1559_txs() { + let context = default_context().await; - add_eip1559_tx_blocks(&context.storage, 100, 10); + add_eip1559_tx_blocks(&context.storage, 100, 10).await; let gas_price = MaxPriorityFee {}; - let response = gas_price.handle(context).unwrap(); + let response = gas_price.handle(context).await.unwrap(); let parsed_result = parse_json_hex(&response).unwrap(); assert_eq!(parsed_result, BASE_PRICE_IN_WEI); } - #[test] - fn test_with_mixed_transactions() { - let context = default_context(); + #[tokio::test] + async fn test_with_mixed_transactions() { + let context = default_context().await; - add_mixed_tx_blocks(&context.storage, 100, 10); + add_mixed_tx_blocks(&context.storage, 100, 10).await; let gas_price = MaxPriorityFee {}; - let response = gas_price.handle(context).unwrap(); + let response = gas_price.handle(context).await.unwrap(); let parsed_result = parse_json_hex(&response).unwrap(); assert_eq!(parsed_result, BASE_PRICE_IN_WEI); } - #[test] - fn test_with_not_enough_blocks_or_transactions() { - let context = default_context(); + #[tokio::test] + async fn test_with_not_enough_blocks_or_transactions() { + let context = default_context().await; - add_mixed_tx_blocks(&context.storage, 100, 0); + add_mixed_tx_blocks(&context.storage, 100, 0).await; let gas_price = MaxPriorityFee {}; - let response = gas_price.handle(context).unwrap(); + let response = gas_price.handle(context).await.unwrap(); assert_eq!(response, Value::Null); } - #[test] - fn test_with_no_blocks_but_genesis() { - let context = default_context(); + #[tokio::test] + async fn test_with_no_blocks_but_genesis() { + let context = default_context().await; let gas_price = MaxPriorityFee {}; - let response = gas_price.handle(context).unwrap(); + let response = gas_price.handle(context).await.unwrap(); assert_eq!(response, Value::Null); } #[tokio::test] @@ -140,10 +140,10 @@ mod tests { }); let expected_response = json!("0x3b9aca00"); let request: RpcRequest = serde_json::from_value(raw_json).expect("Test json is not valid"); - let mut context = default_context(); + let mut context = default_context().await; context.local_p2p_node = example_p2p_node(); - add_eip1559_tx_blocks(&context.storage, 100, 3); + add_eip1559_tx_blocks(&context.storage, 100, 3).await; let response = map_http_requests(&request, context).await.unwrap(); assert_eq!(response, expected_response) diff --git a/crates/networking/rpc/eth/mod.rs b/crates/networking/rpc/eth/mod.rs index 28c2064bb1..332af4d340 100644 --- a/crates/networking/rpc/eth/mod.rs +++ b/crates/networking/rpc/eth/mod.rs @@ -73,7 +73,7 @@ pub mod test_utils { } } - fn add_blocks_with_transactions( + async fn add_blocks_with_transactions( storage: &Store, block_count: u64, txs_per_block: Vec, @@ -86,11 +86,12 @@ pub mod test_utils { }; let block_header = test_header(block_num); let block = Block::new(block_header.clone(), block_body); - storage.add_block(block).unwrap(); + storage.add_block(block).await.unwrap(); storage .set_canonical_block(block_num, block_header.compute_block_hash()) + .await .unwrap(); - storage.update_latest_block_number(block_num).unwrap(); + storage.update_latest_block_number(block_num).await.unwrap(); } } @@ -131,37 +132,37 @@ pub mod test_utils { }) } - pub fn setup_store() -> Store { + pub async fn setup_store() -> Store { let genesis: &str = include_str!("../../../../test_data/genesis-l1.json"); let genesis: Genesis = serde_json::from_str(genesis).expect("Fatal: test config is invalid"); let store = Store::new("test-store", EngineType::InMemory) .expect("Fail to create in-memory db test"); - store.add_initial_state(genesis).unwrap(); + store.add_initial_state(genesis).await.unwrap(); store } - pub fn add_legacy_tx_blocks(storage: &Store, block_count: u64, tx_count: u64) { + pub async fn add_legacy_tx_blocks(storage: &Store, block_count: u64, tx_count: u64) { for block_num in 1..=block_count { let mut txs = vec![]; for nonce in 1..=tx_count { txs.push(legacy_tx_for_test(nonce)); } - add_blocks_with_transactions(storage, block_num, txs); + add_blocks_with_transactions(storage, block_num, txs).await; } } - pub fn add_eip1559_tx_blocks(storage: &Store, block_count: u64, tx_count: u64) { + pub async fn add_eip1559_tx_blocks(storage: &Store, block_count: u64, tx_count: u64) { for block_num in 1..=block_count { let mut txs = vec![]; for nonce in 1..=tx_count { txs.push(eip1559_tx_for_test(nonce)); } - add_blocks_with_transactions(storage, block_num, txs); + add_blocks_with_transactions(storage, block_num, txs).await; } } - pub fn add_mixed_tx_blocks(storage: &Store, block_count: u64, tx_count: u64) { + pub async fn add_mixed_tx_blocks(storage: &Store, block_count: u64, tx_count: u64) { for block_num in 1..=block_count { let mut txs = vec![]; for nonce in 1..=tx_count { @@ -171,7 +172,7 @@ pub mod test_utils { txs.push(eip1559_tx_for_test(nonce)); } } - add_blocks_with_transactions(storage, block_num, txs); + add_blocks_with_transactions(storage, block_num, txs).await; } } } diff --git a/crates/networking/rpc/eth/transaction.rs b/crates/networking/rpc/eth/transaction.rs index cd8081c5f9..bbf812d15c 100644 --- a/crates/networking/rpc/eth/transaction.rs +++ b/crates/networking/rpc/eth/transaction.rs @@ -102,7 +102,7 @@ impl RpcHandler for CallRequest { block, }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let block = self.block.clone().unwrap_or_default(); info!("Requested call on block: {}", block); let header = match block.resolve_block_header(&context.storage)? { @@ -144,7 +144,7 @@ impl RpcHandler for GetTransactionByBlockNumberAndIndexRequest { }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { info!( "Requested transaction at index: {} of block with number: {}", self.transaction_index, self.block, @@ -195,7 +195,7 @@ impl RpcHandler for GetTransactionByBlockHashAndIndexRequest { .map_err(|error| RpcErr::BadParams(error.to_string()))?, }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { info!( "Requested transaction at index: {} of block with hash: {:#x}", self.transaction_index, self.block, @@ -233,7 +233,7 @@ impl RpcHandler for GetTransactionByHashRequest { transaction_hash: serde_json::from_value(params[0].clone())?, }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let storage = &context.storage; info!( "Requested transaction with hash: {:#x}", @@ -272,7 +272,7 @@ impl RpcHandler for GetTransactionReceiptRequest { transaction_hash: serde_json::from_value(params[0].clone())?, }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let storage = &context.storage; info!( "Requested receipt for transaction {:#x}", @@ -319,7 +319,7 @@ impl RpcHandler for CreateAccessListRequest { block, }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let block = self.block.clone().unwrap_or_default(); info!("Requested access list creation for tx on block: {}", block); let block_number = match block.resolve_block_number(&context.storage)? { @@ -379,7 +379,7 @@ impl RpcHandler for GetRawTransaction { }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let tx = context .storage .get_transaction_by_hash(self.transaction_hash)?; @@ -418,7 +418,7 @@ impl RpcHandler for EstimateGasRequest { block, }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let storage = &context.storage; let blockchain = &context.blockchain; let block = self.block.clone().unwrap_or_default(); @@ -597,7 +597,7 @@ impl RpcHandler for SendRawTransactionRequest { let gateway_request = gateway_eth_client.send_raw_transaction(&tx_data); - let client_response = Self::call(req, context); + let client_response = Self::call(req, context).await; let gateway_response = gateway_request .await @@ -620,7 +620,7 @@ impl RpcHandler for SendRawTransactionRequest { gateway_response.or(client_response) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { let hash = if let SendRawTransactionRequest::EIP4844(wrapped_blob_tx) = self { context.blockchain.add_blob_transaction_to_pool( wrapped_blob_tx.tx.clone(), diff --git a/crates/networking/rpc/l2/transaction.rs b/crates/networking/rpc/l2/transaction.rs index 8c21fbec86..f84c5e184f 100644 --- a/crates/networking/rpc/l2/transaction.rs +++ b/crates/networking/rpc/l2/transaction.rs @@ -61,7 +61,7 @@ impl RpcHandler for SponsoredTx { }) } - fn handle(&self, context: RpcApiContext) -> Result { + async fn handle(&self, context: RpcApiContext) -> Result { // Dont allow create txs if self.to.is_zero() { return Err(RpcErr::InvalidEthrexL2Message( @@ -132,7 +132,7 @@ impl RpcHandler for SponsoredTx { let max_priority_fee_per_gas = estimate_gas_tip(&context.storage) .map_err(RpcErr::from)? .unwrap_or_default(); - let gas_price_request = GasPrice {}.handle(context.clone())?; + let gas_price_request = GasPrice {}.handle(context.clone()).await?; let max_fee_per_gas = u64::from_str_radix( gas_price_request .as_str() @@ -185,7 +185,8 @@ impl RpcHandler for SponsoredTx { transaction: generic, block: None, } - .handle(context.clone())?; + .handle(context.clone()) + .await?; let gas_limit = u64::from_str_radix( estimate_gas_request .as_str() @@ -222,6 +223,6 @@ impl RpcHandler for SponsoredTx { } } - tx.handle(context) + tx.handle(context).await } } diff --git a/crates/networking/rpc/rpc.rs b/crates/networking/rpc/rpc.rs index 709862c8a1..33ab5de831 100644 --- a/crates/networking/rpc/rpc.rs +++ b/crates/networking/rpc/rpc.rs @@ -107,9 +107,9 @@ pub struct RpcApiContext { pub trait RpcHandler: Sized { fn parse(params: &Option>) -> Result; - fn call(req: &RpcRequest, context: RpcApiContext) -> Result { + async fn call(req: &RpcRequest, context: RpcApiContext) -> Result { let request = Self::parse(&req.params)?; - request.handle(context) + request.handle(context).await } /// Relay the request to the gateway client, if the request fails, fallback to the local node @@ -121,10 +121,10 @@ pub trait RpcHandler: Sized { req: &RpcRequest, context: RpcApiContext, ) -> Result { - Self::call(req, context) + Self::call(req, context).await } - fn handle(&self, context: RpcApiContext) -> Result; + async fn handle(&self, context: RpcApiContext) -> Result; } pub const FILTER_DURATION: Duration = { @@ -198,8 +198,10 @@ pub async fn start_api( .with_state(service_context.clone()); let http_listener = TcpListener::bind(http_addr).await.unwrap(); + // We need a lambda with an async block because async lambdas are not supported + let authrpc_handler = |ctx, auth, body| async { handle_authrpc_request(ctx, auth, body).await }; let authrpc_router = Router::new() - .route("/", post(handle_authrpc_request)) + .route("/", post(authrpc_handler)) .with_state(service_context); let authrpc_listener = TcpListener::bind(authrpc_addr).await.unwrap(); @@ -287,7 +289,7 @@ pub async fn map_http_requests(req: &RpcRequest, context: RpcApiContext) -> Resu Ok(RpcNamespace::Based) => map_based_requests(req, context), Err(rpc_err) => Err(rpc_err), #[cfg(feature = "l2")] - Ok(RpcNamespace::EthrexL2) => map_l2_requests(req, context), + Ok(RpcNamespace::EthrexL2) => map_l2_requests(req, context).await, } } @@ -305,34 +307,36 @@ pub async fn map_authrpc_requests( pub async fn map_eth_requests(req: &RpcRequest, context: RpcApiContext) -> Result { match req.method.as_str() { - "eth_chainId" => ChainId::call(req, context), - "eth_syncing" => Syncing::call(req, context), - "eth_getBlockByNumber" => GetBlockByNumberRequest::call(req, context), - "eth_getBlockByHash" => GetBlockByHashRequest::call(req, context), - "eth_getBalance" => GetBalanceRequest::call(req, context), - "eth_getCode" => GetCodeRequest::call(req, context), - "eth_getStorageAt" => GetStorageAtRequest::call(req, context), + "eth_chainId" => ChainId::call(req, context).await, + "eth_syncing" => Syncing::call(req, context).await, + "eth_getBlockByNumber" => GetBlockByNumberRequest::call(req, context).await, + "eth_getBlockByHash" => GetBlockByHashRequest::call(req, context).await, + "eth_getBalance" => GetBalanceRequest::call(req, context).await, + "eth_getCode" => GetCodeRequest::call(req, context).await, + "eth_getStorageAt" => GetStorageAtRequest::call(req, context).await, "eth_getBlockTransactionCountByNumber" => { - GetBlockTransactionCountRequest::call(req, context) + GetBlockTransactionCountRequest::call(req, context).await + } + "eth_getBlockTransactionCountByHash" => { + GetBlockTransactionCountRequest::call(req, context).await } - "eth_getBlockTransactionCountByHash" => GetBlockTransactionCountRequest::call(req, context), "eth_getTransactionByBlockNumberAndIndex" => { - GetTransactionByBlockNumberAndIndexRequest::call(req, context) + GetTransactionByBlockNumberAndIndexRequest::call(req, context).await } "eth_getTransactionByBlockHashAndIndex" => { - GetTransactionByBlockHashAndIndexRequest::call(req, context) + GetTransactionByBlockHashAndIndexRequest::call(req, context).await } - "eth_getBlockReceipts" => GetBlockReceiptsRequest::call(req, context), - "eth_getTransactionByHash" => GetTransactionByHashRequest::call(req, context), - "eth_getTransactionReceipt" => GetTransactionReceiptRequest::call(req, context), - "eth_createAccessList" => CreateAccessListRequest::call(req, context), - "eth_blockNumber" => BlockNumberRequest::call(req, context), - "eth_call" => CallRequest::call(req, context), - "eth_blobBaseFee" => GetBlobBaseFee::call(req, context), - "eth_getTransactionCount" => GetTransactionCountRequest::call(req, context), - "eth_feeHistory" => FeeHistoryRequest::call(req, context), - "eth_estimateGas" => EstimateGasRequest::call(req, context), - "eth_getLogs" => LogsFilter::call(req, context), + "eth_getBlockReceipts" => GetBlockReceiptsRequest::call(req, context).await, + "eth_getTransactionByHash" => GetTransactionByHashRequest::call(req, context).await, + "eth_getTransactionReceipt" => GetTransactionReceiptRequest::call(req, context).await, + "eth_createAccessList" => CreateAccessListRequest::call(req, context).await, + "eth_blockNumber" => BlockNumberRequest::call(req, context).await, + "eth_call" => CallRequest::call(req, context).await, + "eth_blobBaseFee" => GetBlobBaseFee::call(req, context).await, + "eth_getTransactionCount" => GetTransactionCountRequest::call(req, context).await, + "eth_feeHistory" => FeeHistoryRequest::call(req, context).await, + "eth_estimateGas" => EstimateGasRequest::call(req, context).await, + "eth_getLogs" => LogsFilter::call(req, context).await, "eth_newFilter" => { NewFilterRequest::stateful_call(req, context.storage, context.active_filters) } @@ -347,23 +351,25 @@ pub async fn map_eth_requests(req: &RpcRequest, context: RpcApiContext) -> Resul if #[cfg(feature = "based")] { SendRawTransactionRequest::relay_to_gateway_or_fallback(req, context).await } else { - SendRawTransactionRequest::call(req, context) + SendRawTransactionRequest::call(req, context).await } } } - "eth_getProof" => GetProofRequest::call(req, context), - "eth_gasPrice" => GasPrice::call(req, context), - "eth_maxPriorityFeePerGas" => eth::max_priority_fee::MaxPriorityFee::call(req, context), + "eth_getProof" => GetProofRequest::call(req, context).await, + "eth_gasPrice" => GasPrice::call(req, context).await, + "eth_maxPriorityFeePerGas" => { + eth::max_priority_fee::MaxPriorityFee::call(req, context).await + } unknown_eth_method => Err(RpcErr::MethodNotFound(unknown_eth_method.to_owned())), } } pub async fn map_debug_requests(req: &RpcRequest, context: RpcApiContext) -> Result { match req.method.as_str() { - "debug_getRawHeader" => GetRawHeaderRequest::call(req, context), - "debug_getRawBlock" => GetRawBlockRequest::call(req, context), - "debug_getRawTransaction" => GetRawTransaction::call(req, context), - "debug_getRawReceipts" => GetRawReceipts::call(req, context), + "debug_getRawHeader" => GetRawHeaderRequest::call(req, context).await, + "debug_getRawBlock" => GetRawBlockRequest::call(req, context).await, + "debug_getRawTransaction" => GetRawTransaction::call(req, context).await, + "debug_getRawReceipts" => GetRawReceipts::call(req, context).await, unknown_debug_method => Err(RpcErr::MethodNotFound(unknown_debug_method.to_owned())), } } @@ -373,47 +379,51 @@ pub async fn map_engine_requests( context: RpcApiContext, ) -> Result { match req.method.as_str() { - "engine_exchangeCapabilities" => ExchangeCapabilitiesRequest::call(req, context), - "engine_forkchoiceUpdatedV1" => ForkChoiceUpdatedV1::call(req, context), - "engine_forkchoiceUpdatedV2" => ForkChoiceUpdatedV2::call(req, context), + "engine_exchangeCapabilities" => ExchangeCapabilitiesRequest::call(req, context).await, + "engine_forkchoiceUpdatedV1" => ForkChoiceUpdatedV1::call(req, context).await, + "engine_forkchoiceUpdatedV2" => ForkChoiceUpdatedV2::call(req, context).await, "engine_forkchoiceUpdatedV3" => { cfg_if::cfg_if! { if #[cfg(feature = "based")] { ForkChoiceUpdatedV3::relay_to_gateway_or_fallback(req, context).await } else { - ForkChoiceUpdatedV3::call(req, context) + ForkChoiceUpdatedV3::call(req, context).await } } } - "engine_newPayloadV4" => NewPayloadV4Request::call(req, context), + "engine_newPayloadV4" => NewPayloadV4Request::call(req, context).await, "engine_newPayloadV3" => { cfg_if::cfg_if! { if #[cfg(feature = "based")] { NewPayloadV3Request::relay_to_gateway_or_fallback(req, context).await } else { - NewPayloadV3Request::call(req, context) + NewPayloadV3Request::call(req, context).await } } } - "engine_newPayloadV2" => NewPayloadV2Request::call(req, context), - "engine_newPayloadV1" => NewPayloadV1Request::call(req, context), + "engine_newPayloadV2" => NewPayloadV2Request::call(req, context).await, + "engine_newPayloadV1" => NewPayloadV1Request::call(req, context).await, "engine_exchangeTransitionConfigurationV1" => { - ExchangeTransitionConfigV1Req::call(req, context) + ExchangeTransitionConfigV1Req::call(req, context).await } - "engine_getPayloadV4" => GetPayloadV4Request::call(req, context), + "engine_getPayloadV4" => GetPayloadV4Request::call(req, context).await, "engine_getPayloadV3" => { cfg_if::cfg_if! { if #[cfg(feature = "based")] { GetPayloadV3Request::relay_to_gateway_or_fallback(req, context).await } else { - GetPayloadV3Request::call(req, context) + GetPayloadV3Request::call(req, context).await } } } - "engine_getPayloadV2" => GetPayloadV2Request::call(req, context), - "engine_getPayloadV1" => GetPayloadV1Request::call(req, context), - "engine_getPayloadBodiesByHashV1" => GetPayloadBodiesByHashV1Request::call(req, context), - "engine_getPayloadBodiesByRangeV1" => GetPayloadBodiesByRangeV1Request::call(req, context), + "engine_getPayloadV2" => GetPayloadV2Request::call(req, context).await, + "engine_getPayloadV1" => GetPayloadV1Request::call(req, context).await, + "engine_getPayloadBodiesByHashV1" => { + GetPayloadBodiesByHashV1Request::call(req, context).await + } + "engine_getPayloadBodiesByRangeV1" => { + GetPayloadBodiesByRangeV1Request::call(req, context).await + } unknown_engine_method => Err(RpcErr::MethodNotFound(unknown_engine_method.to_owned())), } } @@ -454,9 +464,9 @@ pub fn map_based_requests(req: &RpcRequest, context: RpcApiContext) -> Result Result { +pub async fn map_l2_requests(req: &RpcRequest, context: RpcApiContext) -> Result { match req.method.as_str() { - "ethrex_sendTransaction" => SponsoredTx::call(req, context), + "ethrex_sendTransaction" => SponsoredTx::call(req, context).await, unknown_ethrex_l2_method => { Err(RpcErr::MethodNotFound(unknown_ethrex_l2_method.to_owned())) } @@ -517,7 +527,10 @@ mod tests { let local_p2p_node = example_p2p_node(); let storage = Store::new("temp.db", EngineType::InMemory).expect("Failed to create test DB"); - storage.set_chain_config(&example_chain_config()).unwrap(); + storage + .set_chain_config(&example_chain_config()) + .await + .unwrap(); let blockchain = Arc::new(Blockchain::default_with_store(storage.clone())); let context = RpcApiContext { local_p2p_node, @@ -614,6 +627,7 @@ mod tests { let genesis = read_execution_api_genesis_file(); storage .add_initial_state(genesis) + .await .expect("Failed to add genesis block to DB"); let local_p2p_node = example_p2p_node(); // Process request @@ -657,6 +671,7 @@ mod tests { let genesis = read_execution_api_genesis_file(); storage .add_initial_state(genesis) + .await .expect("Failed to add genesis block to DB"); let local_p2p_node = example_p2p_node(); // Process request @@ -727,7 +742,10 @@ mod tests { // Setup initial storage let storage = Store::new("temp.db", EngineType::InMemory).expect("Failed to create test DB"); - storage.set_chain_config(&example_chain_config()).unwrap(); + storage + .set_chain_config(&example_chain_config()) + .await + .unwrap(); let blockchain = Arc::new(Blockchain::default_with_store(storage.clone())); let chain_id = storage .get_chain_config() diff --git a/crates/networking/rpc/utils.rs b/crates/networking/rpc/utils.rs index c77293bf74..6ce7d1225d 100644 --- a/crates/networking/rpc/utils.rs +++ b/crates/networking/rpc/utils.rs @@ -342,6 +342,7 @@ pub mod test_utils { Store::new("", EngineType::InMemory).expect("Failed to create in-memory storage"); storage .add_initial_state(serde_json::from_str(TEST_GENESIS).unwrap()) + .await .expect("Failed to build test genesis"); let blockchain = Arc::new(Blockchain::default_with_store(storage.clone())); let jwt_secret = Default::default(); diff --git a/crates/storage/Cargo.toml b/crates/storage/Cargo.toml index fcace9632b..94ba6d65da 100644 --- a/crates/storage/Cargo.toml +++ b/crates/storage/Cargo.toml @@ -10,6 +10,7 @@ ethrex-rlp.workspace = true ethrex-common.workspace = true ethrex-trie.workspace = true +async-trait.workspace = true ethereum-types.workspace = true anyhow = "1.0.86" bytes.workspace = true @@ -21,11 +22,14 @@ serde = { version = "1.0.203", features = ["derive"] } serde_json = "1.0.117" libmdbx = { workspace = true, optional = true } redb = { workspace = true, optional = true } +# NOTE: intentionally avoiding the workspace dep as it brings "full" features, breaking the provers +# We only need the runtime for the blocking databases to spawn blocking tasks +tokio = { version = "1.41.1", optional = true, default-features = false, features = ["rt"] } [features] default = [] -libmdbx = ["dep:libmdbx", "ethrex-trie/libmdbx"] -redb = ["dep:redb"] +libmdbx = ["dep:libmdbx", "ethrex-trie/libmdbx", "dep:tokio"] +redb = ["dep:redb", "dep:tokio"] [dev-dependencies] hex.workspace = true diff --git a/crates/storage/api.rs b/crates/storage/api.rs index 1ebe823bc6..3b0cb6a9d6 100644 --- a/crates/storage/api.rs +++ b/crates/storage/api.rs @@ -9,23 +9,26 @@ use std::{collections::HashMap, fmt::Debug, panic::RefUnwindSafe}; use crate::{error::StoreError, store::STATE_TRIE_SEGMENTS}; use ethrex_trie::{Nibbles, Trie}; +// We need async_trait because the stabilized feature lacks support for object safety +// (i.e. dyn StoreEngine) +#[async_trait::async_trait] pub trait StoreEngine: Debug + Send + Sync + RefUnwindSafe { /// Add a batch of blocks in a single transaction. /// This will store -> BlockHeader, BlockBody, BlockTransactions, BlockNumber. - fn add_blocks(&self, blocks: &[Block]) -> Result<(), StoreError>; + async fn add_blocks(&self, blocks: Vec) -> Result<(), StoreError>; /// Sets the blocks as part of the canonical chain - fn mark_chain_as_canonical(&self, blocks: &[Block]) -> Result<(), StoreError>; + async fn mark_chain_as_canonical(&self, blocks: &[Block]) -> Result<(), StoreError>; /// Add block header - fn add_block_header( + async fn add_block_header( &self, block_hash: BlockHash, block_header: BlockHeader, ) -> Result<(), StoreError>; /// Add a batch of block headers - fn add_block_headers( + async fn add_block_headers( &self, block_hashes: Vec, block_headers: Vec, @@ -38,7 +41,7 @@ pub trait StoreEngine: Debug + Send + Sync + RefUnwindSafe { ) -> Result, StoreError>; /// Add block body - fn add_block_body( + async fn add_block_body( &self, block_hash: BlockHash, block_body: BlockBody, @@ -58,11 +61,11 @@ pub trait StoreEngine: Debug + Send + Sync + RefUnwindSafe { block_hash: BlockHash, ) -> Result, StoreError>; - fn add_pending_block(&self, block: Block) -> Result<(), StoreError>; + async fn add_pending_block(&self, block: Block) -> Result<(), StoreError>; fn get_pending_block(&self, block_hash: BlockHash) -> Result, StoreError>; /// Add block number for a given hash - fn add_block_number( + async fn add_block_number( &self, block_hash: BlockHash, block_number: BlockNumber, @@ -72,7 +75,7 @@ pub trait StoreEngine: Debug + Send + Sync + RefUnwindSafe { fn get_block_number(&self, block_hash: BlockHash) -> Result, StoreError>; /// Store transaction location (block number and index of the transaction within the block) - fn add_transaction_location( + async fn add_transaction_location( &self, transaction_hash: H256, block_number: BlockNumber, @@ -81,7 +84,7 @@ pub trait StoreEngine: Debug + Send + Sync + RefUnwindSafe { ) -> Result<(), StoreError>; /// Store transaction locations in batch (one db transaction for all) - fn add_transaction_locations( + async fn add_transaction_locations( &self, locations: Vec<(H256, BlockNumber, BlockHash, Index)>, ) -> Result<(), StoreError>; @@ -93,7 +96,7 @@ pub trait StoreEngine: Debug + Send + Sync + RefUnwindSafe { ) -> Result, StoreError>; /// Add receipt - fn add_receipt( + async fn add_receipt( &self, block_hash: BlockHash, index: Index, @@ -101,11 +104,14 @@ pub trait StoreEngine: Debug + Send + Sync + RefUnwindSafe { ) -> Result<(), StoreError>; /// Add receipts - fn add_receipts(&self, block_hash: BlockHash, receipts: Vec) - -> Result<(), StoreError>; + async fn add_receipts( + &self, + block_hash: BlockHash, + receipts: Vec, + ) -> Result<(), StoreError>; /// Adds receipts for a batch of blocks - fn add_receipts_for_blocks( + async fn add_receipts_for_blocks( &self, receipts: HashMap>, ) -> Result<(), StoreError>; @@ -118,7 +124,7 @@ pub trait StoreEngine: Debug + Send + Sync + RefUnwindSafe { ) -> Result, StoreError>; /// Add account code - fn add_account_code(&self, code_hash: H256, code: Bytes) -> Result<(), StoreError>; + async fn add_account_code(&self, code_hash: H256, code: Bytes) -> Result<(), StoreError>; /// Obtain account code via code hash fn get_account_code(&self, code_hash: H256) -> Result, StoreError>; @@ -170,37 +176,47 @@ pub trait StoreEngine: Debug + Send + Sync + RefUnwindSafe { /// Stores the chain configuration values, should only be called once after reading the genesis file /// Ignores previously stored values if present - fn set_chain_config(&self, chain_config: &ChainConfig) -> Result<(), StoreError>; + async fn set_chain_config(&self, chain_config: &ChainConfig) -> Result<(), StoreError>; /// Returns the stored chain configuration fn get_chain_config(&self) -> Result; /// Update earliest block number - fn update_earliest_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError>; + async fn update_earliest_block_number( + &self, + block_number: BlockNumber, + ) -> Result<(), StoreError>; /// Obtain earliest block number fn get_earliest_block_number(&self) -> Result, StoreError>; /// Update finalized block number - fn update_finalized_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError>; + async fn update_finalized_block_number( + &self, + block_number: BlockNumber, + ) -> Result<(), StoreError>; /// Obtain finalized block number fn get_finalized_block_number(&self) -> Result, StoreError>; /// Update safe block number - fn update_safe_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError>; + async fn update_safe_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError>; /// Obtain safe block number fn get_safe_block_number(&self) -> Result, StoreError>; /// Update latest block number - fn update_latest_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError>; + async fn update_latest_block_number(&self, block_number: BlockNumber) + -> Result<(), StoreError>; /// Obtain latest block number fn get_latest_block_number(&self) -> Result, StoreError>; /// Update pending block number - fn update_pending_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError>; + async fn update_pending_block_number( + &self, + block_number: BlockNumber, + ) -> Result<(), StoreError>; /// Obtain pending block number fn get_pending_block_number(&self) -> Result, StoreError>; @@ -216,29 +232,38 @@ pub trait StoreEngine: Debug + Send + Sync + RefUnwindSafe { fn open_state_trie(&self, state_root: H256) -> Trie; /// Set the canonical block hash for a given block number. - fn set_canonical_block(&self, number: BlockNumber, hash: BlockHash) -> Result<(), StoreError>; + async fn set_canonical_block( + &self, + number: BlockNumber, + hash: BlockHash, + ) -> Result<(), StoreError>; /// Unsets canonical block for a block number. - fn unset_canonical_block(&self, number: BlockNumber) -> Result<(), StoreError>; + async fn unset_canonical_block(&self, number: BlockNumber) -> Result<(), StoreError>; - fn add_payload(&self, payload_id: u64, block: Block) -> Result<(), StoreError>; + async fn add_payload(&self, payload_id: u64, block: Block) -> Result<(), StoreError>; fn get_payload(&self, payload_id: u64) -> Result, StoreError>; - fn update_payload(&self, payload_id: u64, payload: PayloadBundle) -> Result<(), StoreError>; + async fn update_payload( + &self, + payload_id: u64, + payload: PayloadBundle, + ) -> Result<(), StoreError>; fn get_receipts_for_block(&self, block_hash: &BlockHash) -> Result, StoreError>; // Snap State methods /// Sets the hash of the last header downloaded during a snap sync - fn set_header_download_checkpoint(&self, block_hash: BlockHash) -> Result<(), StoreError>; + async fn set_header_download_checkpoint(&self, block_hash: BlockHash) + -> Result<(), StoreError>; /// Gets the hash of the last header downloaded during a snap sync fn get_header_download_checkpoint(&self) -> Result, StoreError>; /// Sets the last key fetched from the state trie being fetched during snap sync - fn set_state_trie_key_checkpoint( + async fn set_state_trie_key_checkpoint( &self, last_keys: [H256; STATE_TRIE_SEGMENTS], ) -> Result<(), StoreError>; @@ -249,35 +274,37 @@ pub trait StoreEngine: Debug + Send + Sync + RefUnwindSafe { ) -> Result, StoreError>; /// Sets the storage trie paths in need of healing, grouped by hashed address - fn set_storage_heal_paths(&self, accounts: Vec<(H256, Vec)>) - -> Result<(), StoreError>; + async fn set_storage_heal_paths( + &self, + accounts: Vec<(H256, Vec)>, + ) -> Result<(), StoreError>; /// Gets the storage trie paths in need of healing, grouped by hashed address #[allow(clippy::type_complexity)] fn get_storage_heal_paths(&self) -> Result)>>, StoreError>; /// Sets the state trie paths in need of healing - fn set_state_heal_paths(&self, paths: Vec) -> Result<(), StoreError>; + async fn set_state_heal_paths(&self, paths: Vec) -> Result<(), StoreError>; /// Gets the state trie paths in need of healing fn get_state_heal_paths(&self) -> Result>, StoreError>; /// Clears all checkpoint data created during the last snap sync - fn clear_snap_state(&self) -> Result<(), StoreError>; + async fn clear_snap_state(&self) -> Result<(), StoreError>; fn is_synced(&self) -> Result; - fn update_sync_status(&self, status: bool) -> Result<(), StoreError>; + async fn update_sync_status(&self, status: bool) -> Result<(), StoreError>; /// Write an account batch into the current state snapshot - fn write_snapshot_account_batch( + async fn write_snapshot_account_batch( &self, account_hashes: Vec, account_states: Vec, ) -> Result<(), StoreError>; /// Write a storage batch into the current storage snapshot - fn write_snapshot_storage_batch( + async fn write_snapshot_storage_batch( &self, account_hash: H256, storage_keys: Vec, @@ -285,7 +312,7 @@ pub trait StoreEngine: Debug + Send + Sync + RefUnwindSafe { ) -> Result<(), StoreError>; /// Write multiple storage batches belonging to different accounts into the current storage snapshot - fn write_snapshot_storage_batches( + async fn write_snapshot_storage_batches( &self, account_hashes: Vec, storage_keys: Vec>, @@ -293,7 +320,7 @@ pub trait StoreEngine: Debug + Send + Sync + RefUnwindSafe { ) -> Result<(), StoreError>; /// Set the latest root of the rebuilt state trie and the last downloaded hashes from each segment - fn set_state_trie_rebuild_checkpoint( + async fn set_state_trie_rebuild_checkpoint( &self, checkpoint: (H256, [H256; STATE_TRIE_SEGMENTS]), ) -> Result<(), StoreError>; @@ -304,7 +331,7 @@ pub trait StoreEngine: Debug + Send + Sync + RefUnwindSafe { ) -> Result, StoreError>; /// Get the accont hashes and roots of the storage tries awaiting rebuild - fn set_storage_trie_rebuild_pending( + async fn set_storage_trie_rebuild_pending( &self, pending: Vec<(H256, H256)>, ) -> Result<(), StoreError>; @@ -313,7 +340,7 @@ pub trait StoreEngine: Debug + Send + Sync + RefUnwindSafe { fn get_storage_trie_rebuild_pending(&self) -> Result>, StoreError>; /// Clears the state and storage snapshots - fn clear_snapshot(&self) -> Result<(), StoreError>; + async fn clear_snapshot(&self) -> Result<(), StoreError>; /// Reads the next `MAX_SNAPSHOT_READS` accounts from the state snapshot as from the `start` hash fn read_account_snapshot(&self, start: H256) -> Result, StoreError>; diff --git a/crates/storage/store.rs b/crates/storage/store.rs index ede3ec8597..5cb798c867 100644 --- a/crates/storage/store.rs +++ b/crates/storage/store.rs @@ -94,7 +94,7 @@ impl Store { Ok(store) } - pub fn new_from_genesis( + pub async fn new_from_genesis( store_path: &str, engine_type: EngineType, genesis_path: &str, @@ -105,7 +105,7 @@ impl Store { let genesis: Genesis = serde_json::from_reader(reader).expect("Failed to deserialize genesis file"); let store = Self::new(store_path, engine_type)?; - store.add_initial_state(genesis)?; + store.add_initial_state(genesis).await?; Ok(store) } @@ -140,20 +140,22 @@ impl Store { })) } - pub fn add_block_header( + pub async fn add_block_header( &self, block_hash: BlockHash, block_header: BlockHeader, ) -> Result<(), StoreError> { - self.engine.add_block_header(block_hash, block_header) + self.engine.add_block_header(block_hash, block_header).await } - pub fn add_block_headers( + pub async fn add_block_headers( &self, block_hashes: Vec, block_headers: Vec, ) -> Result<(), StoreError> { - self.engine.add_block_headers(block_hashes, block_headers) + self.engine + .add_block_headers(block_hashes, block_headers) + .await } pub fn get_block_header( @@ -177,12 +179,12 @@ impl Store { self.engine.get_block_body_by_hash(block_hash) } - pub fn add_block_body( + pub async fn add_block_body( &self, block_hash: BlockHash, block_body: BlockBody, ) -> Result<(), StoreError> { - self.engine.add_block_body(block_hash, block_body) + self.engine.add_block_body(block_hash, block_body).await } pub fn get_block_body( @@ -192,12 +194,12 @@ impl Store { self.engine.get_block_body(block_number) } - pub fn add_pending_block(&self, block: Block) -> Result<(), StoreError> { + pub async fn add_pending_block(&self, block: Block) -> Result<(), StoreError> { info!( "Adding block to pending: {}", block.header.compute_block_hash() ); - self.engine.add_pending_block(block) + self.engine.add_pending_block(block).await } pub fn get_pending_block(&self, block_hash: BlockHash) -> Result, StoreError> { @@ -205,7 +207,7 @@ impl Store { self.engine.get_pending_block(block_hash) } - pub fn add_block_number( + pub async fn add_block_number( &self, block_hash: BlockHash, block_number: BlockNumber, @@ -213,6 +215,7 @@ impl Store { self.engine .clone() .add_block_number(block_hash, block_number) + .await } pub fn get_block_number( @@ -222,7 +225,7 @@ impl Store { self.engine.get_block_number(block_hash) } - pub fn add_transaction_location( + pub async fn add_transaction_location( &self, transaction_hash: H256, block_number: BlockNumber, @@ -231,9 +234,10 @@ impl Store { ) -> Result<(), StoreError> { self.engine .add_transaction_location(transaction_hash, block_number, block_hash, index) + .await } - pub fn add_transaction_locations( + pub async fn add_transaction_locations( &self, transactions: &[Transaction], block_number: BlockNumber, @@ -250,7 +254,7 @@ impl Store { )); } - self.engine.add_transaction_locations(locations) + self.engine.add_transaction_locations(locations).await } pub fn get_transaction_location( @@ -260,8 +264,8 @@ impl Store { self.engine.get_transaction_location(transaction_hash) } - pub fn add_account_code(&self, code_hash: H256, code: Bytes) -> Result<(), StoreError> { - self.engine.add_account_code(code_hash, code) + pub async fn add_account_code(&self, code_hash: H256, code: Bytes) -> Result<(), StoreError> { + self.engine.add_account_code(code_hash, code).await } pub fn get_account_code(&self, code_hash: H256) -> Result, StoreError> { @@ -308,7 +312,7 @@ impl Store { /// Applies account updates based on the block's latest storage state /// and returns the new state root after the updates have been applied. - pub fn apply_account_updates( + pub async fn apply_account_updates( &self, block_hash: BlockHash, account_updates: &[AccountUpdate], @@ -317,11 +321,13 @@ impl Store { return Ok(None); }; - let mut state_trie = self.apply_account_updates_from_trie(state_trie, account_updates)?; + let mut state_trie = self + .apply_account_updates_from_trie(state_trie, account_updates) + .await?; Ok(Some(state_trie.hash()?)) } - pub fn apply_account_updates_from_trie( + pub async fn apply_account_updates_from_trie( &self, mut state_trie: Trie, account_updates: &[AccountUpdate], @@ -344,7 +350,7 @@ impl Store { account_state.code_hash = info.code_hash; // Store updated code in DB if let Some(code) = &update.code { - self.add_account_code(info.code_hash, code.clone())?; + self.add_account_code(info.code_hash, code.clone()).await?; } } // Store the added storage in the account's storage trie and compute its new root @@ -371,7 +377,7 @@ impl Store { } /// Adds all genesis accounts and returns the genesis block's state_root - pub fn setup_genesis_state_trie( + pub async fn setup_genesis_state_trie( &self, genesis_accounts: BTreeMap, ) -> Result { @@ -380,7 +386,7 @@ impl Store { let hashed_address = hash_address(&address); // Store account code (as this won't be stored in the trie) let code_hash = code_hash(&account.code); - self.add_account_code(code_hash, account.code)?; + self.add_account_code(code_hash, account.code).await?; // Store the account's storage in a clean storage trie and compute its root let mut storage_trie = self .engine @@ -404,28 +410,28 @@ impl Store { Ok(genesis_state_trie.hash()?) } - pub fn add_receipt( + pub async fn add_receipt( &self, block_hash: BlockHash, index: Index, receipt: Receipt, ) -> Result<(), StoreError> { - self.engine.add_receipt(block_hash, index, receipt) + self.engine.add_receipt(block_hash, index, receipt).await } - pub fn add_receipts( + pub async fn add_receipts( &self, block_hash: BlockHash, receipts: Vec, ) -> Result<(), StoreError> { - self.engine.add_receipts(block_hash, receipts) + self.engine.add_receipts(block_hash, receipts).await } - pub fn add_receipts_for_blocks( + pub async fn add_receipts_for_blocks( &self, receipts: HashMap>, ) -> Result<(), StoreError> { - self.engine.add_receipts_for_blocks(receipts) + self.engine.add_receipts_for_blocks(receipts).await } pub fn get_receipt( @@ -436,21 +442,21 @@ impl Store { self.engine.get_receipt(block_number, index) } - pub fn add_block(&self, block: Block) -> Result<(), StoreError> { - self.add_blocks(&[block]) + pub async fn add_block(&self, block: Block) -> Result<(), StoreError> { + self.add_blocks(vec![block]).await } - pub fn add_blocks(&self, blocks: &[Block]) -> Result<(), StoreError> { - self.engine.add_blocks(blocks) + pub async fn add_blocks(&self, blocks: Vec) -> Result<(), StoreError> { + self.engine.add_blocks(blocks).await } - pub fn mark_chain_as_canonical(&self, blocks: &[Block]) -> Result<(), StoreError> { - self.engine.mark_chain_as_canonical(blocks) + pub async fn mark_chain_as_canonical(&self, blocks: &[Block]) -> Result<(), StoreError> { + self.engine.mark_chain_as_canonical(blocks).await } - pub fn add_initial_state(&self, genesis: Genesis) -> Result<(), StoreError> { + pub async fn add_initial_state(&self, genesis: Genesis) -> Result<(), StoreError> { info!("Setting initial sync status to false"); - self.update_sync_status(false)?; + self.update_sync_status(false).await?; info!("Storing initial state from genesis"); @@ -470,7 +476,7 @@ impl Store { } // Store genesis accounts // TODO: Should we use this root instead of computing it before the block hash check? - let genesis_state_root = self.setup_genesis_state_trie(genesis.alloc)?; + let genesis_state_root = self.setup_genesis_state_trie(genesis.alloc).await?; debug_assert_eq!(genesis_state_root, genesis_block.header.state_root); // Store genesis block @@ -479,13 +485,16 @@ impl Store { genesis_block_number, genesis_hash ); - self.add_block(genesis_block)?; - self.update_earliest_block_number(genesis_block_number)?; - self.update_latest_block_number(genesis_block_number)?; - self.set_canonical_block(genesis_block_number, genesis_hash)?; + self.add_block(genesis_block).await?; + self.update_earliest_block_number(genesis_block_number) + .await?; + self.update_latest_block_number(genesis_block_number) + .await?; + self.set_canonical_block(genesis_block_number, genesis_hash) + .await?; // Set chain config - self.set_chain_config(&genesis.config) + self.set_chain_config(&genesis.config).await } pub fn get_transaction_by_hash( @@ -535,19 +544,19 @@ impl Store { .transpose() } - pub fn set_chain_config(&self, chain_config: &ChainConfig) -> Result<(), StoreError> { - self.engine.set_chain_config(chain_config) + pub async fn set_chain_config(&self, chain_config: &ChainConfig) -> Result<(), StoreError> { + self.engine.set_chain_config(chain_config).await } pub fn get_chain_config(&self) -> Result { self.engine.get_chain_config() } - pub fn update_earliest_block_number( + pub async fn update_earliest_block_number( &self, block_number: BlockNumber, ) -> Result<(), StoreError> { - self.engine.update_earliest_block_number(block_number) + self.engine.update_earliest_block_number(block_number).await } pub fn get_earliest_block_number(&self) -> Result { @@ -556,27 +565,35 @@ impl Store { .ok_or(StoreError::MissingEarliestBlockNumber) } - pub fn update_finalized_block_number( + pub async fn update_finalized_block_number( &self, block_number: BlockNumber, ) -> Result<(), StoreError> { - self.engine.update_finalized_block_number(block_number) + self.engine + .update_finalized_block_number(block_number) + .await } pub fn get_finalized_block_number(&self) -> Result, StoreError> { self.engine.get_finalized_block_number() } - pub fn update_safe_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { - self.engine.update_safe_block_number(block_number) + pub async fn update_safe_block_number( + &self, + block_number: BlockNumber, + ) -> Result<(), StoreError> { + self.engine.update_safe_block_number(block_number).await } pub fn get_safe_block_number(&self) -> Result, StoreError> { self.engine.get_safe_block_number() } - pub fn update_latest_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { - self.engine.update_latest_block_number(block_number) + pub async fn update_latest_block_number( + &self, + block_number: BlockNumber, + ) -> Result<(), StoreError> { + self.engine.update_latest_block_number(block_number).await } pub fn get_latest_block_number(&self) -> Result { @@ -585,20 +602,23 @@ impl Store { .ok_or(StoreError::MissingLatestBlockNumber) } - pub fn update_pending_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { - self.engine.update_pending_block_number(block_number) + pub async fn update_pending_block_number( + &self, + block_number: BlockNumber, + ) -> Result<(), StoreError> { + self.engine.update_pending_block_number(block_number).await } pub fn get_pending_block_number(&self) -> Result, StoreError> { self.engine.get_pending_block_number() } - pub fn set_canonical_block( + pub async fn set_canonical_block( &self, number: BlockNumber, hash: BlockHash, ) -> Result<(), StoreError> { - self.engine.set_canonical_block(number, hash) + self.engine.set_canonical_block(number, hash).await } pub fn get_canonical_block_hash( @@ -619,8 +639,8 @@ impl Store { /// Marks a block number as not having any canonical blocks associated with it. /// Used for reorgs. /// Note: Should we also remove all others up to the head here? - pub fn unset_canonical_block(&self, number: BlockNumber) -> Result<(), StoreError> { - self.engine.unset_canonical_block(number) + pub async fn unset_canonical_block(&self, number: BlockNumber) -> Result<(), StoreError> { + self.engine.unset_canonical_block(number).await } /// Obtain the storage trie for the given block @@ -838,20 +858,20 @@ impl Store { Ok(nodes) } - pub fn add_payload(&self, payload_id: u64, block: Block) -> Result<(), StoreError> { - self.engine.add_payload(payload_id, block) + pub async fn add_payload(&self, payload_id: u64, block: Block) -> Result<(), StoreError> { + self.engine.add_payload(payload_id, block).await } pub fn get_payload(&self, payload_id: u64) -> Result, StoreError> { self.engine.get_payload(payload_id) } - pub fn update_payload( + pub async fn update_payload( &self, payload_id: u64, payload: PayloadBundle, ) -> Result<(), StoreError> { - self.engine.update_payload(payload_id, payload) + self.engine.update_payload(payload_id, payload).await } pub fn get_receipts_for_block( @@ -905,8 +925,11 @@ impl Store { } /// Sets the hash of the last header downloaded during a snap sync - pub fn set_header_download_checkpoint(&self, block_hash: BlockHash) -> Result<(), StoreError> { - self.engine.set_header_download_checkpoint(block_hash) + pub async fn set_header_download_checkpoint( + &self, + block_hash: BlockHash, + ) -> Result<(), StoreError> { + self.engine.set_header_download_checkpoint(block_hash).await } /// Gets the hash of the last header downloaded during a snap sync @@ -915,11 +938,11 @@ impl Store { } /// Sets the last key fetched from the state trie being fetched during snap sync - pub fn set_state_trie_key_checkpoint( + pub async fn set_state_trie_key_checkpoint( &self, last_keys: [H256; STATE_TRIE_SEGMENTS], ) -> Result<(), StoreError> { - self.engine.set_state_trie_key_checkpoint(last_keys) + self.engine.set_state_trie_key_checkpoint(last_keys).await } /// Gets the last key fetched from the state trie being fetched during snap sync @@ -930,11 +953,11 @@ impl Store { } /// Sets the storage trie paths in need of healing, grouped by hashed address - pub fn set_storage_heal_paths( + pub async fn set_storage_heal_paths( &self, accounts: Vec<(H256, Vec)>, ) -> Result<(), StoreError> { - self.engine.set_storage_heal_paths(accounts) + self.engine.set_storage_heal_paths(accounts).await } /// Gets the storage trie paths in need of healing, grouped by hashed address @@ -944,8 +967,8 @@ impl Store { } /// Sets the state trie paths in need of healing - pub fn set_state_heal_paths(&self, paths: Vec) -> Result<(), StoreError> { - self.engine.set_state_heal_paths(paths) + pub async fn set_state_heal_paths(&self, paths: Vec) -> Result<(), StoreError> { + self.engine.set_state_heal_paths(paths).await } /// Gets the state trie paths in need of healing @@ -956,22 +979,23 @@ impl Store { pub fn is_synced(&self) -> Result { self.engine.is_synced() } - pub fn update_sync_status(&self, status: bool) -> Result<(), StoreError> { - self.engine.update_sync_status(status) + pub async fn update_sync_status(&self, status: bool) -> Result<(), StoreError> { + self.engine.update_sync_status(status).await } /// Write an account batch into the current state snapshot - pub fn write_snapshot_account_batch( + pub async fn write_snapshot_account_batch( &self, account_hashes: Vec, account_states: Vec, ) -> Result<(), StoreError> { self.engine .write_snapshot_account_batch(account_hashes, account_states) + .await } /// Write a storage batch into the current storage snapshot - pub fn write_snapshot_storage_batch( + pub async fn write_snapshot_storage_batch( &self, account_hash: H256, storage_keys: Vec, @@ -979,10 +1003,11 @@ impl Store { ) -> Result<(), StoreError> { self.engine .write_snapshot_storage_batch(account_hash, storage_keys, storage_values) + .await } /// Write multiple storage batches belonging to different accounts into the current storage snapshot - pub fn write_snapshot_storage_batches( + pub async fn write_snapshot_storage_batches( &self, account_hashes: Vec, storage_keys: Vec>, @@ -990,19 +1015,22 @@ impl Store { ) -> Result<(), StoreError> { self.engine .write_snapshot_storage_batches(account_hashes, storage_keys, storage_values) + .await } /// Clears all checkpoint data created during the last snap sync - pub fn clear_snap_state(&self) -> Result<(), StoreError> { - self.engine.clear_snap_state() + pub async fn clear_snap_state(&self) -> Result<(), StoreError> { + self.engine.clear_snap_state().await } /// Set the latest root of the rebuilt state trie and the last downloaded hashes from each segment - pub fn set_state_trie_rebuild_checkpoint( + pub async fn set_state_trie_rebuild_checkpoint( &self, checkpoint: (H256, [H256; STATE_TRIE_SEGMENTS]), ) -> Result<(), StoreError> { - self.engine.set_state_trie_rebuild_checkpoint(checkpoint) + self.engine + .set_state_trie_rebuild_checkpoint(checkpoint) + .await } /// Get the latest root of the rebuilt state trie and the last downloaded hashes from each segment @@ -1013,11 +1041,11 @@ impl Store { } /// Set the accont hashes and roots of the storage tries awaiting rebuild - pub fn set_storage_trie_rebuild_pending( + pub async fn set_storage_trie_rebuild_pending( &self, pending: Vec<(H256, H256)>, ) -> Result<(), StoreError> { - self.engine.set_storage_trie_rebuild_pending(pending) + self.engine.set_storage_trie_rebuild_pending(pending).await } /// Get the accont hashes and roots of the storage tries awaiting rebuild @@ -1028,8 +1056,8 @@ impl Store { } /// Clears the state and storage snapshots - pub fn clear_snapshot(&self) -> Result<(), StoreError> { - self.engine.clear_snapshot() + pub async fn clear_snapshot(&self) -> Result<(), StoreError> { + self.engine.clear_snapshot().await } /// Reads the next `MAX_SNAPSHOT_READS` accounts from the state snapshot as from the `start` hash @@ -1082,25 +1110,29 @@ mod tests { use super::*; - #[test] - fn test_in_memory_store() { - test_store_suite(EngineType::InMemory); + #[tokio::test] + async fn test_in_memory_store() { + test_store_suite(EngineType::InMemory).await; } #[cfg(feature = "libmdbx")] - #[test] - fn test_libmdbx_store() { - test_store_suite(EngineType::Libmdbx); + #[tokio::test] + async fn test_libmdbx_store() { + test_store_suite(EngineType::Libmdbx).await; } #[cfg(feature = "redb")] - #[test] - fn test_redb_store() { - test_store_suite(EngineType::RedB); + #[tokio::test] + async fn test_redb_store() { + test_store_suite(EngineType::RedB).await; } // Creates an empty store, runs the test and then removes the store (if needed) - fn run_test(test_func: &dyn Fn(Store), engine_type: EngineType) { + async fn run_test(test_func: F, engine_type: EngineType) + where + F: FnOnce(Store) -> Fut, + Fut: std::future::Future, + { // Remove preexistent DBs in case of a failed previous test if !matches!(engine_type, EngineType::InMemory) { remove_test_dbs("store-test-db"); @@ -1108,26 +1140,26 @@ mod tests { // Build a new store let store = Store::new("store-test-db", engine_type).expect("Failed to create test db"); // Run the test - test_func(store); + test_func(store).await; // Remove store (if needed) if !matches!(engine_type, EngineType::InMemory) { remove_test_dbs("store-test-db"); }; } - fn test_store_suite(engine_type: EngineType) { - run_test(&test_store_block, engine_type); - run_test(&test_store_block_number, engine_type); - run_test(&test_store_transaction_location, engine_type); - run_test(&test_store_transaction_location_not_canonical, engine_type); - run_test(&test_store_block_receipt, engine_type); - run_test(&test_store_account_code, engine_type); - run_test(&test_store_block_tags, engine_type); - run_test(&test_chain_config_storage, engine_type); - run_test(&test_genesis_block, engine_type); + async fn test_store_suite(engine_type: EngineType) { + run_test(test_store_block, engine_type).await; + run_test(test_store_block_number, engine_type).await; + run_test(test_store_transaction_location, engine_type).await; + run_test(test_store_transaction_location_not_canonical, engine_type).await; + run_test(test_store_block_receipt, engine_type).await; + run_test(test_store_account_code, engine_type).await; + run_test(test_store_block_tags, engine_type).await; + run_test(test_chain_config_storage, engine_type).await; + run_test(test_genesis_block, engine_type).await; } - fn test_genesis_block(store: Store) { + async fn test_genesis_block(store: Store) { const GENESIS_KURTOSIS: &str = include_str!("../../test_data/genesis-kurtosis.json"); const GENESIS_HIVE: &str = include_str!("../../test_data/genesis-hive.json"); assert_ne!(GENESIS_KURTOSIS, GENESIS_HIVE); @@ -1137,12 +1169,15 @@ mod tests { serde_json::from_str(GENESIS_HIVE).expect("deserialize genesis-hive.json"); store .add_initial_state(genesis_kurtosis.clone()) + .await .expect("first genesis"); store .add_initial_state(genesis_kurtosis) + .await .expect("second genesis with same block"); panic::catch_unwind(move || { - let _ = store.add_initial_state(genesis_hive); + let rt = tokio::runtime::Runtime::new().expect("runtime creation failed"); + let _ = rt.block_on(store.add_initial_state(genesis_hive)); }) .expect_err("genesis with a different block should panic"); } @@ -1154,14 +1189,20 @@ mod tests { } } - fn test_store_block(store: Store) { + async fn test_store_block(store: Store) { let (block_header, block_body) = create_block_for_testing(); let block_number = 6; let hash = block_header.compute_block_hash(); - store.add_block_header(hash, block_header.clone()).unwrap(); - store.add_block_body(hash, block_body.clone()).unwrap(); - store.set_canonical_block(block_number, hash).unwrap(); + store + .add_block_header(hash, block_header.clone()) + .await + .unwrap(); + store + .add_block_body(hash, block_body.clone()) + .await + .unwrap(); + store.set_canonical_block(block_number, hash).await.unwrap(); let stored_header = store.get_block_header(block_number).unwrap().unwrap(); let stored_body = store.get_block_body(block_number).unwrap().unwrap(); @@ -1223,18 +1264,21 @@ mod tests { (block_header, block_body) } - fn test_store_block_number(store: Store) { + async fn test_store_block_number(store: Store) { let block_hash = H256::random(); let block_number = 6; - store.add_block_number(block_hash, block_number).unwrap(); + store + .add_block_number(block_hash, block_number) + .await + .unwrap(); let stored_number = store.get_block_number(block_hash).unwrap().unwrap(); assert_eq!(stored_number, block_number); } - fn test_store_transaction_location(store: Store) { + async fn test_store_transaction_location(store: Store) { let transaction_hash = H256::random(); let block_hash = H256::random(); let block_number = 6; @@ -1242,9 +1286,13 @@ mod tests { store .add_transaction_location(transaction_hash, block_number, block_hash, index) + .await .unwrap(); - store.set_canonical_block(block_number, block_hash).unwrap(); + store + .set_canonical_block(block_number, block_hash) + .await + .unwrap(); let stored_location = store .get_transaction_location(transaction_hash) @@ -1254,7 +1302,7 @@ mod tests { assert_eq!(stored_location, (block_number, block_hash, index)); } - fn test_store_transaction_location_not_canonical(store: Store) { + async fn test_store_transaction_location_not_canonical(store: Store) { let transaction_hash = H256::random(); let block_hash = H256::random(); let block_number = 6; @@ -1262,10 +1310,12 @@ mod tests { store .add_transaction_location(transaction_hash, block_number, block_hash, index) + .await .unwrap(); store .set_canonical_block(block_number, H256::random()) + .await .unwrap(); assert_eq!( @@ -1274,7 +1324,7 @@ mod tests { ) } - fn test_store_block_receipt(store: Store) { + async fn test_store_block_receipt(store: Store) { let receipt = Receipt { tx_type: TxType::EIP2930, succeeded: true, @@ -1288,27 +1338,34 @@ mod tests { store .add_receipt(block_hash, index, receipt.clone()) + .await .unwrap(); - store.set_canonical_block(block_number, block_hash).unwrap(); + store + .set_canonical_block(block_number, block_hash) + .await + .unwrap(); let stored_receipt = store.get_receipt(block_number, index).unwrap().unwrap(); assert_eq!(stored_receipt, receipt); } - fn test_store_account_code(store: Store) { + async fn test_store_account_code(store: Store) { let code_hash = H256::random(); let code = Bytes::from("kiwi"); - store.add_account_code(code_hash, code.clone()).unwrap(); + store + .add_account_code(code_hash, code.clone()) + .await + .unwrap(); let stored_code = store.get_account_code(code_hash).unwrap().unwrap(); assert_eq!(stored_code, code); } - fn test_store_block_tags(store: Store) { + async fn test_store_block_tags(store: Store) { let earliest_block_number = 0; let finalized_block_number = 7; let safe_block_number = 6; @@ -1317,16 +1374,23 @@ mod tests { store .update_earliest_block_number(earliest_block_number) + .await .unwrap(); store .update_finalized_block_number(finalized_block_number) + .await + .unwrap(); + store + .update_safe_block_number(safe_block_number) + .await .unwrap(); - store.update_safe_block_number(safe_block_number).unwrap(); store .update_latest_block_number(latest_block_number) + .await .unwrap(); store .update_pending_block_number(pending_block_number) + .await .unwrap(); let stored_earliest_block_number = store.get_earliest_block_number().unwrap(); @@ -1342,9 +1406,9 @@ mod tests { assert_eq!(pending_block_number, stored_pending_block_number); } - fn test_chain_config_storage(store: Store) { + async fn test_chain_config_storage(store: Store) { let chain_config = example_chain_config(); - store.set_chain_config(&chain_config).unwrap(); + store.set_chain_config(&chain_config).await.unwrap(); let retrieved_chain_config = store.get_chain_config().unwrap(); assert_eq!(chain_config, retrieved_chain_config); } diff --git a/crates/storage/store_db/in_memory.rs b/crates/storage/store_db/in_memory.rs index a378ba5809..48e75bded3 100644 --- a/crates/storage/store_db/in_memory.rs +++ b/crates/storage/store_db/in_memory.rs @@ -84,6 +84,7 @@ impl Store { } } +#[async_trait::async_trait] impl StoreEngine for Store { fn get_block_header(&self, block_number: u64) -> Result, StoreError> { let store = self.inner(); @@ -103,7 +104,7 @@ impl StoreEngine for Store { } } - fn add_pending_block(&self, block: Block) -> Result<(), StoreError> { + async fn add_pending_block(&self, block: Block) -> Result<(), StoreError> { self.inner() .pending_blocks .insert(block.header.compute_block_hash(), block); @@ -114,7 +115,7 @@ impl StoreEngine for Store { Ok(self.inner().pending_blocks.get(&block_hash).cloned()) } - fn add_block_header( + async fn add_block_header( &self, block_hash: BlockHash, block_header: BlockHeader, @@ -123,7 +124,7 @@ impl StoreEngine for Store { Ok(()) } - fn add_block_headers( + async fn add_block_headers( &self, block_hashes: Vec, block_headers: Vec, @@ -134,7 +135,7 @@ impl StoreEngine for Store { Ok(()) } - fn add_block_body( + async fn add_block_body( &self, block_hash: BlockHash, block_body: BlockBody, @@ -143,9 +144,9 @@ impl StoreEngine for Store { Ok(()) } - fn add_blocks(&self, blocks: &[Block]) -> Result<(), StoreError> { + async fn add_blocks(&self, blocks: Vec) -> Result<(), StoreError> { for block in blocks { - let header = block.header.clone(); + let header = block.header; let number = header.number; let hash = header.compute_block_hash(); let locations = block @@ -155,16 +156,16 @@ impl StoreEngine for Store { .enumerate() .map(|(i, tx)| (tx.compute_hash(), number, hash, i as u64)); - self.add_transaction_locations(locations.collect())?; - self.add_block_body(hash, block.body.clone())?; - self.add_block_header(hash, header)?; - self.add_block_number(hash, number)?; + self.add_transaction_locations(locations.collect()).await?; + self.add_block_body(hash, block.body.clone()).await?; + self.add_block_header(hash, header).await?; + self.add_block_number(hash, number).await?; } Ok(()) } - fn mark_chain_as_canonical(&self, blocks: &[Block]) -> Result<(), StoreError> { + async fn mark_chain_as_canonical(&self, blocks: &[Block]) -> Result<(), StoreError> { for block in blocks { self.inner() .canonical_hashes @@ -174,7 +175,7 @@ impl StoreEngine for Store { Ok(()) } - fn add_block_number( + async fn add_block_number( &self, block_hash: BlockHash, block_number: BlockNumber, @@ -187,7 +188,7 @@ impl StoreEngine for Store { Ok(self.inner().block_numbers.get(&block_hash).copied()) } - fn add_transaction_location( + async fn add_transaction_location( &self, transaction_hash: H256, block_number: BlockNumber, @@ -217,7 +218,7 @@ impl StoreEngine for Store { })) } - fn add_receipt( + async fn add_receipt( &self, block_hash: BlockHash, index: Index, @@ -246,7 +247,7 @@ impl StoreEngine for Store { } } - fn add_account_code(&self, code_hash: H256, code: Bytes) -> Result<(), StoreError> { + async fn add_account_code(&self, code_hash: H256, code: Bytes) -> Result<(), StoreError> { self.inner().account_codes.insert(code_hash, code); Ok(()) } @@ -255,7 +256,7 @@ impl StoreEngine for Store { Ok(self.inner().account_codes.get(&code_hash).cloned()) } - fn set_chain_config(&self, chain_config: &ChainConfig) -> Result<(), StoreError> { + async fn set_chain_config(&self, chain_config: &ChainConfig) -> Result<(), StoreError> { // Store cancun timestamp self.inner().chain_data.chain_config = Some(*chain_config); Ok(()) @@ -265,7 +266,10 @@ impl StoreEngine for Store { Ok(self.inner().chain_data.chain_config.unwrap()) } - fn update_earliest_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { + async fn update_earliest_block_number( + &self, + block_number: BlockNumber, + ) -> Result<(), StoreError> { self.inner() .chain_data .earliest_block_number @@ -277,7 +281,10 @@ impl StoreEngine for Store { Ok(self.inner().chain_data.earliest_block_number) } - fn update_finalized_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { + async fn update_finalized_block_number( + &self, + block_number: BlockNumber, + ) -> Result<(), StoreError> { self.inner() .chain_data .finalized_block_number @@ -289,7 +296,7 @@ impl StoreEngine for Store { Ok(self.inner().chain_data.finalized_block_number) } - fn update_safe_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { + async fn update_safe_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { self.inner() .chain_data .safe_block_number @@ -301,7 +308,10 @@ impl StoreEngine for Store { Ok(self.inner().chain_data.safe_block_number) } - fn update_latest_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { + async fn update_latest_block_number( + &self, + block_number: BlockNumber, + ) -> Result<(), StoreError> { self.inner() .chain_data .latest_block_number @@ -312,7 +322,10 @@ impl StoreEngine for Store { Ok(self.inner().chain_data.latest_block_number) } - fn update_pending_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { + async fn update_pending_block_number( + &self, + block_number: BlockNumber, + ) -> Result<(), StoreError> { self.inner() .chain_data .pending_block_number @@ -351,7 +364,11 @@ impl StoreEngine for Store { Ok(self.inner().headers.get(&block_hash).cloned()) } - fn set_canonical_block(&self, number: BlockNumber, hash: BlockHash) -> Result<(), StoreError> { + async fn set_canonical_block( + &self, + number: BlockNumber, + hash: BlockHash, + ) -> Result<(), StoreError> { self.inner().canonical_hashes.insert(number, hash); Ok(()) } @@ -363,12 +380,12 @@ impl StoreEngine for Store { Ok(self.inner().canonical_hashes.get(&block_number).cloned()) } - fn unset_canonical_block(&self, number: BlockNumber) -> Result<(), StoreError> { + async fn unset_canonical_block(&self, number: BlockNumber) -> Result<(), StoreError> { self.inner().canonical_hashes.remove(&number); Ok(()) } - fn add_payload(&self, payload_id: u64, block: Block) -> Result<(), StoreError> { + async fn add_payload(&self, payload_id: u64, block: Block) -> Result<(), StoreError> { self.inner() .payloads .insert(payload_id, PayloadBundle::from_block(block)); @@ -396,7 +413,7 @@ impl StoreEngine for Store { .collect()) } - fn add_receipts( + async fn add_receipts( &self, block_hash: BlockHash, receipts: Vec, @@ -409,18 +426,18 @@ impl StoreEngine for Store { Ok(()) } - fn add_receipts_for_blocks( + async fn add_receipts_for_blocks( &self, receipts: HashMap>, ) -> Result<(), StoreError> { for (block_hash, receipts) in receipts.into_iter() { - self.add_receipts(block_hash, receipts)?; + self.add_receipts(block_hash, receipts).await?; } Ok(()) } - fn add_transaction_locations( + async fn add_transaction_locations( &self, locations: Vec<(H256, BlockNumber, BlockHash, Index)>, ) -> Result<(), StoreError> { @@ -435,12 +452,19 @@ impl StoreEngine for Store { Ok(()) } - fn update_payload(&self, payload_id: u64, payload: PayloadBundle) -> Result<(), StoreError> { + async fn update_payload( + &self, + payload_id: u64, + payload: PayloadBundle, + ) -> Result<(), StoreError> { self.inner().payloads.insert(payload_id, payload); Ok(()) } - fn set_header_download_checkpoint(&self, block_hash: BlockHash) -> Result<(), StoreError> { + async fn set_header_download_checkpoint( + &self, + block_hash: BlockHash, + ) -> Result<(), StoreError> { self.inner().snap_state.header_download_checkpoint = Some(block_hash); Ok(()) } @@ -449,7 +473,7 @@ impl StoreEngine for Store { Ok(self.inner().snap_state.header_download_checkpoint) } - fn set_state_trie_key_checkpoint( + async fn set_state_trie_key_checkpoint( &self, last_keys: [H256; STATE_TRIE_SEGMENTS], ) -> Result<(), StoreError> { @@ -463,7 +487,7 @@ impl StoreEngine for Store { Ok(self.inner().snap_state.state_trie_key_checkpoint) } - fn set_storage_heal_paths( + async fn set_storage_heal_paths( &self, accounts: Vec<(H256, Vec)>, ) -> Result<(), StoreError> { @@ -475,7 +499,7 @@ impl StoreEngine for Store { Ok(self.inner().snap_state.storage_heal_paths.clone()) } - fn clear_snap_state(&self) -> Result<(), StoreError> { + async fn clear_snap_state(&self) -> Result<(), StoreError> { self.inner().snap_state = Default::default(); Ok(()) } @@ -484,12 +508,12 @@ impl StoreEngine for Store { Ok(self.inner().chain_data.is_synced) } - fn update_sync_status(&self, status: bool) -> Result<(), StoreError> { + async fn update_sync_status(&self, status: bool) -> Result<(), StoreError> { self.inner().chain_data.is_synced = status; Ok(()) } - fn set_state_heal_paths(&self, paths: Vec) -> Result<(), StoreError> { + async fn set_state_heal_paths(&self, paths: Vec) -> Result<(), StoreError> { self.inner().snap_state.state_heal_paths = Some(paths); Ok(()) } @@ -498,7 +522,7 @@ impl StoreEngine for Store { Ok(self.inner().snap_state.state_heal_paths.clone()) } - fn write_snapshot_account_batch( + async fn write_snapshot_account_batch( &self, account_hashes: Vec, account_states: Vec, @@ -509,7 +533,7 @@ impl StoreEngine for Store { Ok(()) } - fn write_snapshot_storage_batch( + async fn write_snapshot_storage_batch( &self, account_hash: H256, storage_keys: Vec, @@ -522,7 +546,7 @@ impl StoreEngine for Store { .extend(storage_keys.into_iter().zip(storage_values)); Ok(()) } - fn write_snapshot_storage_batches( + async fn write_snapshot_storage_batches( &self, account_hashes: Vec, storage_keys: Vec>, @@ -541,7 +565,7 @@ impl StoreEngine for Store { Ok(()) } - fn set_state_trie_rebuild_checkpoint( + async fn set_state_trie_rebuild_checkpoint( &self, checkpoint: (H256, [H256; STATE_TRIE_SEGMENTS]), ) -> Result<(), StoreError> { @@ -555,7 +579,7 @@ impl StoreEngine for Store { Ok(self.inner().snap_state.state_trie_rebuild_checkpoint) } - fn clear_snapshot(&self) -> Result<(), StoreError> { + async fn clear_snapshot(&self) -> Result<(), StoreError> { self.inner().snap_state.state_trie_rebuild_checkpoint = None; self.inner().snap_state.storage_trie_rebuild_pending = None; Ok(()) @@ -592,7 +616,7 @@ impl StoreEngine for Store { } } - fn set_storage_trie_rebuild_pending( + async fn set_storage_trie_rebuild_pending( &self, pending: Vec<(H256, H256)>, ) -> Result<(), StoreError> { diff --git a/crates/storage/store_db/libmdbx.rs b/crates/storage/store_db/libmdbx.rs index 7358442be9..fe319e5c28 100644 --- a/crates/storage/store_db/libmdbx.rs +++ b/crates/storage/store_db/libmdbx.rs @@ -43,33 +43,37 @@ impl Store { } // Helper method to write into a libmdbx table - fn write(&self, key: T::Key, value: T::Value) -> Result<(), StoreError> { - let txn = self - .db - .begin_readwrite() - .map_err(StoreError::LibmdbxError)?; - txn.upsert::(key, value) - .map_err(StoreError::LibmdbxError)?; - txn.commit().map_err(StoreError::LibmdbxError) + async fn write(&self, key: T::Key, value: T::Value) -> Result<(), StoreError> { + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let txn = db.begin_readwrite().map_err(StoreError::LibmdbxError)?; + txn.upsert::(key, value) + .map_err(StoreError::LibmdbxError)?; + txn.commit().map_err(StoreError::LibmdbxError) + }) + .await + .map_err(|e| StoreError::Custom(format!("task panicked: {e}")))? } // Helper method to write into a libmdbx table in batch - fn write_batch( + async fn write_batch( &self, - key_values: impl Iterator, + key_values: Vec<(T::Key, T::Value)>, ) -> Result<(), StoreError> { - let txn = self - .db - .begin_readwrite() - .map_err(StoreError::LibmdbxError)?; - - let mut cursor = txn.cursor::().map_err(StoreError::LibmdbxError)?; - for (key, value) in key_values { - cursor - .upsert(key, value) - .map_err(StoreError::LibmdbxError)?; - } - txn.commit().map_err(StoreError::LibmdbxError) + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let txn = db.begin_readwrite().map_err(StoreError::LibmdbxError)?; + + let mut cursor = txn.cursor::().map_err(StoreError::LibmdbxError)?; + for (key, value) in key_values { + cursor + .upsert(key, value) + .map_err(StoreError::LibmdbxError)?; + } + txn.commit().map_err(StoreError::LibmdbxError) + }) + .await + .map_err(|e| StoreError::Custom(format!("task panicked: {e}")))? } // Helper method to read from a libmdbx table @@ -86,16 +90,18 @@ impl Store { } } +#[async_trait::async_trait] impl StoreEngine for Store { - fn add_block_header( + async fn add_block_header( &self, block_hash: BlockHash, block_header: BlockHeader, ) -> Result<(), StoreError> { self.write::(block_hash.into(), block_header.into()) + .await } - fn add_block_headers( + async fn add_block_headers( &self, block_hashes: Vec, block_headers: Vec, @@ -103,8 +109,9 @@ impl StoreEngine for Store { let hashes_and_headers = block_hashes .into_iter() .zip(block_headers) - .map(|(hash, header)| (hash.into(), header.into())); - self.write_batch::(hashes_and_headers) + .map(|(hash, header)| (hash.into(), header.into())) + .collect(); + self.write_batch::(hashes_and_headers).await } fn get_block_header( @@ -118,55 +125,61 @@ impl StoreEngine for Store { } } - fn add_block_body( + async fn add_block_body( &self, block_hash: BlockHash, block_body: BlockBody, ) -> Result<(), StoreError> { self.write::(block_hash.into(), block_body.into()) + .await } - fn add_blocks(&self, blocks: &[Block]) -> Result<(), StoreError> { - let tx = self - .db - .begin_readwrite() - .map_err(StoreError::LibmdbxError)?; + async fn add_blocks(&self, blocks: Vec) -> Result<(), StoreError> { + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let tx = db.begin_readwrite().map_err(StoreError::LibmdbxError)?; + + for block in blocks { + let number = block.header.number; + let hash = block.hash(); - for block in blocks { - let number = block.header.number; - let hash = block.hash(); + for (index, transaction) in block.body.transactions.iter().enumerate() { + tx.upsert::( + transaction.compute_hash().into(), + (number, hash, index as u64).into(), + ) + .map_err(StoreError::LibmdbxError)?; + } - for (index, transaction) in block.body.transactions.iter().enumerate() { - tx.upsert::( - transaction.compute_hash().into(), - (number, hash, index as u64).into(), + tx.upsert::( + hash.into(), + BlockBodyRLP::from_bytes(block.body.encode_to_vec()), ) .map_err(StoreError::LibmdbxError)?; - } - - tx.upsert::( - hash.into(), - BlockBodyRLP::from_bytes(block.body.encode_to_vec()), - ) - .map_err(StoreError::LibmdbxError)?; - tx.upsert::( - hash.into(), - BlockHeaderRLP::from_bytes(block.header.encode_to_vec()), - ) - .map_err(StoreError::LibmdbxError)?; - - tx.upsert::(hash.into(), number) + tx.upsert::( + hash.into(), + BlockHeaderRLP::from_bytes(block.header.encode_to_vec()), + ) .map_err(StoreError::LibmdbxError)?; - } - tx.commit().map_err(StoreError::LibmdbxError) + tx.upsert::(hash.into(), number) + .map_err(StoreError::LibmdbxError)?; + } + + tx.commit().map_err(StoreError::LibmdbxError) + }) + .await + .map_err(|e| StoreError::Custom(format!("task panicked: {e}")))? } - fn mark_chain_as_canonical(&self, blocks: &[Block]) -> Result<(), StoreError> { - let key_values = blocks.iter().map(|e| (e.header.number, e.hash().into())); + async fn mark_chain_as_canonical(&self, blocks: &[Block]) -> Result<(), StoreError> { + let key_values = blocks + .iter() + .map(|e| (e.header.number, e.hash().into())) + .collect(); - self.write_batch::(key_values) + self.write_batch::(key_values).await } fn get_block_body(&self, block_number: BlockNumber) -> Result, StoreError> { @@ -191,27 +204,29 @@ impl StoreEngine for Store { Ok(self.read::(block_hash.into())?.map(|b| b.to())) } - fn add_block_number( + async fn add_block_number( &self, block_hash: BlockHash, block_number: BlockNumber, ) -> Result<(), StoreError> { self.write::(block_hash.into(), block_number) + .await } fn get_block_number(&self, block_hash: BlockHash) -> Result, StoreError> { self.read::(block_hash.into()) } - fn add_account_code(&self, code_hash: H256, code: Bytes) -> Result<(), StoreError> { + async fn add_account_code(&self, code_hash: H256, code: Bytes) -> Result<(), StoreError> { self.write::(code_hash.into(), code.into()) + .await } fn get_account_code(&self, code_hash: H256) -> Result, StoreError> { Ok(self.read::(code_hash.into())?.map(|b| b.to())) } - fn add_receipt( + async fn add_receipt( &self, block_hash: BlockHash, index: Index, @@ -221,7 +236,7 @@ impl StoreEngine for Store { let Some(entries) = IndexedChunk::from::(key, &receipt.encode_to_vec()) else { return Err(StoreError::Custom("Invalid size".to_string())); }; - self.write_batch::(entries.into_iter()) + self.write_batch::(entries).await } fn get_receipt( @@ -239,7 +254,7 @@ impl StoreEngine for Store { } } - fn add_transaction_location( + async fn add_transaction_location( &self, transaction_hash: H256, block_number: BlockNumber, @@ -250,6 +265,7 @@ impl StoreEngine for Store { transaction_hash.into(), (block_number, block_hash, index).into(), ) + .await } fn get_transaction_location( @@ -270,13 +286,14 @@ impl StoreEngine for Store { } /// Stores the chain config serialized as json - fn set_chain_config(&self, chain_config: &ChainConfig) -> Result<(), StoreError> { + async fn set_chain_config(&self, chain_config: &ChainConfig) -> Result<(), StoreError> { self.write::( ChainDataIndex::ChainConfig, serde_json::to_string(chain_config) .map_err(|_| StoreError::DecodeError)? .into_bytes(), ) + .await } fn get_chain_config(&self) -> Result { @@ -291,11 +308,15 @@ impl StoreEngine for Store { } } - fn update_earliest_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { + async fn update_earliest_block_number( + &self, + block_number: BlockNumber, + ) -> Result<(), StoreError> { self.write::( ChainDataIndex::EarliestBlockNumber, block_number.encode_to_vec(), ) + .await } fn get_earliest_block_number(&self) -> Result, StoreError> { @@ -307,11 +328,15 @@ impl StoreEngine for Store { } } - fn update_finalized_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { + async fn update_finalized_block_number( + &self, + block_number: BlockNumber, + ) -> Result<(), StoreError> { self.write::( ChainDataIndex::FinalizedBlockNumber, block_number.encode_to_vec(), ) + .await } fn get_finalized_block_number(&self) -> Result, StoreError> { @@ -323,11 +348,12 @@ impl StoreEngine for Store { } } - fn update_safe_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { + async fn update_safe_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { self.write::( ChainDataIndex::SafeBlockNumber, block_number.encode_to_vec(), ) + .await } fn get_safe_block_number(&self) -> Result, StoreError> { @@ -339,11 +365,15 @@ impl StoreEngine for Store { } } - fn update_latest_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { + async fn update_latest_block_number( + &self, + block_number: BlockNumber, + ) -> Result<(), StoreError> { self.write::( ChainDataIndex::LatestBlockNumber, block_number.encode_to_vec(), ) + .await } fn get_latest_block_number(&self) -> Result, StoreError> { @@ -355,11 +385,15 @@ impl StoreEngine for Store { } } - fn update_pending_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { + async fn update_pending_block_number( + &self, + block_number: BlockNumber, + ) -> Result<(), StoreError> { self.write::( ChainDataIndex::PendingBlockNumber, block_number.encode_to_vec(), ) + .await } fn get_pending_block_number(&self) -> Result, StoreError> { @@ -384,8 +418,13 @@ impl StoreEngine for Store { Trie::open(db, state_root) } - fn set_canonical_block(&self, number: BlockNumber, hash: BlockHash) -> Result<(), StoreError> { + async fn set_canonical_block( + &self, + number: BlockNumber, + hash: BlockHash, + ) -> Result<(), StoreError> { self.write::(number, hash.into()) + .await } fn get_canonical_block_hash( @@ -396,8 +435,9 @@ impl StoreEngine for Store { .map(|o| o.map(|hash_rlp| hash_rlp.to())) } - fn add_payload(&self, payload_id: u64, block: Block) -> Result<(), StoreError> { + async fn add_payload(&self, payload_id: u64, block: Block) -> Result<(), StoreError> { self.write::(payload_id, PayloadBundle::from_block(block).into()) + .await } fn get_payload(&self, payload_id: u64) -> Result, StoreError> { @@ -405,8 +445,12 @@ impl StoreEngine for Store { Ok(r.map(|b| b.to())) } - fn update_payload(&self, payload_id: u64, payload: PayloadBundle) -> Result<(), StoreError> { - self.write::(payload_id, payload.into()) + async fn update_payload( + &self, + payload_id: u64, + payload: PayloadBundle, + ) -> Result<(), StoreError> { + self.write::(payload_id, payload.into()).await } fn get_transaction_by_hash( @@ -448,17 +492,22 @@ impl StoreEngine for Store { Ok(Some(Block::new(header, body))) } - fn unset_canonical_block(&self, number: BlockNumber) -> Result<(), StoreError> { - self.db - .begin_readwrite() - .map_err(StoreError::LibmdbxError)? - .delete::(number, None) - .map(|_| ()) - .map_err(StoreError::LibmdbxError) + async fn unset_canonical_block(&self, number: BlockNumber) -> Result<(), StoreError> { + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + db.begin_readwrite() + .map_err(StoreError::LibmdbxError)? + .delete::(number, None) + .map(|_| ()) + .map_err(StoreError::LibmdbxError) + }) + .await + .map_err(|e| StoreError::Custom(format!("task panicked: {e}")))? } - fn add_pending_block(&self, block: Block) -> Result<(), StoreError> { + async fn add_pending_block(&self, block: Block) -> Result<(), StoreError> { self.write::(block.header.compute_block_hash().into(), block.into()) + .await } fn get_pending_block(&self, block_hash: BlockHash) -> Result, StoreError> { @@ -467,7 +516,7 @@ impl StoreEngine for Store { .map(|b| b.to())) } - fn add_transaction_locations( + async fn add_transaction_locations( &self, locations: Vec<(H256, BlockNumber, BlockHash, Index)>, ) -> Result<(), StoreError> { @@ -476,12 +525,13 @@ impl StoreEngine for Store { .into_iter() .map(|(tx_hash, block_number, block_hash, index)| { (tx_hash.into(), (block_number, block_hash, index).into()) - }); + }) + .collect(); - self.write_batch::(key_values) + self.write_batch::(key_values).await } - fn add_receipts( + async fn add_receipts( &self, block_hash: BlockHash, receipts: Vec, @@ -498,10 +548,10 @@ impl StoreEngine for Store { key_values.append(&mut entries); } - self.write_batch::(key_values.into_iter()) + self.write_batch::(key_values).await } - fn add_receipts_for_blocks( + async fn add_receipts_for_blocks( &self, receipts: HashMap>, ) -> Result<(), StoreError> { @@ -519,7 +569,7 @@ impl StoreEngine for Store { } } - self.write_batch::(key_values.into_iter()) + self.write_batch::(key_values).await } fn get_receipts_for_block(&self, block_hash: &BlockHash) -> Result, StoreError> { @@ -545,11 +595,15 @@ impl StoreEngine for Store { Ok(receipts) } - fn set_header_download_checkpoint(&self, block_hash: BlockHash) -> Result<(), StoreError> { + async fn set_header_download_checkpoint( + &self, + block_hash: BlockHash, + ) -> Result<(), StoreError> { self.write::( SnapStateIndex::HeaderDownloadCheckpoint, block_hash.encode_to_vec(), ) + .await } fn get_header_download_checkpoint(&self) -> Result, StoreError> { @@ -559,7 +613,7 @@ impl StoreEngine for Store { .map_err(StoreError::RLPDecode) } - fn set_state_trie_key_checkpoint( + async fn set_state_trie_key_checkpoint( &self, last_keys: [H256; STATE_TRIE_SEGMENTS], ) -> Result<(), StoreError> { @@ -567,6 +621,7 @@ impl StoreEngine for Store { SnapStateIndex::StateTrieKeyCheckpoint, last_keys.to_vec().encode_to_vec(), ) + .await } fn get_state_trie_key_checkpoint( @@ -582,11 +637,12 @@ impl StoreEngine for Store { .map_err(StoreError::RLPDecode) } - fn set_storage_heal_paths( + async fn set_storage_heal_paths( &self, accounts: Vec<(H256, Vec)>, ) -> Result<(), StoreError> { self.write::(SnapStateIndex::StorageHealPaths, accounts.encode_to_vec()) + .await } fn get_storage_heal_paths(&self) -> Result)>>, StoreError> { @@ -603,12 +659,14 @@ impl StoreEngine for Store { } } - fn update_sync_status(&self, status: bool) -> Result<(), StoreError> { + async fn update_sync_status(&self, status: bool) -> Result<(), StoreError> { self.write::(ChainDataIndex::IsSynced, status.encode_to_vec()) + .await } - fn set_state_heal_paths(&self, paths: Vec) -> Result<(), StoreError> { + async fn set_state_heal_paths(&self, paths: Vec) -> Result<(), StoreError> { self.write::(SnapStateIndex::StateHealPaths, paths.encode_to_vec()) + .await } fn get_state_heal_paths(&self) -> Result>, StoreError> { @@ -618,17 +676,19 @@ impl StoreEngine for Store { .map_err(StoreError::RLPDecode) } - fn clear_snap_state(&self) -> Result<(), StoreError> { - let txn = self - .db - .begin_readwrite() - .map_err(StoreError::LibmdbxError)?; - txn.clear_table::() - .map_err(StoreError::LibmdbxError)?; - txn.commit().map_err(StoreError::LibmdbxError) + async fn clear_snap_state(&self) -> Result<(), StoreError> { + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let txn = db.begin_readwrite().map_err(StoreError::LibmdbxError)?; + txn.clear_table::() + .map_err(StoreError::LibmdbxError)?; + txn.commit().map_err(StoreError::LibmdbxError) + }) + .await + .map_err(|e| StoreError::Custom(format!("task panicked: {e}")))? } - fn write_snapshot_account_batch( + async fn write_snapshot_account_batch( &self, account_hashes: Vec, account_states: Vec, @@ -637,52 +697,58 @@ impl StoreEngine for Store { account_hashes .into_iter() .map(|h| h.into()) - .zip(account_states.into_iter().map(|a| a.into())), + .zip(account_states.into_iter().map(|a| a.into())) + .collect(), ) + .await } - fn write_snapshot_storage_batch( + async fn write_snapshot_storage_batch( &self, account_hash: H256, storage_keys: Vec, storage_values: Vec, ) -> Result<(), StoreError> { - let txn = self - .db - .begin_readwrite() - .map_err(StoreError::LibmdbxError)?; + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let txn = db.begin_readwrite().map_err(StoreError::LibmdbxError)?; - for (key, value) in storage_keys.into_iter().zip(storage_values.into_iter()) { - txn.upsert::(account_hash.into(), (key.into(), value.into())) - .map_err(StoreError::LibmdbxError)?; - } + for (key, value) in storage_keys.into_iter().zip(storage_values.into_iter()) { + txn.upsert::(account_hash.into(), (key.into(), value.into())) + .map_err(StoreError::LibmdbxError)?; + } - txn.commit().map_err(StoreError::LibmdbxError) + txn.commit().map_err(StoreError::LibmdbxError) + }) + .await + .map_err(|e| StoreError::Custom(format!("task panicked: {e}")))? } - fn write_snapshot_storage_batches( + async fn write_snapshot_storage_batches( &self, account_hashes: Vec, storage_keys: Vec>, storage_values: Vec>, ) -> Result<(), StoreError> { - let txn = self - .db - .begin_readwrite() - .map_err(StoreError::LibmdbxError)?; - for (account_hash, (storage_keys, storage_values)) in account_hashes - .into_iter() - .zip(storage_keys.into_iter().zip(storage_values.into_iter())) - { - for (key, value) in storage_keys.into_iter().zip(storage_values.into_iter()) { - txn.upsert::(account_hash.into(), (key.into(), value.into())) - .map_err(StoreError::LibmdbxError)?; + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let txn = db.begin_readwrite().map_err(StoreError::LibmdbxError)?; + for (account_hash, (storage_keys, storage_values)) in account_hashes + .into_iter() + .zip(storage_keys.into_iter().zip(storage_values.into_iter())) + { + for (key, value) in storage_keys.into_iter().zip(storage_values.into_iter()) { + txn.upsert::(account_hash.into(), (key.into(), value.into())) + .map_err(StoreError::LibmdbxError)?; + } } - } - txn.commit().map_err(StoreError::LibmdbxError) + txn.commit().map_err(StoreError::LibmdbxError) + }) + .await + .map_err(|e| StoreError::Custom(format!("task panicked: {e}")))? } - fn set_state_trie_rebuild_checkpoint( + async fn set_state_trie_rebuild_checkpoint( &self, checkpoint: (H256, [H256; STATE_TRIE_SEGMENTS]), ) -> Result<(), StoreError> { @@ -690,6 +756,7 @@ impl StoreEngine for Store { SnapStateIndex::StateTrieRebuildCheckpoint, (checkpoint.0, checkpoint.1.to_vec()).encode_to_vec(), ) + .await } fn get_state_trie_rebuild_checkpoint( @@ -710,7 +777,7 @@ impl StoreEngine for Store { ))) } - fn set_storage_trie_rebuild_pending( + async fn set_storage_trie_rebuild_pending( &self, pending: Vec<(H256, H256)>, ) -> Result<(), StoreError> { @@ -718,6 +785,7 @@ impl StoreEngine for Store { SnapStateIndex::StorageTrieRebuildPending, pending.encode_to_vec(), ) + .await } fn get_storage_trie_rebuild_pending(&self) -> Result>, StoreError> { @@ -727,17 +795,19 @@ impl StoreEngine for Store { .map_err(StoreError::RLPDecode) } - fn clear_snapshot(&self) -> Result<(), StoreError> { - let txn = self - .db - .begin_readwrite() - .map_err(StoreError::LibmdbxError)?; - txn.clear_table::() - .map_err(StoreError::LibmdbxError)?; - txn.clear_table::() - .map_err(StoreError::LibmdbxError)?; - txn.commit().map_err(StoreError::LibmdbxError)?; - Ok(()) + async fn clear_snapshot(&self) -> Result<(), StoreError> { + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let txn = db.begin_readwrite().map_err(StoreError::LibmdbxError)?; + txn.clear_table::() + .map_err(StoreError::LibmdbxError)?; + txn.clear_table::() + .map_err(StoreError::LibmdbxError)?; + txn.commit().map_err(StoreError::LibmdbxError)?; + Ok(()) + }) + .await + .map_err(|e| StoreError::Custom(format!("task panicked: {e}")))? } fn read_account_snapshot(&self, start: H256) -> Result, StoreError> { diff --git a/crates/storage/store_db/redb.rs b/crates/storage/store_db/redb.rs index dfa7f4d999..693025263f 100644 --- a/crates/storage/store_db/redb.rs +++ b/crates/storage/store_db/redb.rs @@ -74,83 +74,126 @@ impl RedBStore { } // Helper method to write into a redb table - fn write<'k, 'v, 'a, K, V>( + async fn write<'k, 'v, 'a, K, V>( &self, table: TableDefinition<'a, K, V>, - key: impl Borrow>, - value: impl Borrow>, + key: K::SelfType<'k>, + value: V::SelfType<'v>, ) -> Result<(), StoreError> where - K: Key + 'static, - V: Value + 'static, + K: Key + Send + 'static, + V: Value + Send + 'static, + K::SelfType<'k>: Send, + V::SelfType<'v>: Send, + 'a: 'static, + 'k: 'static, + 'v: 'static, { - let write_txn = self.db.begin_write()?; - write_txn.open_table(table)?.insert(key, value)?; - write_txn.commit()?; + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let write_txn = db.begin_write()?; + write_txn.open_table(table)?.insert(key, value)?; + write_txn.commit()?; - Ok(()) + Ok(()) + }) + .await + .map_err(|e| StoreError::Custom(format!("task panicked: {e}")))? } // Helper method to write into a redb table - fn write_to_multi<'k, 'v, 'a, K, V>( + async fn write_to_multi<'k, 'v, 'a, K, V>( &self, table: MultimapTableDefinition<'a, K, V>, - key: impl Borrow>, - value: impl Borrow>, + key: K::SelfType<'k>, + value: V::SelfType<'v>, ) -> Result<(), StoreError> where K: Key + 'static, V: Key + 'static, + K::SelfType<'k>: Send, + V::SelfType<'v>: Send, + MultimapTableDefinition<'a, K, V>: Send, + 'a: 'static, + 'k: 'static, + 'v: 'static, { - let write_txn = self.db.begin_write()?; - write_txn.open_multimap_table(table)?.insert(key, value)?; - write_txn.commit()?; + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let write_txn = db.begin_write()?; + write_txn.open_multimap_table(table)?.insert(key, value)?; + write_txn.commit()?; - Ok(()) + Ok(()) + }) + .await + .map_err(|e| StoreError::Custom(format!("task panicked: {e}")))? } // Helper method to write into a redb table - fn write_batch<'k, 'v, 'a, K, V>( + async fn write_batch<'k, 'v, 'a, K, V>( &self, table: TableDefinition<'a, K, V>, - key_values: Vec<(impl Borrow>, impl Borrow>)>, + key_values: Vec<(K::SelfType<'k>, V::SelfType<'v>)>, ) -> Result<(), StoreError> where - K: Key + 'static, - V: Value + 'static, + K: Key + Send + 'static, + V: Value + Send + 'static, + K::SelfType<'k>: Send, + V::SelfType<'v>: Send, + TableDefinition<'a, K, V>: Send, + 'a: 'static, + 'k: 'static, + 'v: 'static, { - let write_txn = self.db.begin_write()?; - { - let mut table = write_txn.open_table(table)?; - for (key, value) in key_values { - table.insert(key, value)?; + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let write_txn = db.begin_write()?; + { + let mut table = write_txn.open_table(table)?; + for (key, value) in key_values { + table.insert(key, value)?; + } } - } - write_txn.commit()?; + write_txn.commit()?; - Ok(()) + Ok(()) + }) + .await + .map_err(|e| StoreError::Custom(format!("task panicked: {e}")))? } // Helper method to write into a redb table - fn write_to_multi_batch<'k, 'v, 'a, K, V>( + async fn write_to_multi_batch<'k, 'v, 'a, K, V>( &self, table: MultimapTableDefinition<'a, K, V>, - key_values: Vec<(impl Borrow>, impl Borrow>)>, + key_values: Vec<(K::SelfType<'k>, V::SelfType<'v>)>, ) -> Result<(), StoreError> where - K: Key + 'static, - V: Key + 'static, + K: Key + Send + 'static, + V: Key + Send + 'static, + K::SelfType<'k>: Send, + V::SelfType<'v>: Send, + MultimapTableDefinition<'a, K, V>: Send, + 'a: 'static, + 'k: 'static, + 'v: 'static, { - let write_txn = self.db.begin_write()?; - { - let mut table = write_txn.open_multimap_table(table)?; - for (key, value) in key_values { - table.insert(key, value)?; + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let write_txn = db.begin_write()?; + { + let mut table = write_txn.open_multimap_table(table)?; + for (key, value) in key_values { + table.insert(key, value)?; + } } - } - write_txn.commit()?; + write_txn.commit()?; - Ok(()) + Ok(()) + }) + .await + .map_err(|e| StoreError::Custom(format!("task panicked: {e}")))? } // Helper method to read from a redb table @@ -197,8 +240,9 @@ impl RedBStore { } } +#[async_trait::async_trait] impl StoreEngine for RedBStore { - fn add_block_header( + async fn add_block_header( &self, block_hash: BlockHash, block_header: BlockHeader, @@ -208,9 +252,10 @@ impl StoreEngine for RedBStore { >::into(block_hash), >::into(block_header), ) + .await } - fn add_block_headers( + async fn add_block_headers( &self, block_hashes: Vec, block_headers: Vec, @@ -225,7 +270,7 @@ impl StoreEngine for RedBStore { ) }) .collect(); - self.write_batch(HEADERS_TABLE, key_values) + self.write_batch(HEADERS_TABLE, key_values).await } fn get_block_header( @@ -241,7 +286,7 @@ impl StoreEngine for RedBStore { } } - fn add_block_body( + async fn add_block_body( &self, block_hash: BlockHash, block_body: BlockBody, @@ -251,54 +296,60 @@ impl StoreEngine for RedBStore { >::into(block_hash), >::into(block_body), ) + .await } - fn add_blocks(&self, blocks: &[Block]) -> Result<(), StoreError> { - let write_txn = self.db.begin_write()?; + async fn add_blocks(&self, blocks: Vec) -> Result<(), StoreError> { + let db = self.db.clone(); + tokio::task::spawn_blocking(move || { + let write_txn = db.begin_write()?; - { - // Begin block so that tables are opened once and dropped at the end. - // This prevents ownership errors when to committing changes at the end. { - let mut transaction_table = - write_txn.open_multimap_table(TRANSACTION_LOCATIONS_TABLE)?; - let mut headers_table = write_txn.open_table(HEADERS_TABLE)?; - let mut block_bodies_table = write_txn.open_table(BLOCK_BODIES_TABLE)?; - let mut block_numbers_table = write_txn.open_table(BLOCK_NUMBERS_TABLE)?; - - for block in blocks { - let block_number = block.header.number; - let block_hash = block.hash(); - - for (index, transaction) in block.body.transactions.iter().enumerate() { - transaction_table.insert( - >::into(transaction.compute_hash()), - <(u64, H256, u64) as Into>>::into( - (block_number, block_hash, index as u64), - ), + // Begin block so that tables are opened once and dropped at the end. + // This prevents ownership errors when to committing changes at the end. + { + let mut transaction_table = + write_txn.open_multimap_table(TRANSACTION_LOCATIONS_TABLE)?; + let mut headers_table = write_txn.open_table(HEADERS_TABLE)?; + let mut block_bodies_table = write_txn.open_table(BLOCK_BODIES_TABLE)?; + let mut block_numbers_table = write_txn.open_table(BLOCK_NUMBERS_TABLE)?; + + for block in blocks { + let block_number = block.header.number; + let block_hash = block.hash(); + + for (index, transaction) in block.body.transactions.iter().enumerate() { + transaction_table.insert( + >::into(transaction.compute_hash()), + <(u64, H256, u64) as Into>>::into( + (block_number, block_hash, index as u64), + ), + )?; + } + + headers_table.insert( + >::into(block_hash), + >::into(block.header.clone()), )?; + block_bodies_table.insert( + >::into(block_hash), + >::into(block.body.clone()), + )?; + block_numbers_table + .insert(>::into(block_hash), block_number)?; } - - headers_table.insert( - >::into(block_hash), - >::into(block.header.clone()), - )?; - block_bodies_table.insert( - >::into(block_hash), - >::into(block.body.clone()), - )?; - block_numbers_table - .insert(>::into(block_hash), block_number)?; } - } - write_txn.commit()?; + write_txn.commit()?; - Ok(()) - } + Ok(()) + } + }) + .await + .map_err(|e| StoreError::Custom(format!("task panicked: {e}")))? } - fn mark_chain_as_canonical(&self, blocks: &[Block]) -> Result<(), StoreError> { + async fn mark_chain_as_canonical(&self, blocks: &[Block]) -> Result<(), StoreError> { let key_values = blocks .iter() .map(|e| { @@ -310,6 +361,7 @@ impl StoreEngine for RedBStore { .collect(); self.write_batch(CANONICAL_BLOCK_HASHES_TABLE, key_values) + .await } fn get_block_body(&self, block_number: BlockNumber) -> Result, StoreError> { @@ -344,12 +396,13 @@ impl StoreEngine for RedBStore { .map(|b| b.value().to())) } - fn add_pending_block(&self, block: Block) -> Result<(), StoreError> { + async fn add_pending_block(&self, block: Block) -> Result<(), StoreError> { self.write( PENDING_BLOCKS_TABLE, >::into(block.header.compute_block_hash()), >::into(block), ) + .await } fn get_pending_block(&self, block_hash: BlockHash) -> Result, StoreError> { @@ -361,7 +414,7 @@ impl StoreEngine for RedBStore { .map(|b| b.value().to())) } - fn add_block_number( + async fn add_block_number( &self, block_hash: BlockHash, block_number: BlockNumber, @@ -371,6 +424,7 @@ impl StoreEngine for RedBStore { >::into(block_hash), block_number, ) + .await } fn get_block_number(&self, block_hash: BlockHash) -> Result, StoreError> { @@ -382,7 +436,7 @@ impl StoreEngine for RedBStore { .map(|b| b.value())) } - fn add_transaction_location( + async fn add_transaction_location( &self, transaction_hash: ethrex_common::H256, block_number: BlockNumber, @@ -398,6 +452,7 @@ impl StoreEngine for RedBStore { index, )), ) + .await } fn get_transaction_location( @@ -416,7 +471,7 @@ impl StoreEngine for RedBStore { })) } - fn add_receipt( + async fn add_receipt( &self, block_hash: BlockHash, index: Index, @@ -427,9 +482,10 @@ impl StoreEngine for RedBStore { <(H256, u64) as Into>>::into((block_hash, index)), >::into(receipt), ) + .await } - fn add_receipts_for_blocks( + async fn add_receipts_for_blocks( &self, receipts: HashMap>, ) -> Result<(), StoreError> { @@ -453,7 +509,7 @@ impl StoreEngine for RedBStore { key_values.append(&mut kv); } - self.write_batch(RECEIPTS_TABLE, key_values) + self.write_batch(RECEIPTS_TABLE, key_values).await } fn get_receipt( @@ -473,7 +529,7 @@ impl StoreEngine for RedBStore { } } - fn add_account_code( + async fn add_account_code( &self, code_hash: ethrex_common::H256, code: bytes::Bytes, @@ -483,6 +539,7 @@ impl StoreEngine for RedBStore { >::into(code_hash), >::into(code), ) + .await } fn get_account_code( @@ -505,7 +562,7 @@ impl StoreEngine for RedBStore { .map(|o| o.map(|hash_rlp| hash_rlp.value().to())) } - fn set_chain_config(&self, chain_config: &ChainConfig) -> Result<(), StoreError> { + async fn set_chain_config(&self, chain_config: &ChainConfig) -> Result<(), StoreError> { self.write( CHAIN_DATA_TABLE, ChainDataIndex::ChainConfig, @@ -513,6 +570,7 @@ impl StoreEngine for RedBStore { .map_err(|_| StoreError::DecodeError)? .into_bytes(), ) + .await } fn get_chain_config(&self) -> Result { @@ -527,12 +585,16 @@ impl StoreEngine for RedBStore { } } - fn update_earliest_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { + async fn update_earliest_block_number( + &self, + block_number: BlockNumber, + ) -> Result<(), StoreError> { self.write( CHAIN_DATA_TABLE, ChainDataIndex::EarliestBlockNumber, block_number.encode_to_vec(), ) + .await } fn get_earliest_block_number(&self) -> Result, StoreError> { @@ -544,12 +606,16 @@ impl StoreEngine for RedBStore { } } - fn update_finalized_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { + async fn update_finalized_block_number( + &self, + block_number: BlockNumber, + ) -> Result<(), StoreError> { self.write( CHAIN_DATA_TABLE, ChainDataIndex::FinalizedBlockNumber, block_number.encode_to_vec(), ) + .await } fn get_finalized_block_number(&self) -> Result, StoreError> { @@ -561,12 +627,13 @@ impl StoreEngine for RedBStore { } } - fn update_safe_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { + async fn update_safe_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { self.write( CHAIN_DATA_TABLE, ChainDataIndex::SafeBlockNumber, block_number.encode_to_vec(), ) + .await } fn get_safe_block_number(&self) -> Result, StoreError> { @@ -578,12 +645,16 @@ impl StoreEngine for RedBStore { } } - fn update_latest_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { + async fn update_latest_block_number( + &self, + block_number: BlockNumber, + ) -> Result<(), StoreError> { self.write( CHAIN_DATA_TABLE, ChainDataIndex::LatestBlockNumber, block_number.encode_to_vec(), ) + .await } fn get_latest_block_number(&self) -> Result, StoreError> { @@ -595,12 +666,16 @@ impl StoreEngine for RedBStore { } } - fn update_pending_block_number(&self, block_number: BlockNumber) -> Result<(), StoreError> { + async fn update_pending_block_number( + &self, + block_number: BlockNumber, + ) -> Result<(), StoreError> { self.write( CHAIN_DATA_TABLE, ChainDataIndex::PendingBlockNumber, block_number.encode_to_vec(), ) + .await } fn get_pending_block_number(&self) -> Result, StoreError> { @@ -626,24 +701,30 @@ impl StoreEngine for RedBStore { Trie::open(db, state_root) } - fn set_canonical_block(&self, number: BlockNumber, hash: BlockHash) -> Result<(), StoreError> { + async fn set_canonical_block( + &self, + number: BlockNumber, + hash: BlockHash, + ) -> Result<(), StoreError> { self.write( CANONICAL_BLOCK_HASHES_TABLE, number, >::into(hash), ) + .await } - fn unset_canonical_block(&self, number: BlockNumber) -> Result<(), StoreError> { + async fn unset_canonical_block(&self, number: BlockNumber) -> Result<(), StoreError> { self.delete(CANONICAL_BLOCK_HASHES_TABLE, number) } - fn add_payload(&self, payload_id: u64, block: Block) -> Result<(), StoreError> { + async fn add_payload(&self, payload_id: u64, block: Block) -> Result<(), StoreError> { self.write( PAYLOADS_TABLE, payload_id, >::into(PayloadBundle::from_block(block)), ) + .await } fn get_payload(&self, payload_id: u64) -> Result, StoreError> { @@ -652,7 +733,7 @@ impl StoreEngine for RedBStore { .map(|b| b.value().to())) } - fn add_receipts( + async fn add_receipts( &self, block_hash: BlockHash, receipts: Vec, @@ -670,10 +751,10 @@ impl StoreEngine for RedBStore { ) }) .collect(); - self.write_batch(RECEIPTS_TABLE, key_values) + self.write_batch(RECEIPTS_TABLE, key_values).await } - fn add_transaction_locations( + async fn add_transaction_locations( &self, locations: Vec<(H256, BlockNumber, BlockHash, Index)>, ) -> Result<(), StoreError> { @@ -691,17 +772,23 @@ impl StoreEngine for RedBStore { }) .collect(); - self.write_to_multi_batch(TRANSACTION_LOCATIONS_TABLE, key_values)?; + self.write_to_multi_batch(TRANSACTION_LOCATIONS_TABLE, key_values) + .await?; Ok(()) } - fn update_payload(&self, payload_id: u64, payload: PayloadBundle) -> Result<(), StoreError> { + async fn update_payload( + &self, + payload_id: u64, + payload: PayloadBundle, + ) -> Result<(), StoreError> { self.write( PAYLOADS_TABLE, payload_id, >::into(payload), ) + .await } fn get_receipts_for_block( @@ -733,12 +820,16 @@ impl StoreEngine for RedBStore { .collect()) } - fn set_header_download_checkpoint(&self, block_hash: BlockHash) -> Result<(), StoreError> { + async fn set_header_download_checkpoint( + &self, + block_hash: BlockHash, + ) -> Result<(), StoreError> { self.write( SNAP_STATE_TABLE, SnapStateIndex::HeaderDownloadCheckpoint, block_hash.encode_to_vec(), ) + .await } fn get_header_download_checkpoint(&self) -> Result, StoreError> { @@ -748,12 +839,13 @@ impl StoreEngine for RedBStore { .map_err(StoreError::RLPDecode) } - fn set_state_trie_key_checkpoint(&self, last_key: [H256; 2]) -> Result<(), StoreError> { + async fn set_state_trie_key_checkpoint(&self, last_key: [H256; 2]) -> Result<(), StoreError> { self.write( SNAP_STATE_TABLE, SnapStateIndex::StateTrieKeyCheckpoint, last_key.to_vec().encode_to_vec(), ) + .await } fn get_state_trie_key_checkpoint(&self) -> Result, StoreError> { @@ -767,7 +859,7 @@ impl StoreEngine for RedBStore { .map_err(StoreError::RLPDecode) } - fn set_storage_heal_paths( + async fn set_storage_heal_paths( &self, accounts: Vec<(H256, Vec)>, ) -> Result<(), StoreError> { @@ -776,6 +868,7 @@ impl StoreEngine for RedBStore { SnapStateIndex::StorageHealPaths, accounts.encode_to_vec(), ) + .await } fn get_storage_heal_paths(&self) -> Result)>>, StoreError> { @@ -792,20 +885,22 @@ impl StoreEngine for RedBStore { } } - fn update_sync_status(&self, status: bool) -> Result<(), StoreError> { + async fn update_sync_status(&self, status: bool) -> Result<(), StoreError> { self.write( CHAIN_DATA_TABLE, ChainDataIndex::IsSynced, status.encode_to_vec(), ) + .await } - fn set_state_heal_paths(&self, paths: Vec) -> Result<(), StoreError> { + async fn set_state_heal_paths(&self, paths: Vec) -> Result<(), StoreError> { self.write( SNAP_STATE_TABLE, SnapStateIndex::StateHealPaths, paths.encode_to_vec(), ) + .await } fn get_state_heal_paths(&self) -> Result>, StoreError> { @@ -815,7 +910,7 @@ impl StoreEngine for RedBStore { .map_err(StoreError::RLPDecode) } - fn clear_snap_state(&self) -> Result<(), StoreError> { + async fn clear_snap_state(&self) -> Result<(), StoreError> { let write_txn = self.db.begin_write()?; // Delete the whole table as it will be re-crated when we next open it write_txn.delete_table(SNAP_STATE_TABLE)?; @@ -823,7 +918,7 @@ impl StoreEngine for RedBStore { Ok(()) } - fn write_snapshot_account_batch( + async fn write_snapshot_account_batch( &self, account_hashes: Vec, account_states: Vec, @@ -840,9 +935,10 @@ impl StoreEngine for RedBStore { ) .collect::>(), ) + .await } - fn write_snapshot_storage_batch( + async fn write_snapshot_storage_batch( &self, account_hash: H256, storage_keys: Vec, @@ -861,7 +957,7 @@ impl StoreEngine for RedBStore { write_tx.commit()?; Ok(()) } - fn write_snapshot_storage_batches( + async fn write_snapshot_storage_batches( &self, account_hashes: Vec, storage_keys: Vec>, @@ -886,7 +982,7 @@ impl StoreEngine for RedBStore { Ok(()) } - fn set_state_trie_rebuild_checkpoint( + async fn set_state_trie_rebuild_checkpoint( &self, checkpoint: (H256, [H256; crate::STATE_TRIE_SEGMENTS]), ) -> Result<(), StoreError> { @@ -895,6 +991,7 @@ impl StoreEngine for RedBStore { SnapStateIndex::StateTrieRebuildCheckpoint, (checkpoint.0, checkpoint.1.to_vec()).encode_to_vec(), ) + .await } fn get_state_trie_rebuild_checkpoint( @@ -915,7 +1012,7 @@ impl StoreEngine for RedBStore { ))) } - fn set_storage_trie_rebuild_pending( + async fn set_storage_trie_rebuild_pending( &self, pending: Vec<(H256, H256)>, ) -> Result<(), StoreError> { @@ -924,6 +1021,7 @@ impl StoreEngine for RedBStore { SnapStateIndex::StorageTrieRebuildPending, pending.encode_to_vec(), ) + .await } fn get_storage_trie_rebuild_pending(&self) -> Result>, StoreError> { @@ -933,7 +1031,7 @@ impl StoreEngine for RedBStore { .map_err(StoreError::RLPDecode) } - fn clear_snapshot(&self) -> Result<(), StoreError> { + async fn clear_snapshot(&self) -> Result<(), StoreError> { let write_tx = self.db.begin_write()?; write_tx.delete_table(STATE_SNAPSHOT_TABLE)?; write_tx.delete_multimap_table(STORAGE_SNAPSHOT_TABLE)?; diff --git a/crates/vm/levm/src/db/mod.rs b/crates/vm/levm/src/db/mod.rs index 1895db5860..5454450a92 100644 --- a/crates/vm/levm/src/db/mod.rs +++ b/crates/vm/levm/src/db/mod.rs @@ -10,7 +10,7 @@ pub mod cache; pub use cache::CacheDB; pub mod error; -pub trait Database { +pub trait Database: Send + Sync { fn get_account_info(&self, address: Address) -> Result; fn get_storage_slot(&self, address: Address, key: H256) -> Result; fn get_block_hash(&self, block_number: u64) -> Result, DatabaseError>; diff --git a/crates/vm/levm/src/vm.rs b/crates/vm/levm/src/vm.rs index d46bc2c7b1..bd7b3957c4 100644 --- a/crates/vm/levm/src/vm.rs +++ b/crates/vm/levm/src/vm.rs @@ -183,7 +183,7 @@ pub struct GeneralizedDatabase { } impl GeneralizedDatabase { - pub fn new(store: Arc, cache: CacheDB) -> Self { + pub fn new(store: Arc, cache: CacheDB) -> Self { Self { store, cache } } } From cc812b9bfcc75350495a377880b7bc7c4cc49ed1 Mon Sep 17 00:00:00 2001 From: Mario Rugiero Date: Thu, 3 Apr 2025 17:54:53 -0300 Subject: [PATCH 2/2] remove redundante bounds --- crates/vm/levm/src/vm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/vm/levm/src/vm.rs b/crates/vm/levm/src/vm.rs index bd7b3957c4..d46bc2c7b1 100644 --- a/crates/vm/levm/src/vm.rs +++ b/crates/vm/levm/src/vm.rs @@ -183,7 +183,7 @@ pub struct GeneralizedDatabase { } impl GeneralizedDatabase { - pub fn new(store: Arc, cache: CacheDB) -> Self { + pub fn new(store: Arc, cache: CacheDB) -> Self { Self { store, cache } } }