From 9bb86d86689a8b972dbb2bf9ef6410f9a61082d8 Mon Sep 17 00:00:00 2001 From: f3kilo Date: Fri, 24 Jan 2025 12:22:09 +0300 Subject: [PATCH 1/3] engine v2 --- bin/reth/Cargo.toml | 1 + bin/reth/src/commands/bitfinity_import.rs | 11 +- bin/reth/src/main.rs | 33 +++-- bin/reth/tests/commands/bitfinity_node_it.rs | 125 ++++++++++++++++++- bin/reth/tests/commands/utils.rs | 25 +--- 5 files changed, 148 insertions(+), 47 deletions(-) diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index ca31910c019..63ca9f2e0ce 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -120,6 +120,7 @@ revm-primitives.workspace = true reth-db-common.workspace = true reth-db = { workspace = true, features = ["mdbx", "test-utils"] } serial_test.workspace = true +reth-node-ethereum = { workspace = true, features = ["test-utils"] } [features] default = ["jemalloc"] diff --git a/bin/reth/src/commands/bitfinity_import.rs b/bin/reth/src/commands/bitfinity_import.rs index f6a6a714843..3a14285a13d 100644 --- a/bin/reth/src/commands/bitfinity_import.rs +++ b/bin/reth/src/commands/bitfinity_import.rs @@ -21,14 +21,16 @@ use reth_node_core::{args::BitfinityImportArgs, dirs::ChainPath}; use reth_node_ethereum::{EthExecutorProvider, EthereumNode}; use reth_node_events::node::NodeEvent; use reth_primitives::{EthPrimitives, SealedHeader}; -use reth_provider::providers::BlockchainProvider; +use reth_provider::providers::BlockchainProvider2; use reth_provider::{ BlockNumReader, CanonChainTracker, ChainSpecProvider, DatabaseProviderFactory, HeaderProvider, ProviderError, ProviderFactory, }; use reth_prune::PruneModes; use reth_stages::{ - prelude::*, stages::{ExecutionStage, SenderRecoveryStage}, ExecutionStageThresholds, Pipeline, StageSet + prelude::*, + stages::{ExecutionStage, SenderRecoveryStage}, + ExecutionStageThresholds, Pipeline, StageSet, }; use reth_static_file::StaticFileProducer; use std::{path::PathBuf, sync::Arc, time::Duration}; @@ -52,7 +54,8 @@ pub struct BitfinityImportCommand { provider_factory: ProviderFactory>>, - blockchain_provider: BlockchainProvider>>, + blockchain_provider: + BlockchainProvider2>>, } /// Manually implement `Debug` for `ImportCommand` because `BlockchainProvider` doesn't implement it. @@ -77,7 +80,7 @@ impl BitfinityImportCommand { chain: Arc, bitfinity: BitfinityImportArgs, provider_factory: ProviderFactory>>, - blockchain_provider: BlockchainProvider< + blockchain_provider: BlockchainProvider2< NodeTypesWithDBAdapter>, >, ) -> Self { diff --git a/bin/reth/src/main.rs b/bin/reth/src/main.rs index f7d9bf62499..b99a7f4ce6b 100644 --- a/bin/reth/src/main.rs +++ b/bin/reth/src/main.rs @@ -91,32 +91,31 @@ fn main() { builder.launch_with(launcher) }) .await?; - handle.node_exit_future.await - } - true => { - info!(target: "reth::cli", "Running with legacy engine"); - let handle = builder.launch_node(EthereumNode::default()).await?; + let blockchain_provider = handle.node.provider.clone(); let config = handle.node.config.config.clone(); let chain = handle.node.chain_spec(); let datadir = handle.node.data_dir.clone(); - let (provider_factory, bitfinity) = handle.bitfinity_import.clone().expect("Bitfinity import not configured"); + let (provider_factory, bitfinity) = handle.bitfinity_import.clone().expect("Bitfinity import not configured"); // Init bitfinity import - { - let import = BitfinityImportCommand::new( - config, - datadir, - chain, - bitfinity, - provider_factory, - blockchain_provider, - ); - let _import_handle = import.schedule_execution().await?; - }; + let import = BitfinityImportCommand::new( + config, + datadir, + chain, + bitfinity.clone(), + provider_factory, + blockchain_provider, + ); + let _import_handle = import.schedule_execution().await?; handle.node_exit_future.await } + true => { + info!(target: "reth::cli", "Running with legacy engine"); + let handle = builder.launch_node(EthereumNode::default()).await?; + handle.node_exit_future.await + } } }) { diff --git a/bin/reth/tests/commands/bitfinity_node_it.rs b/bin/reth/tests/commands/bitfinity_node_it.rs index 8c9de89f861..89f2d1cc9ee 100644 --- a/bin/reth/tests/commands/bitfinity_node_it.rs +++ b/bin/reth/tests/commands/bitfinity_node_it.rs @@ -17,18 +17,20 @@ use reth::{ dirs::{DataDirPath, MaybePlatformPath}, }; use reth_consensus::FullConsensus; +use reth_db::test_utils::TempDatabase; use reth_db::DatabaseEnv; use reth_db::{init_db, test_utils::tempdir_path}; use reth_network::NetworkHandle; use reth_node_api::{FullNodeTypesAdapter, NodeTypesWithDBAdapter}; use reth_node_builder::components::Components; +use reth_node_builder::engine_tree_config::TreeConfig; use reth_node_builder::rpc::RpcAddOns; -use reth_node_builder::{NodeAdapter, NodeBuilder, NodeConfig, NodeHandle}; -use reth_node_ethereum::node::EthereumEngineValidatorBuilder; +use reth_node_builder::{EngineNodeLauncher, NodeAdapter, NodeBuilder, NodeConfig, NodeHandle}; +use reth_node_ethereum::node::{EthereumAddOns, EthereumEngineValidatorBuilder}; use reth_node_ethereum::{ BasicBlockExecutorProvider, EthEvmConfig, EthExecutionStrategyFactory, EthereumNode, }; -use reth_provider::providers::BlockchainProvider; +use reth_provider::providers::BlockchainProvider2; use reth_rpc::EthApi; use reth_tasks::TaskManager; use reth_transaction_pool::blobstore::DiskFileBlobStore; @@ -207,7 +209,107 @@ async fn start_reth_node( import_data: Option, ) -> ( EthJsonRpcClient, - NodeHandle, BlockchainProvider>>>, Components, BlockchainProvider>>>, reth_network::EthNetworkPrimitives, Pool>>, EthPooledTransaction>>, CoinbaseTipOrdering, DiskFileBlobStore>, EthEvmConfig, BasicBlockExecutorProvider, Arc>>, RpcAddOns, BlockchainProvider>>>, Components, BlockchainProvider>>>, reth_network::EthNetworkPrimitives, Pool>>, EthPooledTransaction>>, CoinbaseTipOrdering, DiskFileBlobStore>, EthEvmConfig, BasicBlockExecutorProvider, Arc>>, EthApi>>, Pool>>, EthPooledTransaction>>, CoinbaseTipOrdering, DiskFileBlobStore>, NetworkHandle, EthEvmConfig>, EthereumEngineValidatorBuilder>>, + NodeHandle< + NodeAdapter< + FullNodeTypesAdapter< + EthereumNode, + Arc>, + BlockchainProvider2< + NodeTypesWithDBAdapter>>, + >, + >, + Components< + FullNodeTypesAdapter< + EthereumNode, + Arc>, + BlockchainProvider2< + NodeTypesWithDBAdapter>>, + >, + >, + reth_network::EthNetworkPrimitives, + Pool< + TransactionValidationTaskExecutor< + EthTransactionValidator< + BlockchainProvider2< + NodeTypesWithDBAdapter< + EthereumNode, + Arc>, + >, + >, + EthPooledTransaction, + >, + >, + CoinbaseTipOrdering, + DiskFileBlobStore, + >, + EthEvmConfig, + BasicBlockExecutorProvider, + Arc, + >, + >, + RpcAddOns< + NodeAdapter< + FullNodeTypesAdapter< + EthereumNode, + Arc>, + BlockchainProvider2< + NodeTypesWithDBAdapter>>, + >, + >, + Components< + FullNodeTypesAdapter< + EthereumNode, + Arc>, + BlockchainProvider2< + NodeTypesWithDBAdapter>>, + >, + >, + reth_network::EthNetworkPrimitives, + Pool< + TransactionValidationTaskExecutor< + EthTransactionValidator< + BlockchainProvider2< + NodeTypesWithDBAdapter< + EthereumNode, + Arc>, + >, + >, + EthPooledTransaction, + >, + >, + CoinbaseTipOrdering, + DiskFileBlobStore, + >, + EthEvmConfig, + BasicBlockExecutorProvider, + Arc, + >, + >, + EthApi< + BlockchainProvider2< + NodeTypesWithDBAdapter>>, + >, + Pool< + TransactionValidationTaskExecutor< + EthTransactionValidator< + BlockchainProvider2< + NodeTypesWithDBAdapter< + EthereumNode, + Arc>, + >, + >, + EthPooledTransaction, + >, + >, + CoinbaseTipOrdering, + DiskFileBlobStore, + >, + NetworkHandle, + EthEvmConfig, + >, + EthereumEngineValidatorBuilder, + >, + >, ) { let tasks = TaskManager::current(); @@ -240,10 +342,21 @@ async fn start_reth_node( Arc::new(init_db(data_dir.db(), Default::default()).unwrap()) }; + let exec = tasks.executor(); let node_handle = NodeBuilder::new(node_config) .with_database(database) - .with_launch_context(tasks.executor()) - .launch_node(EthereumNode::default()) + .testing_node(exec) + .with_types_and_provider::>() + .with_components(EthereumNode::components()) + .with_add_ons(EthereumAddOns::default()) + .launch_with_fn(|builder| { + let launcher = EngineNodeLauncher::new( + builder.task_executor().clone(), + builder.config().datadir(), + TreeConfig::default(), + ); + builder.launch_with(launcher) + }) .await .unwrap(); diff --git a/bin/reth/tests/commands/utils.rs b/bin/reth/tests/commands/utils.rs index 3f2b102f22a..e0e3a5572f7 100644 --- a/bin/reth/tests/commands/utils.rs +++ b/bin/reth/tests/commands/utils.rs @@ -1,6 +1,6 @@ //! //! Utils for bitfinity integration tests -//! +//! use std::{ fmt::{Debug, Display, Formatter}, path::PathBuf, @@ -18,20 +18,16 @@ use reth::{ commands::bitfinity_import::BitfinityImportCommand, dirs::{ChainPath, DataDirPath, PlatformPath}, }; -use reth_beacon_consensus::EthBeaconConsensus; -use reth_blockchain_tree::{BlockchainTreeConfig, ShareableBlockchainTree, TreeExternals}; use reth_chainspec::ChainSpec; use reth_db::{init_db, DatabaseEnv}; use reth_downloaders::bitfinity_evm_client::BitfinityEvmClient; use reth_errors::BlockExecutionError; -use reth_evm::execute::{ - BatchExecutor, BlockExecutionOutput, BlockExecutorProvider, Executor, -}; +use reth_evm::execute::{BatchExecutor, BlockExecutionOutput, BlockExecutorProvider, Executor}; use reth_node_api::NodeTypesWithDBAdapter; use reth_node_ethereum::EthereumNode; use reth_primitives::{BlockWithSenders, EthPrimitives, Receipt}; use reth_provider::{ - providers::{BlockchainProvider, StaticFileProvider}, + providers::{BlockchainProvider2, StaticFileProvider}, BlockNumReader, ExecutionOutcome, ProviderError, ProviderFactory, }; use reth_prune::PruneModes; @@ -64,7 +60,6 @@ pub fn init_logs() -> eyre::Result> { /// Type alias for the node types. pub type NodeTypes = NodeTypesWithDBAdapter>; - #[derive(Clone)] /// Data needed for the import tests. pub struct ImportData { @@ -77,7 +72,7 @@ pub struct ImportData { /// The provider factory. pub provider_factory: ProviderFactory, /// The blockchain provider. - pub blockchain_db: BlockchainProvider, + pub blockchain_db: BlockchainProvider2, /// The bitfinity import arguments. pub bitfinity_args: BitfinityImportArgs, } @@ -148,17 +143,7 @@ pub async fn bitfinity_import_config_data( reth_db_common::init::init_genesis(&provider_factory)?; - let consensus = Arc::new(EthBeaconConsensus::new(chain.clone())); - - let executor = MockExecutorProvider::default(); //EvmExecutorFac::new(self.chain.clone(), EthEvmConfig::default()); - - let blockchain_tree = - Arc::new(ShareableBlockchainTree::new(reth_blockchain_tree::BlockchainTree::new( - TreeExternals::new(provider_factory.clone(), consensus, executor), - BlockchainTreeConfig::default(), - )?)); - - let blockchain_db = BlockchainProvider::new(provider_factory.clone(), blockchain_tree)?; + let blockchain_db = BlockchainProvider2::new(provider_factory.clone())?; let bitfinity_args = BitfinityImportArgs { rpc_url: evm_datasource_url.to_string(), From 15a87df3db5f9fdcfd486b1795d9c19b34a7e191 Mon Sep 17 00:00:00 2001 From: f3kilo Date: Mon, 27 Jan 2025 09:55:16 +0300 Subject: [PATCH 2/3] use only new engine --- bin/reth/src/main.rs | 5 +++-- bitfinity.md | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/bin/reth/src/main.rs b/bin/reth/src/main.rs index b99a7f4ce6b..0eb6e44ca51 100644 --- a/bin/reth/src/main.rs +++ b/bin/reth/src/main.rs @@ -70,8 +70,9 @@ fn main() { } // let use_legacy_engine = engine_args.legacy; - // Bitfinity import is implemented only for the legacy engine - let use_legacy_engine = true; + + // Bitfinity import is implemented only for the new engine + let use_legacy_engine = false; match use_legacy_engine { false => { diff --git a/bitfinity.md b/bitfinity.md index e687051e77a..9fa7a6f06fa 100644 --- a/bitfinity.md +++ b/bitfinity.md @@ -34,7 +34,7 @@ reth node -vvv --http --http.port 8080 --http.addr 0.0.0.0 --http.api "debug,eth With cargo: ```sh -cargo run -p reth -- node -vvv --http --http.port 8080 --http.addr 0.0.0.0 --http.api "debug,eth,net,trace,txpool,web3" --disable-discovery --ipcdisable --no-persist-peers -r https://orca-app-5yyst.ondigitalocean.app -i 30 -b 100 --max-fetch-blocks 5000 --log.file.directory ./target/logs --datadir ./target/reth +cargo run -p reth -- node -vvv --http --http.port 8080 --http.addr 0.0.0.0 --http.api "debug,eth,net,trace,txpool,web3" --disable-discovery --ipcdisable --no-persist-peers -r https://block-extractor-testnet-1052151659755.europe-west9.run.app -i 30 -b 100 --max-fetch-blocks 5000 --log.file.directory ./target/logs --datadir ./target/reth ``` You can query the node using the JSON-RPC API. For example, to get the block number, you can use the following command: From 1211929ff2041e2cd47ea86e86efb4da0d762f65 Mon Sep 17 00:00:00 2001 From: Francesco Date: Fri, 28 Mar 2025 09:57:22 +0100 Subject: [PATCH 3/3] remove unused field --- bin/reth/src/commands/bitfinity_import.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/reth/src/commands/bitfinity_import.rs b/bin/reth/src/commands/bitfinity_import.rs index ed25300f23c..e6b9c307eda 100644 --- a/bin/reth/src/commands/bitfinity_import.rs +++ b/bin/reth/src/commands/bitfinity_import.rs @@ -44,8 +44,6 @@ use tracing::{debug, error, info, warn}; pub struct BitfinityImportCommand { config: Config, - datadir: ChainPath, - /// The chain this node is running. /// /// Possible values are either a built-in chain or the path to a chain specification file. @@ -86,7 +84,7 @@ impl BitfinityImportCommand { config.stages.etl.dir = Some(EtlConfig::from_datadir(datadir.data_dir())); } - Self { config, datadir, chain, bitfinity, provider_factory, blockchain_provider } + Self { config, chain, bitfinity, provider_factory, blockchain_provider } } /// Schedule the import job and return a handle to it.