[EPROD-1111] Engine v2

bin/reth/Cargo.toml

@@ -124,6 +124,7 @@ reth-db-common.workspace = true
 reth-db = { workspace = true, features = ["mdbx", "test-utils"] }
 serial_test.workspace = true
 reth-discv5.workspace = true
+reth-node-ethereum = { workspace = true, features = ["test-utils"] }
 reth-transaction-pool = { workspace = true, features = ["test-utils"] }
 
 [features]

bin/reth/src/commands/bitfinity_import.rs

@@ -23,9 +23,10 @@ use reth_node_core::{args::BitfinityImportArgs, dirs::ChainPath};
 use reth_node_ethereum::{EthExecutorProvider, EthereumNode};
 use reth_node_events::node::NodeEvent;
 use reth_primitives::{EthPrimitives, SealedHeader};
+use reth_provider::providers::BlockchainProvider2;
 use reth_provider::{
-    providers::BlockchainProvider, BlockHashReader, BlockNumReader, CanonChainTracker,
-    ChainSpecProvider, DatabaseProviderFactory, HeaderProvider, ProviderError, ProviderFactory,
+    BlockHashReader, BlockNumReader, CanonChainTracker, ChainSpecProvider, DatabaseProviderFactory,
+    HeaderProvider, ProviderError, ProviderFactory,
 };
 use reth_prune::PruneModes;
 use reth_stages::{
@@ -39,12 +40,10 @@ use tokio::sync::watch;
 use tracing::{debug, error, info, warn};
 
 /// Syncs RLP encoded blocks from a file.
-#[derive(Clone)]
+#[derive(Debug, Clone)]
 pub struct BitfinityImportCommand {
     config: Config,
 
-    datadir: ChainPath<DataDirPath>,
-
     /// The chain this node is running.
     ///
     /// Possible values are either a built-in chain or the path to a chain specification file.
@@ -55,20 +54,8 @@ pub struct BitfinityImportCommand {
 
     provider_factory: ProviderFactory<NodeTypesWithDBAdapter<EthereumNode, Arc<DatabaseEnv>>>,
 
-    blockchain_provider: BlockchainProvider<NodeTypesWithDBAdapter<EthereumNode, Arc<DatabaseEnv>>>,
-}
-
-/// Manually implement `Debug` for `ImportCommand` because `BlockchainProvider` doesn't implement
-/// it.
-impl std::fmt::Debug for BitfinityImportCommand {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("ImportCommand")
-            .field("config", &self.config)
-            .field("datadir", &self.datadir)
-            .field("chain", &self.chain)
-            .field("bitfinity", &self.bitfinity)
-            .finish()
-    }
+    blockchain_provider:
+        BlockchainProvider2<NodeTypesWithDBAdapter<EthereumNode, Arc<DatabaseEnv>>>,
 }
 
 type TypedPipeline = Pipeline<NodeTypesWithDBAdapter<EthereumNode, Arc<DatabaseEnv>>>;
@@ -81,7 +68,7 @@ impl BitfinityImportCommand {
         chain: Arc<ChainSpec>,
         bitfinity: BitfinityImportArgs,
         provider_factory: ProviderFactory<NodeTypesWithDBAdapter<EthereumNode, Arc<DatabaseEnv>>>,
-        blockchain_provider: BlockchainProvider<
+        blockchain_provider: BlockchainProvider2<
             NodeTypesWithDBAdapter<EthereumNode, Arc<DatabaseEnv>>,
         >,
     ) -> Self {
@@ -97,7 +84,7 @@ impl BitfinityImportCommand {
             config.stages.etl.dir = Some(EtlConfig::from_datadir(datadir.data_dir()));
         }
 
-        Self { config, datadir, chain, bitfinity, provider_factory, blockchain_provider }
+        Self { config, chain, bitfinity, provider_factory, blockchain_provider }
     }
 
     /// Schedule the import job and return a handle to it.
@@ -188,12 +175,19 @@ impl BitfinityImportCommand {
                     Some(v) => v,
                     None => {
                         error!(target: "reth::cli - BitfinityImportCommand", "Hash of latest safe block ({}) is not in the blockchain state", safe_block_number);
-                        return Err(eyre!("Hash of latest safe block ({}) is not in the blockchain state", safe_block_number));
+                        return Err(eyre!(
+                            "Hash of latest safe block ({}) is not in the blockchain state",
+                            safe_block_number
+                        ));
                     }
                 },
                 false => {
                     error!(target: "reth::cli - BitfinityImportCommand", "Last imported block ({}) is not the last safe block ({})", last_imported_block, safe_block_number);
-                    return Err(eyre!("Last imported block ({}) is not the last safe block ({})", last_imported_block, safe_block_number));
+                    return Err(eyre!(
+                        "Last imported block ({}) is not the last safe block ({})",
+                        last_imported_block,
+                        safe_block_number
+                    ));
                 }
             }
         };

bin/reth/src/main.rs

@@ -70,8 +70,9 @@ fn main() {
             }
 
             // let use_legacy_engine = engine_args.legacy;
-            // Bitfinity import is implemented only for the legacy engine
-            let use_legacy_engine = true;
+
+            // Bitfinity import is implemented only for the new engine
+            let use_legacy_engine = false;
 
             match use_legacy_engine {
                 false => {
@@ -91,32 +92,31 @@ fn main() {
                             builder.launch_with(launcher)
                         })
                         .await?;
-                    handle.node_exit_future.await
-                }
-                true => {
-                    info!(target: "reth::cli", "Running with legacy engine");
-                    let handle = builder.launch_node(EthereumNode::default()).await?;
+
                     let blockchain_provider = handle.node.provider.clone();
                     let config = handle.node.config.config.clone();
                     let chain = handle.node.chain_spec();
                     let datadir = handle.node.data_dir.clone();
-                    let (provider_factory, bitfinity) = handle.bitfinity_import.clone().expect("Bitfinity import not configured");
+                    let (provider_factory, bitfinity) = handle.bitfinity_import.clone().expect("Bitfinity import not configured");                    
 
                     // Init bitfinity import
-                    {
-                        let import = BitfinityImportCommand::new(
-                            config,
-                            datadir,
-                            chain,
-                            bitfinity,
-                            provider_factory,
-                            blockchain_provider,
-                        );
-                        let _import_handle = import.schedule_execution().await?;
-                    };
+                    let import = BitfinityImportCommand::new(
+                        config,
+                        datadir,
+                        chain,
+                        bitfinity.clone(),
+                        provider_factory,
+                        blockchain_provider,
+                    );
+                    let _import_handle = import.schedule_execution().await?;
 
                     handle.node_exit_future.await
                 }
+                true => {
+                    info!(target: "reth::cli", "Running with legacy engine");
+                    let handle = builder.launch_node(EthereumNode::default()).await?;
+                    handle.node_exit_future.await
+                }
             }
         })
     {

bin/reth/tests/commands/bitfinity_node_it.rs

@@ -15,26 +15,27 @@ use reth::{
     dirs::{DataDirPath, MaybePlatformPath},
 };
 use reth_consensus::FullConsensus;
-use reth_db::{init_db, test_utils::tempdir_path, DatabaseEnv};
+use reth_db::test_utils::TempDatabase;
+use reth_db::DatabaseEnv;
+use reth_db::{init_db, test_utils::tempdir_path};
 use reth_discv5::discv5::enr::secp256k1::{Keypair, Secp256k1};
 use reth_network::NetworkHandle;
 use reth_node_api::{FullNodeTypesAdapter, NodeTypesWithDBAdapter};
-use reth_node_builder::{
-    components::Components, rpc::RpcAddOns, NodeAdapter, NodeBuilder, NodeConfig, NodeHandle,
-};
+use reth_node_builder::components::Components;
+use reth_node_builder::engine_tree_config::TreeConfig;
+use reth_node_builder::rpc::RpcAddOns;
+use reth_node_builder::{EngineNodeLauncher, NodeAdapter, NodeBuilder, NodeConfig, NodeHandle};
+use reth_node_ethereum::node::{EthereumAddOns, EthereumEngineValidatorBuilder};
 use reth_node_ethereum::{
-    node::EthereumEngineValidatorBuilder, BasicBlockExecutorProvider, EthEvmConfig,
-    EthExecutionStrategyFactory, EthereumNode,
+    BasicBlockExecutorProvider, EthEvmConfig, EthExecutionStrategyFactory, EthereumNode,
 };
 use reth_primitives::{Transaction, TransactionSigned};
-use reth_provider::providers::BlockchainProvider;
+use reth_provider::providers::BlockchainProvider2;
 use reth_rpc::EthApi;
 use reth_tasks::TaskManager;
-use reth_transaction_pool::blobstore::DiskFileBlobStore;
-use reth_transaction_pool::test_utils::MockTransaction;
 use reth_transaction_pool::{
-    CoinbaseTipOrdering, EthPooledTransaction, EthTransactionValidator, Pool,
-    TransactionValidationTaskExecutor,
+    blobstore::DiskFileBlobStore, CoinbaseTipOrdering, EthPooledTransaction,
+    EthTransactionValidator, Pool, TransactionValidationTaskExecutor,
 };
 use revm_primitives::{hex, Address, B256, U256};
 use std::{net::SocketAddr, str::FromStr, sync::Arc};
@@ -336,21 +337,28 @@ pub async fn start_reth_node(
         NodeAdapter<
             FullNodeTypesAdapter<
                 EthereumNode,
-                Arc<DatabaseEnv>,
-                BlockchainProvider<NodeTypesWithDBAdapter<EthereumNode, Arc<DatabaseEnv>>>,
+                Arc<TempDatabase<DatabaseEnv>>,
+                BlockchainProvider2<
+                    NodeTypesWithDBAdapter<EthereumNode, Arc<TempDatabase<DatabaseEnv>>>,
+                >,
             >,
             Components<
                 FullNodeTypesAdapter<
                     EthereumNode,
-                    Arc<DatabaseEnv>,
-                    BlockchainProvider<NodeTypesWithDBAdapter<EthereumNode, Arc<DatabaseEnv>>>,
+                    Arc<TempDatabase<DatabaseEnv>>,
+                    BlockchainProvider2<
+                        NodeTypesWithDBAdapter<EthereumNode, Arc<TempDatabase<DatabaseEnv>>>,
+                    >,
                 >,
                 reth_network::EthNetworkPrimitives,
                 Pool<
                     TransactionValidationTaskExecutor<
                         EthTransactionValidator<
-                            BlockchainProvider<
-                                NodeTypesWithDBAdapter<EthereumNode, Arc<DatabaseEnv>>,
+                            BlockchainProvider2<
+                                NodeTypesWithDBAdapter<
+                                    EthereumNode,
+                                    Arc<TempDatabase<DatabaseEnv>>,
+                                >,
                             >,
                             EthPooledTransaction,
                         >,
@@ -367,21 +375,28 @@ pub async fn start_reth_node(
             NodeAdapter<
                 FullNodeTypesAdapter<
                     EthereumNode,
-                    Arc<DatabaseEnv>,
-                    BlockchainProvider<NodeTypesWithDBAdapter<EthereumNode, Arc<DatabaseEnv>>>,
+                    Arc<TempDatabase<DatabaseEnv>>,
+                    BlockchainProvider2<
+                        NodeTypesWithDBAdapter<EthereumNode, Arc<TempDatabase<DatabaseEnv>>>,
+                    >,
                 >,
                 Components<
                     FullNodeTypesAdapter<
                         EthereumNode,
-                        Arc<DatabaseEnv>,
-                        BlockchainProvider<NodeTypesWithDBAdapter<EthereumNode, Arc<DatabaseEnv>>>,
+                        Arc<TempDatabase<DatabaseEnv>>,
+                        BlockchainProvider2<
+                            NodeTypesWithDBAdapter<EthereumNode, Arc<TempDatabase<DatabaseEnv>>>,
+                        >,
                     >,
                     reth_network::EthNetworkPrimitives,
                     Pool<
                         TransactionValidationTaskExecutor<
                             EthTransactionValidator<
-                                BlockchainProvider<
-                                    NodeTypesWithDBAdapter<EthereumNode, Arc<DatabaseEnv>>,
+                                BlockchainProvider2<
+                                    NodeTypesWithDBAdapter<
+                                        EthereumNode,
+                                        Arc<TempDatabase<DatabaseEnv>>,
+                                    >,
                                 >,
                                 EthPooledTransaction,
                             >,
@@ -395,12 +410,17 @@ pub async fn start_reth_node(
                 >,
             >,
             EthApi<
-                BlockchainProvider<NodeTypesWithDBAdapter<EthereumNode, Arc<DatabaseEnv>>>,
+                BlockchainProvider2<
+                    NodeTypesWithDBAdapter<EthereumNode, Arc<TempDatabase<DatabaseEnv>>>,
+                >,
                 Pool<
                     TransactionValidationTaskExecutor<
                         EthTransactionValidator<
-                            BlockchainProvider<
-                                NodeTypesWithDBAdapter<EthereumNode, Arc<DatabaseEnv>>,
+                            BlockchainProvider2<
+                                NodeTypesWithDBAdapter<
+                                    EthereumNode,
+                                    Arc<TempDatabase<DatabaseEnv>>,
+                                >,
                             >,
                             EthPooledTransaction,
                         >,
@@ -447,10 +467,21 @@ pub async fn start_reth_node(
         Arc::new(init_db(data_dir.db(), Default::default()).unwrap())
     };
 
+    let exec = tasks.executor();
     let node_handle = NodeBuilder::new(node_config)
         .with_database(database)
-        .with_launch_context(tasks.executor())
-        .launch_node(EthereumNode::default())
+        .testing_node(exec)
+        .with_types_and_provider::<EthereumNode, BlockchainProvider2<_>>()
+        .with_components(EthereumNode::components())
+        .with_add_ons(EthereumAddOns::default())
+        .launch_with_fn(|builder| {
+            let launcher = EngineNodeLauncher::new(
+                builder.task_executor().clone(),
+                builder.config().datadir(),
+                TreeConfig::default(),
+            );
+            builder.launch_with(launcher)
+        })
         .await
         .unwrap();
 

bin/reth/tests/commands/utils.rs

@@ -18,8 +18,6 @@ use reth::{
     commands::bitfinity_import::BitfinityImportCommand,
     dirs::{ChainPath, DataDirPath, PlatformPath},
 };
-use reth_beacon_consensus::EthBeaconConsensus;
-use reth_blockchain_tree::{BlockchainTreeConfig, ShareableBlockchainTree, TreeExternals};
 use reth_chainspec::ChainSpec;
 use reth_db::{init_db, DatabaseEnv};
 use reth_downloaders::bitfinity_evm_client::BitfinityEvmClient;
@@ -29,7 +27,7 @@ use reth_node_api::NodeTypesWithDBAdapter;
 use reth_node_ethereum::EthereumNode;
 use reth_primitives::{BlockWithSenders, EthPrimitives, Receipt};
 use reth_provider::{
-    providers::{BlockchainProvider, StaticFileProvider},
+    providers::{BlockchainProvider2, StaticFileProvider},
     BlockNumReader, ExecutionOutcome, ProviderError, ProviderFactory,
 };
 use reth_prune::PruneModes;
@@ -75,7 +73,7 @@ pub struct ImportData {
     /// The provider factory.
     pub provider_factory: ProviderFactory<NodeTypes>,
     /// The blockchain provider.
-    pub blockchain_db: BlockchainProvider<NodeTypes>,
+    pub blockchain_db: BlockchainProvider2<NodeTypes>,
     /// The bitfinity import arguments.
     pub bitfinity_args: BitfinityImportArgs,
 }
@@ -146,17 +144,7 @@ pub async fn bitfinity_import_config_data(
 
     reth_db_common::init::init_genesis(&provider_factory)?;
 
-    let consensus = Arc::new(EthBeaconConsensus::new(chain.clone()));
-
-    let executor = MockExecutorProvider::default(); //EvmExecutorFac::new(self.chain.clone(), EthEvmConfig::default());
-
-    let blockchain_tree =
-        Arc::new(ShareableBlockchainTree::new(reth_blockchain_tree::BlockchainTree::new(
-            TreeExternals::new(provider_factory.clone(), consensus, executor),
-            BlockchainTreeConfig::default(),
-        )?));
-
-    let blockchain_db = BlockchainProvider::new(provider_factory.clone(), blockchain_tree)?;
+    let blockchain_db = BlockchainProvider2::new(provider_factory.clone())?;
 
     let bitfinity_args = BitfinityImportArgs {
         rpc_url: evm_datasource_url.to_string(),

bitfinity.md

@@ -34,7 +34,7 @@ reth node -vvv --http --http.port 8080 --http.addr 0.0.0.0 --http.api "debug,eth
 With cargo: 
 
 ```sh
-cargo run -p reth -- node -vvv --http --http.port 8080 --http.addr 0.0.0.0 --http.api "debug,eth,net,trace,txpool,web3" --disable-discovery --ipcdisable --no-persist-peers -r https://orca-app-5yyst.ondigitalocean.app -i 30 -b 100 --max-fetch-blocks 5000 --log.file.directory ./target/logs --datadir ./target/reth
+cargo run -p reth -- node -vvv --http --http.port 8080 --http.addr 0.0.0.0 --http.api "debug,eth,net,trace,txpool,web3" --disable-discovery --ipcdisable --no-persist-peers -r https://block-extractor-testnet-1052151659755.europe-west9.run.app -i 30 -b 100 --max-fetch-blocks 5000 --log.file.directory ./target/logs --datadir ./target/reth
 ```
 
 You can query the node using the JSON-RPC API. For example, to get the block number, you can use the following command: