From ec9cd2a18edfe8f677a9fb2c83d8caf2cb07c106 Mon Sep 17 00:00:00 2001 From: Yasir Date: Mon, 18 Nov 2024 12:57:36 +0300 Subject: [PATCH 1/2] chore: add reth redundancy --- Cargo.lock | 1 + Cargo.toml | 1 + bin/reth/src/commands/bitfinity_import.rs | 10 +- bin/reth/src/commands/node/mod.rs | 6 +- .../tests/commands/bitfinity_import_it.rs | 55 ++++- .../commands/bitfinity_reset_evm_state_it.rs | 4 +- bin/reth/tests/commands/utils.rs | 9 +- crates/net/downloaders/Cargo.toml | 1 + .../downloaders/src/bitfinity_evm_client.rs | 188 ++++++++++++++++-- crates/node/core/src/args/bitfinity_args.rs | 26 ++- 10 files changed, 261 insertions(+), 40 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ceb2590d604..6d19aa342f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7721,6 +7721,7 @@ version = "1.0.0" dependencies = [ "alloy-rlp", "assert_matches", + "backon", "candid", "did", "ethereum-json-rpc-client", diff --git a/Cargo.toml b/Cargo.toml index 4db6116eabc..1711e063f9e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -544,3 +544,4 @@ ic-certification = "2.3" hex = "0.4" lightspeed_scheduler = { version = "0.59.0", features = ["tracing"] } rlp = "0.5" + diff --git a/bin/reth/src/commands/bitfinity_import.rs b/bin/reth/src/commands/bitfinity_import.rs index 2200795c60c..8ded202c962 100644 --- a/bin/reth/src/commands/bitfinity_import.rs +++ b/bin/reth/src/commands/bitfinity_import.rs @@ -10,6 +10,7 @@ use reth_db::DatabaseEnv; use reth_consensus::Consensus; use reth_db::database::Database; +use reth_downloaders::bitfinity_evm_client::RpcClientConfig; use reth_downloaders::{ bitfinity_evm_client::{BitfinityEvmClient, CertificateCheckSettings}, bodies::bodies::BodiesDownloaderBuilder, @@ -133,9 +134,16 @@ impl BitfinityImportCommand { debug!(target: "reth::cli - BitfinityImportCommand", "Starting block: {}", start_block); + let rpc_config = RpcClientConfig { + primary_url: self.bitfinity.rpc_url.clone(), + backup_url: self.bitfinity.backup_rpc_url.clone(), + max_retries: self.bitfinity.max_retries, + retry_delay: Duration::from_secs(self.bitfinity.retry_delay_secs), + }; + let remote_client = Arc::new( BitfinityEvmClient::from_rpc_url( - &self.bitfinity.rpc_url, + rpc_config, start_block, self.bitfinity.end_block, self.bitfinity.batch_size, diff --git a/bin/reth/src/commands/node/mod.rs b/bin/reth/src/commands/node/mod.rs index fb9379429dd..a9368d36113 100644 --- a/bin/reth/src/commands/node/mod.rs +++ b/bin/reth/src/commands/node/mod.rs @@ -65,7 +65,7 @@ pub struct NodeCommand { /// Bitfinity Args #[command(flatten)] pub bitfinity: crate::args::BitfinityImportArgs, - + /// All datadir related arguments #[command(flatten)] pub datadir: DatadirArgs, @@ -155,7 +155,7 @@ impl NodeCommand { } = self; let chain = { - let mut chain = reth_downloaders::bitfinity_evm_client::BitfinityEvmClient::fetch_chain_spec(bitfinity.rpc_url.to_owned()).await?; + let mut chain = reth_downloaders::bitfinity_evm_client::BitfinityEvmClient::fetch_chain_spec_with_fallback(bitfinity.rpc_url.to_owned(), bitfinity.backup_rpc_url.clone()).await?; if let Some(send_raw_transaction_rpc_url) = &bitfinity.send_raw_transaction_rpc_url { chain.bitfinity_evm_url = Some(send_raw_transaction_rpc_url.to_owned()); } @@ -197,7 +197,7 @@ impl NodeCommand { let builder = NodeBuilder::new(node_config) .with_database(database) .with_launch_context(ctx.task_executor); - + launcher(builder, ext).await } } diff --git a/bin/reth/tests/commands/bitfinity_import_it.rs b/bin/reth/tests/commands/bitfinity_import_it.rs index 9e1ed860ae0..29181286120 100644 --- a/bin/reth/tests/commands/bitfinity_import_it.rs +++ b/bin/reth/tests/commands/bitfinity_import_it.rs @@ -14,7 +14,7 @@ async fn bitfinity_test_should_import_data_from_evm() { let _log = init_logs(); let evm_datasource_url = DEFAULT_EVM_DATASOURCE_URL; let (_temp_dir, mut import_data) = - bitfinity_import_config_data(evm_datasource_url, None).await.unwrap(); + bitfinity_import_config_data(evm_datasource_url, None, None).await.unwrap(); let end_block = 100; import_data.bitfinity_args.end_block = Some(end_block); @@ -46,7 +46,7 @@ async fn bitfinity_test_should_import_with_small_batch_size() { let _log = init_logs(); let evm_datasource_url = DEFAULT_EVM_DATASOURCE_URL; let (_temp_dir, mut import_data) = - bitfinity_import_config_data(evm_datasource_url, None).await.unwrap(); + bitfinity_import_config_data(evm_datasource_url, None, None).await.unwrap(); let end_block = 101; import_data.bitfinity_args.end_block = Some(end_block); @@ -72,14 +72,13 @@ async fn bitfinity_test_should_import_with_small_batch_size() { } } - #[tokio::test] async fn bitfinity_test_finalized_and_safe_query_params_works() { // Arrange let _log = init_logs(); let evm_datasource_url = DEFAULT_EVM_DATASOURCE_URL; let (_temp_dir, mut import_data) = - bitfinity_import_config_data(evm_datasource_url, None).await.unwrap(); + bitfinity_import_config_data(evm_datasource_url, None, None).await.unwrap(); let end_block = 100; import_data.bitfinity_args.end_block = Some(end_block); @@ -88,10 +87,52 @@ async fn bitfinity_test_finalized_and_safe_query_params_works() { // Act import_blocks(import_data.clone(), Duration::from_secs(20), true).await; - let latest_block = import_data.blockchain_db.block_by_number_or_tag(reth_rpc_types::BlockNumberOrTag::Finalized).unwrap().unwrap(); + let latest_block = import_data + .blockchain_db + .block_by_number_or_tag(reth_rpc_types::BlockNumberOrTag::Finalized) + .unwrap() + .unwrap(); assert_eq!(end_block, latest_block.number); - let safe_block = import_data.blockchain_db.block_by_number_or_tag(reth_rpc_types::BlockNumberOrTag::Safe).unwrap().unwrap(); + let safe_block = import_data + .blockchain_db + .block_by_number_or_tag(reth_rpc_types::BlockNumberOrTag::Safe) + .unwrap() + .unwrap(); assert_eq!(end_block, safe_block.number); +} + +#[tokio::test] +async fn bitfinity_test_should_import_data_from_evm_with_backup_rpc_url() { + // Arrange + let _log = init_logs(); + let evm_datasource_url = "https://fake_rpc_url"; + let backup_rpc_url = DEFAULT_EVM_DATASOURCE_URL; + + let (_temp_dir, mut import_data) = + bitfinity_import_config_data(evm_datasource_url, Some(backup_rpc_url.to_owned()), None) + .await + .unwrap(); + + let end_block = 100; + import_data.bitfinity_args.end_block = Some(end_block); + import_data.bitfinity_args.batch_size = (end_block as usize) * 10; + + // Act + import_blocks(import_data.clone(), Duration::from_secs(20), false).await; -} \ No newline at end of file + // Assert + { + let provider = import_data.provider_factory.provider().unwrap(); + assert_eq!(end_block, provider.last_block_number().unwrap()); + + // create evm client + let evm_rpc_client = EthJsonRpcClient::new(ReqwestClient::new(backup_rpc_url.to_string())); + + let remote_block = evm_rpc_client.get_block_by_number(end_block.into()).await.unwrap(); + let local_block = provider.block_by_number(end_block).unwrap().unwrap(); + + assert_eq!(remote_block.hash.unwrap().0, local_block.header.hash_slow().0); + assert_eq!(remote_block.state_root.0, local_block.state_root.0); + } +} diff --git a/bin/reth/tests/commands/bitfinity_reset_evm_state_it.rs b/bin/reth/tests/commands/bitfinity_reset_evm_state_it.rs index 470c071a61e..78b47a48477 100644 --- a/bin/reth/tests/commands/bitfinity_reset_evm_state_it.rs +++ b/bin/reth/tests/commands/bitfinity_reset_evm_state_it.rs @@ -39,7 +39,7 @@ async fn bitfinity_manual_test_should_reset_evm_state() { let end_block = 30_000; let data_dir = Some(format!("../../target/reth_{end_block}").into()); let (_temp_dir, mut import_data) = - bitfinity_import_config_data(evm_datasource_url, data_dir).await.unwrap(); + bitfinity_import_config_data(evm_datasource_url, None, data_dir).await.unwrap(); let fetch_block_timeout_secs = std::cmp::max(20, end_block / 100); @@ -99,7 +99,7 @@ async fn bitfinity_test_reset_should_extract_all_accounts_data() { let end_block = 30_000; let data_dir = Some(format!("../../target/reth_{end_block}").into()); let (_temp_dir, mut import_data) = - bitfinity_import_config_data(evm_datasource_url, data_dir).await.unwrap(); + bitfinity_import_config_data(evm_datasource_url, None, data_dir).await.unwrap(); let fetch_block_timeout_secs = std::cmp::max(20, end_block / 100); diff --git a/bin/reth/tests/commands/utils.rs b/bin/reth/tests/commands/utils.rs index 6462acc73ed..0a6040440e5 100644 --- a/bin/reth/tests/commands/utils.rs +++ b/bin/reth/tests/commands/utils.rs @@ -99,6 +99,7 @@ pub async fn import_blocks( /// If a `data_dir` is provided, it will be used, otherwise a temporary directory will be created. pub async fn bitfinity_import_config_data( evm_datasource_url: &str, + backup_evm_datasource_url: Option, data_dir: Option, ) -> eyre::Result<(TempDir, ImportData)> { let chain = @@ -144,9 +145,15 @@ pub async fn bitfinity_import_config_data( max_fetch_blocks: 10000, evmc_principal: LOCAL_EVM_CANISTER_ID.to_string(), ic_root_key: IC_MAINNET_KEY.to_string(), + backup_rpc_url: backup_evm_datasource_url, + max_retries: 3, + retry_delay_secs: 3, }; - Ok((temp_dir, ImportData { data_dir, database, chain, provider_factory, blockchain_db, bitfinity_args })) + Ok(( + temp_dir, + ImportData { data_dir, database, chain, provider_factory, blockchain_db, bitfinity_args }, + )) } /// Waits until the block is imported. diff --git a/crates/net/downloaders/Cargo.toml b/crates/net/downloaders/Cargo.toml index a0e17cf8abf..3e7ebb81a4a 100644 --- a/crates/net/downloaders/Cargo.toml +++ b/crates/net/downloaders/Cargo.toml @@ -61,6 +61,7 @@ ic-certification.workspace = true reth-chainspec.workspace = true rlp.workspace = true serde_json.workspace = true +backon.workspace = true [dev-dependencies] reth-chainspec.workspace = true diff --git a/crates/net/downloaders/src/bitfinity_evm_client.rs b/crates/net/downloaders/src/bitfinity_evm_client.rs index a97de56ef3c..d95b593e623 100644 --- a/crates/net/downloaders/src/bitfinity_evm_client.rs +++ b/crates/net/downloaders/src/bitfinity_evm_client.rs @@ -27,10 +27,26 @@ use reth_primitives::{ use rlp::Encodable; use serde_json::json; +use std::time::Duration; use std::{self, cmp::min, collections::HashMap}; use thiserror::Error; -use tracing::{debug, error, info, trace}; +use backon::{ExponentialBuilder, Retryable}; + +use tracing::{debug, error, info, trace, warn}; + +/// RPC client configuration +#[derive(Debug, Clone)] +pub struct RpcClientConfig { + /// Primary RPC URL + pub primary_url: String, + /// Backup RPC URL + pub backup_url: Option, + /// Maximum number of retries + pub max_retries: u32, + /// Delay between retries + pub retry_delay: Duration, +} /// Front-end API for fetching chain data from remote sources. /// @@ -76,7 +92,7 @@ pub struct CertificateCheckSettings { impl BitfinityEvmClient { /// `BitfinityEvmClient` from rpc url pub async fn from_rpc_url( - rpc: &str, + rpc_config: RpcClientConfig, start_block: u64, end_block: Option, batch_size: usize, @@ -87,8 +103,9 @@ impl BitfinityEvmClient { let mut hash_to_number = HashMap::new(); let mut bodies = HashMap::new(); - let reqwest_client = ethereum_json_rpc_client::reqwest::ReqwestClient::new(rpc.to_string()); - let provider = ethereum_json_rpc_client::EthJsonRpcClient::new(reqwest_client); + let provider = Self::client(rpc_config) + .await + .map_err(|e| RemoteClientError::ProviderError(e.to_string()))?; let block_checker = match certificate_settings { None => None, @@ -196,6 +213,39 @@ impl BitfinityEvmClient { /// Fetch Bitfinity chain spec pub async fn fetch_chain_spec(rpc: String) -> Result { + Self::build_chain_spec(rpc).await + } + + /// Fetch Bitfinity chain spec with fallback + pub async fn fetch_chain_spec_with_fallback( + primary_rpc: String, + backup_rpc: Option, + ) -> Result { + match Self::build_chain_spec(primary_rpc).await { + Ok(spec) => Ok(spec), + Err(e) => { + warn!(target: "downloaders::bitfinity_evm_client", "Failed to fetch chain spec from primary URL: {}. Trying backup URL", e); + + if let Some(backup_rpc) = backup_rpc { + match Self::build_chain_spec(backup_rpc).await { + Ok(spec) => Ok(spec), + Err(e) => { + error!(target: "downloaders::bitfinity_evm_client", "Failed to fetch chain spec from backup URL: {}", e); + + Err(e) + } + } + } else { + error!(target: "downloaders::bitfinity_evm_client", "No backup URL provided, failed to fetch chain spec from primary URL: {}", e); + + Err(e) + } + } + } + } + + /// Fetch Bitfinity chain spec + async fn build_chain_spec(rpc: String) -> Result { let client = ethereum_json_rpc_client::EthJsonRpcClient::new(ReqwestClient::new(rpc.clone())); @@ -289,6 +339,55 @@ impl BitfinityEvmClient { } self } + + /// Creates a new JSON-RPC client with retry functionality + /// + /// Tries the primary URL first, falls back to backup URL if provided + pub async fn client(config: RpcClientConfig) -> Result> { + let backoff = ExponentialBuilder::default() + .with_max_times(config.max_retries as _) + .with_max_delay(config.retry_delay) + .with_jitter(); + + // Helper closure to create and test a client connection + let try_connect = move |url: String| { + let backoff = backoff.clone(); + let client = EthJsonRpcClient::new(ReqwestClient::new(url)); + + async move { + // Test connection by getting chain ID + (|| async { client.get_chain_id().await }) + .retry(&backoff) + .notify(|e, dur| { + error!( + target: "downloaders::bitfinity_evm_client", + "Failed to connect after {}s and {} retries: {}", + dur.as_secs(), + config.max_retries, + e + ); + }) + .await + .map(|_| client) + } + }; + + // Try primary URL first + if let Ok(client) = try_connect(config.primary_url).await { + debug!(target: "downloaders::bitfinity_evm_client", "Connected to primary RPC endpoint"); + return Ok(client); + } + + // Fall back to backup URL if available + if let Some(backup_url) = config.backup_url { + if let Ok(client) = try_connect(backup_url).await { + debug!(target: "downloaders::bitfinity_evm_client", "Connected to backup RPC endpoint"); + return Ok(client); + } + } + + Err(eyre::eyre!("Failed to connect to any RPC endpoint")) + } } struct BlockCertificateChecker { @@ -450,8 +549,8 @@ impl BodiesClient for BitfinityEvmClient { Some(body) => bodies.push(body), None => { error!(%hash, "Could not find body for requested block hash"); - return Box::pin(async move { Err(RequestError::BadResponse) }) - }, + return Box::pin(async move { Err(RequestError::BadResponse) }); + } } } @@ -478,19 +577,61 @@ mod tests { #[tokio::test] async fn bitfinity_remote_client_from_rpc_url() { - let client = - BitfinityEvmClient::from_rpc_url("https://cloudflare-eth.com", 0, Some(5), 5, 1000, None) - .await - .unwrap(); + let client = BitfinityEvmClient::from_rpc_url( + RpcClientConfig { + primary_url: "https://cloudflare-eth.com".to_string(), + backup_url: None, + max_retries: 3, + retry_delay: Duration::from_secs(1), + }, + 0, + Some(5), + 5, + 1000, + None, + ) + .await + .unwrap(); + assert!(client.max_block().is_some()); + } + + #[tokio::test] + async fn bitfinity_remote_client_from_backup_rpc_url() { + let client = BitfinityEvmClient::from_rpc_url( + RpcClientConfig { + primary_url: "https://cloudflare.com".to_string(), + backup_url: Some("https://cloudflare-eth.com".to_string()), + max_retries: 3, + retry_delay: Duration::from_secs(1), + }, + 0, + Some(5), + 5, + 1000, + None, + ) + .await + .unwrap(); assert!(client.max_block().is_some()); } #[tokio::test] async fn bitfinity_test_headers_client() { - let client = - BitfinityEvmClient::from_rpc_url("https://cloudflare-eth.com", 0, Some(5), 5, 1000, None) - .await - .unwrap(); + let client = BitfinityEvmClient::from_rpc_url( + RpcClientConfig { + primary_url: "https://cloudflare-eth.com".to_string(), + backup_url: None, + max_retries: 3, + retry_delay: Duration::from_secs(1), + }, + 0, + Some(5), + 5, + 1000, + None, + ) + .await + .unwrap(); let headers = client .get_headers_with_priority( HeadersRequest { @@ -507,10 +648,21 @@ mod tests { #[tokio::test] async fn bitfinity_test_bodies_client() { - let client = - BitfinityEvmClient::from_rpc_url("https://cloudflare-eth.com", 0, Some(5), 5, 1000, None) - .await - .unwrap(); + let client = BitfinityEvmClient::from_rpc_url( + RpcClientConfig { + primary_url: "https://cloudflare-eth.com".to_string(), + backup_url: None, + max_retries: 3, + retry_delay: Duration::from_secs(1), + }, + 0, + Some(5), + 5, + 1000, + None, + ) + .await + .unwrap(); let headers = client .get_headers_with_priority( HeadersRequest { diff --git a/crates/node/core/src/args/bitfinity_args.rs b/crates/node/core/src/args/bitfinity_args.rs index af9380dfb38..97489ff823a 100644 --- a/crates/node/core/src/args/bitfinity_args.rs +++ b/crates/node/core/src/args/bitfinity_args.rs @@ -5,18 +5,30 @@ use clap::{arg, Args}; pub const IC_MAINNET_KEY: &str = "308182301d060d2b0601040182dc7c0503010201060c2b0601040182dc7c05030201036100814c0e6ec71fab583b08bd81373c255c3c371b2e84863c98a4f1e08b74235d14fb5d9c0cd546d9685f913a0c0b2cc5341583bf4b4392e467db96d65b9bb4cb717112f8472e0d5a4d14505ffd7484b01291091c5f87b98883463f98091a0baaae"; /// Bitfinity Related Args -#[derive(Debug, Args, PartialEq, Default, Clone)] +#[derive(Debug, Args, PartialEq, Eq, Default, Clone)] #[clap(next_help_heading = "Bitfinity Args")] pub struct BitfinityImportArgs { /// Remote node to connect to #[arg(long, short = 'r', value_name = "BITFINITY_RPC_URL")] pub rpc_url: String, - /// Optional RPC URL where the send_raw_transaction requests are forwarded. + /// Backup node to connect to + #[arg(long, value_name = "BACKUP_RPC_URL")] + pub backup_rpc_url: Option, + + /// Optional RPC URL where the `send_raw_transaction` requests are forwarded. /// If not provided, the RPC URL will be used. #[arg(long)] pub send_raw_transaction_rpc_url: Option, + /// Number of retry attempts before switching to backup URL + #[arg(long, value_name = "MAX_RETRIES", default_value = "3")] + pub max_retries: u32, + + /// Constant delay between retries in seconds + #[arg(long, value_name = "RETRY_DELAY_SECS", default_value = "1")] + pub retry_delay_secs: u64, + /// End Block #[arg(long, short = 'e', value_name = "END_BLOCK")] pub end_block: Option, @@ -32,7 +44,7 @@ pub struct BitfinityImportArgs { pub batch_size: usize, /// Sets the number of block to fetch on each block importer run - /// Default: 10_000 + /// Default: `10_000` #[arg(long, value_name = "MAX_FETCH_BLOCKS", default_value = "10000")] pub max_fetch_blocks: u64, @@ -44,14 +56,12 @@ pub struct BitfinityImportArgs { /// Root key for the IC network #[arg(long, value_name = "IC_ROOT_KEY", default_value = IC_MAINNET_KEY)] pub ic_root_key: String, - } /// Bitfinity Related Args -#[derive(Debug, Args, PartialEq, Default, Clone)] +#[derive(Debug, Args, PartialEq, Eq, Default, Clone)] #[clap(next_help_heading = "Bitfinity Args")] pub struct BitfinityResetEvmStateArgs { - /// Canister principal /// Default value corresponds to testnet #[arg(long, default_value = "4fe7g-7iaaa-aaaak-aegcq-cai")] @@ -65,7 +75,7 @@ pub struct BitfinityResetEvmStateArgs { /// Network url /// This is the URL of the IC network. - /// E.g. + /// E.g. /// - https://ic0.app /// - http://127.0.0.1:3333 #[arg(long)] @@ -79,4 +89,4 @@ pub struct BitfinityResetEvmStateArgs { /// Number of parallel requests to send data to the IC. #[arg(long, default_value = "4")] pub parallel_requests: usize, -} \ No newline at end of file +} From ef16ef472a8c9e96be429a93f797bb821322ca31 Mon Sep 17 00:00:00 2001 From: Yasir Date: Mon, 18 Nov 2024 13:34:35 +0300 Subject: [PATCH 2/2] fix: update chain spec fetching to use fallback URL --- bin/reth/tests/commands/utils.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bin/reth/tests/commands/utils.rs b/bin/reth/tests/commands/utils.rs index 0a6040440e5..b2fa8338932 100644 --- a/bin/reth/tests/commands/utils.rs +++ b/bin/reth/tests/commands/utils.rs @@ -102,8 +102,13 @@ pub async fn bitfinity_import_config_data( backup_evm_datasource_url: Option, data_dir: Option, ) -> eyre::Result<(TempDir, ImportData)> { - let chain = - Arc::new(BitfinityEvmClient::fetch_chain_spec(evm_datasource_url.to_owned()).await?); + let chain = Arc::new( + BitfinityEvmClient::fetch_chain_spec_with_fallback( + evm_datasource_url.to_owned(), + backup_evm_datasource_url.clone(), + ) + .await?, + ); let temp_dir = TempDir::new().unwrap();