From f046a0547006d0cb6f85373b76cf8c8daba4eee4 Mon Sep 17 00:00:00 2001 From: finiteops <119664751+finiteops@users.noreply.github.com> Date: Wed, 26 Feb 2025 16:14:58 +0300 Subject: [PATCH 1/6] LB lagging health check --- crates/rpc/rpc-eth-api/src/core.rs | 33 +++++++++++++++++++ .../src/helpers/bitfinity_evm_rpc.rs | 17 ++++++++++ 2 files changed, 50 insertions(+) diff --git a/crates/rpc/rpc-eth-api/src/core.rs b/crates/rpc/rpc-eth-api/src/core.rs index 2120db192ef..452466f2741 100644 --- a/crates/rpc/rpc-eth-api/src/core.rs +++ b/crates/rpc/rpc-eth-api/src/core.rs @@ -47,10 +47,21 @@ impl FullEthApiServer for T where { } +/// Bitfinity lag API +pub const LAG_STATUS_BAD: &str = "LAGGING"; + +/// Bitfinity lag API +pub const LAG_STATUS_OK: &str = "ACCEPTED_LAG"; + /// Eth rpc interface: #[cfg_attr(not(feature = "client"), rpc(server, namespace = "eth"))] #[cfg_attr(feature = "client", rpc(server, client, namespace = "eth"))] pub trait EthApi { + /// Bitfinity LB api extenstion + /// Handler for: `eth_lagging` + #[method(name = "lagging")] + async fn lagging(&self, accepted_lag: Option) -> RpcResult; + /// Returns the protocol version encoded as a string. #[method(name = "protocolVersion")] async fn protocol_version(&self) -> RpcResult; @@ -386,6 +397,28 @@ where T: FullEthApi, jsonrpsee_types::error::ErrorObject<'static>: From, { + /// Handler for: `eth_lagging` + async fn lagging(&self, accepted_lag: Option) -> RpcResult { + let accepted_lag = match accepted_lag { + Some(lag) => U256::from(lag), + // Assuming that lag behind for 3 block is ok + None => U256::from(3), + }; + + let network_block = BitfinityEvmRpc::network_block_number(self).await?; + let node_block = self.block_number()?; + let lag = network_block.saturating_sub(node_block); + + let status = + if lag > accepted_lag { LAG_STATUS_BAD.to_owned() } else { LAG_STATUS_OK.to_owned() }; + + // Better to repond with string that add structure, import serde and stuff + let response = + format!("{}: lag: {} node: {} network: {}", status, lag, node_block, network_block); + + Ok(response) + } + /// Handler for: `eth_protocolVersion` async fn protocol_version(&self) -> RpcResult { trace!(target: "rpc::eth", "Serving eth_protocolVersion"); diff --git a/crates/rpc/rpc-eth-api/src/helpers/bitfinity_evm_rpc.rs b/crates/rpc/rpc-eth-api/src/helpers/bitfinity_evm_rpc.rs index dd18583d5bf..baacd05ffa6 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/bitfinity_evm_rpc.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/bitfinity_evm_rpc.rs @@ -16,6 +16,23 @@ pub trait BitfinityEvmRpc { /// Returns the `ChainSpec`. fn chain_spec(&self) -> Arc; + /// Returns latest block number at the network/sync source. + fn network_block_number(&self) -> impl Future> + Send { + let chain_spec = self.chain_spec(); + async move { + let (rpc_url, client) = get_client(&chain_spec)?; + + let block_number = client.get_block_number().await.map_err(|e| { + internal_rpc_err(format!( + "failed to forward eth_blockNumber request to {}: {}", + rpc_url, e + )) + })?; + + Ok(U256::from(block_number)) + } + } + /// Forwards `eth_gasPrice` calls to the Bitfinity EVM. fn gas_price(&self) -> impl Future> + Send { let chain_spec = self.chain_spec(); From 6a25ecef696dc7d66e74da87bd7f2863c4e8bed4 Mon Sep 17 00:00:00 2001 From: finiteops <119664751+finiteops@users.noreply.github.com> Date: Wed, 26 Feb 2025 18:50:27 +0300 Subject: [PATCH 2/6] LB check functionality updates --- crates/rpc/rpc-eth-api/src/core.rs | 36 +++++++++++++++++------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/core.rs b/crates/rpc/rpc-eth-api/src/core.rs index 452466f2741..d0e3055453a 100644 --- a/crates/rpc/rpc-eth-api/src/core.rs +++ b/crates/rpc/rpc-eth-api/src/core.rs @@ -47,20 +47,14 @@ impl FullEthApiServer for T where { } -/// Bitfinity lag API -pub const LAG_STATUS_BAD: &str = "LAGGING"; - -/// Bitfinity lag API -pub const LAG_STATUS_OK: &str = "ACCEPTED_LAG"; - /// Eth rpc interface: #[cfg_attr(not(feature = "client"), rpc(server, namespace = "eth"))] #[cfg_attr(feature = "client", rpc(server, client, namespace = "eth"))] pub trait EthApi { - /// Bitfinity LB api extenstion - /// Handler for: `eth_lagging` - #[method(name = "lagging")] - async fn lagging(&self, accepted_lag: Option) -> RpcResult; + /// Bitfinity LB api extension + /// Handler for: `eth_lbLagCheck` + #[method(name = "lbLagCheck")] + async fn lb_lag_check(&self, accepted_lag: Option) -> RpcResult; /// Returns the protocol version encoded as a string. #[method(name = "protocolVersion")] @@ -397,22 +391,32 @@ where T: FullEthApi, jsonrpsee_types::error::ErrorObject<'static>: From, { - /// Handler for: `eth_lagging` - async fn lagging(&self, accepted_lag: Option) -> RpcResult { + /// Handler for: `eth_lbLagCheck` + async fn lb_lag_check(&self, accepted_lag: Option) -> RpcResult { + const LAG_STATUS_BAD: &str = "LAGGING"; + const LAG_STATUS_OK: &str = "ACCEPTED_LAG"; + let network_block = match BitfinityEvmRpc::network_block_number(self).await { + Ok(block) => block, + Err(e) => { + // Must not fail if rpc-url/evmc is not responding + // or it would to break nodes cluster -> LB will stop traffic + tracing::error!(target: "rpc::eth", "Failed to get block number from the network. {}", e); + return Ok(format!("{}: NO_SOURCE", LAG_STATUS_OK)); + } + }; + let accepted_lag = match accepted_lag { Some(lag) => U256::from(lag), // Assuming that lag behind for 3 block is ok None => U256::from(3), }; - let network_block = BitfinityEvmRpc::network_block_number(self).await?; let node_block = self.block_number()?; let lag = network_block.saturating_sub(node_block); - let status = - if lag > accepted_lag { LAG_STATUS_BAD.to_owned() } else { LAG_STATUS_OK.to_owned() }; + let status = if lag > accepted_lag { LAG_STATUS_BAD } else { LAG_STATUS_OK }; - // Better to repond with string that add structure, import serde and stuff + // Better to respond with string that add structure, import serde and stuff let response = format!("{}: lag: {} node: {} network: {}", status, lag, node_block, network_block); From efc61625cf684fec387a11cb196f177bf720ea54 Mon Sep 17 00:00:00 2001 From: finiteops <119664751+finiteops@users.noreply.github.com> Date: Thu, 6 Mar 2025 13:22:40 +0300 Subject: [PATCH 3/6] Updated with tests --- bin/reth/tests/commands/bitfinity_node_it.rs | 51 ++++++++++++++++++++ crates/rpc/rpc-eth-api/src/core.rs | 2 +- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/bin/reth/tests/commands/bitfinity_node_it.rs b/bin/reth/tests/commands/bitfinity_node_it.rs index 62eb65c6a3d..e9760b37753 100644 --- a/bin/reth/tests/commands/bitfinity_node_it.rs +++ b/bin/reth/tests/commands/bitfinity_node_it.rs @@ -49,6 +49,57 @@ async fn bitfinity_test_should_start_local_reth_node() { assert!(reth_client.get_chain_id().await.is_ok()); } +#[tokio::test] +async fn bitfinity_test_lb_lag_check() { + // Arrange + let _log = init_logs(); + + let eth_server = EthImpl::new(); + let (_server, eth_server_address) = + mock_eth_server_start(EthServer::into_rpc(eth_server)).await; + let (reth_client, _reth_node) = + start_reth_node(Some(format!("http://{}", eth_server_address)), None).await; + + // Try `eth_lbLagCheck` + let result: String = reth_client + .single_request( + "eth_lbLagCheck".to_owned(), + ethereum_json_rpc_client::Params::Array(vec![10.into()]), + ethereum_json_rpc_client::Id::Num(1), + ) + .await + .unwrap(); + + assert!(result.contains("ACCEPTED_LAG"), "{result:?}"); + + // Need time to generate extra blocks at `eth_server` + // Assuming it ticks 100ms for the next block + tokio::time::sleep(std::time::Duration::from_millis(1500)).await; + + let result: String = reth_client + .single_request( + "eth_lbLagCheck".to_owned(), + ethereum_json_rpc_client::Params::Array(vec![5.into()]), + ethereum_json_rpc_client::Id::Num(1), + ) + .await + .unwrap(); + + assert!(result.contains("LAGGING"), "{result:?}"); + + // Default accepted lag is 3 so it should be lagging too + let result: String = reth_client + .single_request( + "eth_lbLagCheck".to_owned(), + ethereum_json_rpc_client::Params::None, + ethereum_json_rpc_client::Id::Num(1), + ) + .await + .unwrap(); + + assert!(result.contains("LAGGING"), "{result:?}"); +} + #[tokio::test] async fn bitfinity_test_node_forward_ic_or_eth_get_last_certified_block() { // Arrange diff --git a/crates/rpc/rpc-eth-api/src/core.rs b/crates/rpc/rpc-eth-api/src/core.rs index d0e3055453a..42a30ae165c 100644 --- a/crates/rpc/rpc-eth-api/src/core.rs +++ b/crates/rpc/rpc-eth-api/src/core.rs @@ -399,7 +399,7 @@ where Ok(block) => block, Err(e) => { // Must not fail if rpc-url/evmc is not responding - // or it would to break nodes cluster -> LB will stop traffic + // or it could've disable all nodes under LB tracing::error!(target: "rpc::eth", "Failed to get block number from the network. {}", e); return Ok(format!("{}: NO_SOURCE", LAG_STATUS_OK)); } From 81d8acb9dc9c35c8479249cd3a802000d4220a0b Mon Sep 17 00:00:00 2001 From: finiteops <119664751+finiteops@users.noreply.github.com> Date: Thu, 6 Mar 2025 14:04:10 +0300 Subject: [PATCH 4/6] Added TODO --- crates/rpc/rpc-eth-api/src/helpers/bitfinity_evm_rpc.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/rpc/rpc-eth-api/src/helpers/bitfinity_evm_rpc.rs b/crates/rpc/rpc-eth-api/src/helpers/bitfinity_evm_rpc.rs index baacd05ffa6..123ec13ccd8 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/bitfinity_evm_rpc.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/bitfinity_evm_rpc.rs @@ -20,6 +20,8 @@ pub trait BitfinityEvmRpc { fn network_block_number(&self) -> impl Future> + Send { let chain_spec = self.chain_spec(); async move { + // TODO: Expecting that client node would be the active data sorce at this time + // it could be primary or backup URL let (rpc_url, client) = get_client(&chain_spec)?; let block_number = client.get_block_number().await.map_err(|e| { From 29f77af5d3c4e8a7585cbeaa10346833dc98dd48 Mon Sep 17 00:00:00 2001 From: finiteops <119664751+finiteops@users.noreply.github.com> Date: Thu, 6 Mar 2025 15:34:25 +0300 Subject: [PATCH 5/6] Tests updates --- bin/reth/tests/commands/bitfinity_node_it.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/reth/tests/commands/bitfinity_node_it.rs b/bin/reth/tests/commands/bitfinity_node_it.rs index e9760b37753..4834baf79f3 100644 --- a/bin/reth/tests/commands/bitfinity_node_it.rs +++ b/bin/reth/tests/commands/bitfinity_node_it.rs @@ -73,8 +73,11 @@ async fn bitfinity_test_lb_lag_check() { assert!(result.contains("ACCEPTED_LAG"), "{result:?}"); // Need time to generate extra blocks at `eth_server` - // Assuming it ticks 100ms for the next block - tokio::time::sleep(std::time::Duration::from_millis(1500)).await; + // Assuming `EthImpl` ticks 100ms for the each next block + let mut interval = tokio::time::interval(std::time::Duration::from_millis(100)); + for _ in 0..15 { + interval.tick().await; + } let result: String = reth_client .single_request( From 75fd123684a4c341b792152d4641e21330f6ffb6 Mon Sep 17 00:00:00 2001 From: finiteops <119664751+finiteops@users.noreply.github.com> Date: Thu, 6 Mar 2025 17:48:40 +0300 Subject: [PATCH 6/6] Tests refined --- bin/reth/tests/commands/bitfinity_node_it.rs | 43 ++++++++++++++++---- crates/rpc/rpc-eth-api/src/core.rs | 2 +- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/bin/reth/tests/commands/bitfinity_node_it.rs b/bin/reth/tests/commands/bitfinity_node_it.rs index 4834baf79f3..df8c40d665a 100644 --- a/bin/reth/tests/commands/bitfinity_node_it.rs +++ b/bin/reth/tests/commands/bitfinity_node_it.rs @@ -70,37 +70,62 @@ async fn bitfinity_test_lb_lag_check() { .await .unwrap(); - assert!(result.contains("ACCEPTED_LAG"), "{result:?}"); + assert!(result.contains("ACCEPTABLE_LAG"), "{result:?}"); // Need time to generate extra blocks at `eth_server` // Assuming `EthImpl` ticks 100ms for the each next block - let mut interval = tokio::time::interval(std::time::Duration::from_millis(100)); - for _ in 0..15 { + let mut lag_check_ok = false; + let mut interval = tokio::time::interval(std::time::Duration::from_millis(50)); + for _ in 0..100 { interval.tick().await; + + let result = reth_client + .single_request::( + "eth_lbLagCheck".to_owned(), + ethereum_json_rpc_client::Params::Array(vec![5.into()]), + ethereum_json_rpc_client::Id::Num(1), + ) + .await; + if let Ok(message) = result { + if message.contains("LAGGING") { + lag_check_ok = true; + break; + } + } } + assert!(lag_check_ok); + + // And should not lag with bigger acceptable delta let result: String = reth_client .single_request( "eth_lbLagCheck".to_owned(), - ethereum_json_rpc_client::Params::Array(vec![5.into()]), + ethereum_json_rpc_client::Params::Array(vec![1000.into()]), ethereum_json_rpc_client::Id::Num(1), ) .await .unwrap(); - assert!(result.contains("LAGGING"), "{result:?}"); + assert!(result.contains("ACCEPTABLE_LAG"), "{result:?}"); +} + +#[tokio::test] +async fn bitfinity_test_lb_lag_check_fail_safe() { + let (reth_client, _reth_node) = + start_reth_node(Some("http://local_host:11".to_string()), None).await; - // Default accepted lag is 3 so it should be lagging too - let result: String = reth_client + let message: String = reth_client .single_request( "eth_lbLagCheck".to_owned(), - ethereum_json_rpc_client::Params::None, + ethereum_json_rpc_client::Params::Array(vec![1000.into()]), ethereum_json_rpc_client::Id::Num(1), ) .await .unwrap(); - assert!(result.contains("LAGGING"), "{result:?}"); + // Response should be OK to do not break LB if source temporary not available + assert!(message.contains("ACCEPTABLE_LAG"), "{message}"); + assert!(message.contains("NO_SOURCE"), "{message}"); } #[tokio::test] diff --git a/crates/rpc/rpc-eth-api/src/core.rs b/crates/rpc/rpc-eth-api/src/core.rs index 42a30ae165c..aa4f174f29c 100644 --- a/crates/rpc/rpc-eth-api/src/core.rs +++ b/crates/rpc/rpc-eth-api/src/core.rs @@ -394,7 +394,7 @@ where /// Handler for: `eth_lbLagCheck` async fn lb_lag_check(&self, accepted_lag: Option) -> RpcResult { const LAG_STATUS_BAD: &str = "LAGGING"; - const LAG_STATUS_OK: &str = "ACCEPTED_LAG"; + const LAG_STATUS_OK: &str = "ACCEPTABLE_LAG"; let network_block = match BitfinityEvmRpc::network_block_number(self).await { Ok(block) => block, Err(e) => {