diff --git a/.github/workflows/markdown-links.yml b/.github/workflows/markdown-links.yml index 5b9d5ae..7366b47 100644 --- a/.github/workflows/markdown-links.yml +++ b/.github/workflows/markdown-links.yml @@ -6,7 +6,6 @@ on: push: branches: - main - - dev paths: - '**.md' @@ -28,6 +27,5 @@ jobs: folder-path: "docs" check-modified-files-only: "yes" use-quiet-mode: "yes" - base-branch: "dev" - node-version: ${{ matrix.node-version }} + base-branch: "main" if: env.GIT_DIFF diff --git a/.golangci.yml b/.golangci.yml index ee2cdc4..929947d 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -2,6 +2,7 @@ run: tests: false # timeout for analysis, e.g. 30s, 5m, default is 1m # timeout: 5m + go: "1.21" linters: enable: @@ -22,7 +23,6 @@ linters: - misspell - nakedret - prealloc - - exportloopref - staticcheck - stylecheck - typecheck @@ -31,7 +31,6 @@ linters: - unused - nolintlint - asciicheck - - exportloopref - gofumpt - gomodguard - whitespace @@ -63,7 +62,7 @@ linters-settings: require-explanation: false require-specific: false gofumpt: - lang-version: "1.21" + # lang-version option is deprecated, use global run.go instead gomodguard: blocked: versions: # List of blocked module version constraints diff --git a/deploy/mysql.sql b/deploy/mysql.sql index 6270dea..404be4a 100644 --- a/deploy/mysql.sql +++ b/deploy/mysql.sql @@ -13,138 +13,150 @@ DROP TABLE IF EXISTS `sync_blocks`; CREATE TABLE `sync_blocks` ( `id` bigint NOT NULL AUTO_INCREMENT, - `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `updated_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `blockchain` varchar(32) NOT NULL COMMENT ' chain name', - `miner` varchar(42) NOT NULL COMMENT ' miner', - `block_time` bigint NOT NULL COMMENT ' block_time', - `block_number` bigint NOT NULL COMMENT ' block_number', - `block_hash` varchar(66) NOT NULL COMMENT ' block hash', - `tx_count` bigint NOT NULL COMMENT ' tx count', - `event_count` bigint NOT NULL COMMENT ' event count', - `parent_hash` varchar(66) NOT NULL COMMENT ' parent hash', - `status` varchar(32) NOT NULL COMMENT ' status', - `check_count` bigint NOT NULL COMMENT ' check count', + `created_at` datetime DEFAULT CURRENT_TIMESTAMP, + `updated_at` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `blockchain` varchar(32) NOT NULL, + `miner` varchar(42) NOT NULL, + `block_time` bigint NOT NULL, + `block_number` bigint NOT NULL, + `block_hash` varchar(66) NOT NULL, + `tx_count` bigint NOT NULL, + `event_count` bigint NOT NULL, + `parent_hash` varchar(66) NOT NULL, + `status` varchar(32) NOT NULL, + `check_count` bigint NOT NULL, PRIMARY KEY (`id`), + KEY `tx_count` (`tx_count`), KEY `status_index` (`status`), - KEY `tx_count_index` (`tx_count`), - KEY `check_count_index` (`check_count`) + KEY `check_count` (`check_count`) ) ENGINE = InnoDB - AUTO_INCREMENT = 2923365 - DEFAULT CHARSET = utf8mb4 - COLLATE = utf8mb4_0900_ai_ci; - + AUTO_INCREMENT = 3343515 + DEFAULT CHARSET = utf8mb3; -- ---------------------------- -- Table structure for sync_events -- ---------------------------- DROP TABLE IF EXISTS `sync_events`; CREATE TABLE `sync_events` ( - `id` bigint NOT NULL AUTO_INCREMENT, - `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `updated_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `sync_block_id` bigint NOT NULL COMMENT ' sync_block_id', - `blockchain` varchar(32) NOT NULL COMMENT ' blockchain', - `block_time` bigint NOT NULL COMMENT ' block_time', - `block_number` bigint NOT NULL COMMENT ' block_number', - `block_hash` varchar(66) NOT NULL COMMENT ' block_hash', - `block_log_indexed` bigint NOT NULL COMMENT ' block_log_indexed', - `tx_index` bigint NOT NULL COMMENT ' tx_index', - `tx_hash` varchar(66) NOT NULL COMMENT ' tx_hash', - `event_name` varchar(32) NOT NULL COMMENT ' event_name', - `event_hash` varchar(66) NOT NULL COMMENT ' event_hash', - `contract_address` varchar(42) NOT NULL COMMENT ' contract_address', - `data` json NOT NULL COMMENT ' data', - `status` varchar(32) NOT NULL COMMENT ' status', - `retry_count` bigint DEFAULT '0' COMMENT 'retry_count', + `id` bigint NOT NULL AUTO_INCREMENT, + `created_at` datetime DEFAULT CURRENT_TIMESTAMP, + `updated_at` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `sync_block_id` bigint NOT NULL, + `blockchain` varchar(32) NOT NULL, + `block_time` bigint NOT NULL, + `block_number` bigint NOT NULL, + `block_hash` varchar(66) NOT NULL, + `block_log_indexed` bigint NOT NULL, + `tx_index` bigint NOT NULL, + `tx_hash` varchar(66) NOT NULL, + `event_name` varchar(32) NOT NULL, + `event_hash` varchar(66) NOT NULL, + `contract_address` varchar(42) NOT NULL, + `data` varchar(256) NOT NULL, + `status` varchar(32) NOT NULL, + `retry_count` bigint NOT NULL, PRIMARY KEY (`id`), KEY `status_index` (`status`) ) ENGINE = InnoDB - AUTO_INCREMENT = 1011299 - DEFAULT CHARSET = utf8mb4 - COLLATE = utf8mb4_0900_ai_ci; + AUTO_INCREMENT = 71494 + DEFAULT CHARSET = utf8mb3; -- ---------------------------- -- Table structure for dispute_game -- ---------------------------- -DROP TABLE IF EXISTS dispute_game; -CREATE TABLE IF NOT EXISTS dispute_game +DROP TABLE IF EXISTS `dispute_game`; +CREATE TABLE `dispute_game` ( `id` bigint NOT NULL AUTO_INCREMENT, - `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `updated_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `sync_block_id` bigint NOT NULL COMMENT ' sync_block_id', - `blockchain` varchar(32) NOT NULL COMMENT ' blockchain', - `block_time` bigint NOT NULL COMMENT ' block_time', - `block_number` bigint NOT NULL COMMENT ' block_number', - `block_hash` varchar(66) NOT NULL COMMENT ' block_hash', - `block_log_indexed` bigint NOT NULL COMMENT ' block_log_indexed', - `tx_index` bigint NOT NULL COMMENT ' tx_index', - `tx_hash` varchar(66) NOT NULL COMMENT ' tx_hash', - `event_name` varchar(32) NOT NULL COMMENT ' event_name', - `event_hash` varchar(66) NOT NULL COMMENT ' event_hash', - `contract_address` varchar(42) NOT NULL COMMENT ' contract_address', + `created_at` datetime DEFAULT CURRENT_TIMESTAMP, + `updated_at` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `sync_block_id` bigint NOT NULL, + `blockchain` varchar(32) NOT NULL, + `block_time` bigint NOT NULL, + `block_number` bigint NOT NULL, + `block_hash` varchar(66) NOT NULL, + `block_log_indexed` bigint NOT NULL, + `tx_index` bigint NOT NULL, + `tx_hash` varchar(66) NOT NULL, + `event_name` varchar(32) NOT NULL, + `event_hash` varchar(66) NOT NULL, + `contract_address` varchar(42) NOT NULL, `game_contract` varchar(42) NOT NULL, `game_type` int NOT NULL, `l2_block_number` bigint NOT NULL, - `status` int NOT NULL, - `computed` tinyint(1) NOT NULL DEFAULT 0 COMMENT ' 1-already get game credit 0- not yet', + `status` tinyint NOT NULL, + `computed` tinyint(1) NOT NULL DEFAULT '0', + `calculate_lost` tinyint(1) NOT NULL DEFAULT '0', + `on_chain_status` varchar(32) NOT NULL DEFAULT 'valid', + `claim_data_len` bigint NOT NULL DEFAULT '1', + `get_len_status` tinyint(1) NOT NULL DEFAULT '0', PRIMARY KEY (`id`), KEY `status_index` (`status`), - KEY `dispute_game_index` (`contract_address`, `game_contract`) -); + KEY `dispute_game_index` (`contract_address`, `game_contract`), + KEY `dispute_on_chain_status_index` (`on_chain_status`), + KEY `dispute_claim_data_len_index` (`claim_data_len`) +) ENGINE = InnoDB + AUTO_INCREMENT = 35578 + DEFAULT CHARSET = utf8mb3; -- ---------------------------- -- Table structure for game_claim_data -- ---------------------------- -DROP TABLE IF EXISTS game_claim_data; -CREATE TABLE IF NOT EXISTS game_claim_data +DROP TABLE IF EXISTS `game_claim_data`; +CREATE TABLE `game_claim_data` ( - `id` bigint NOT NULL AUTO_INCREMENT, - `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `updated_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `game_contract` varchar(42) NOT NULL, - `data_index` int NOT NULL, - `parent_index` bigint NOT NULL, - `countered_by` varchar(42) NOT NULL, - `claimant` varchar(64) NOT NULL, - `bond` bigint NOT NULL, - `claim` varchar(64) NOT NULL, - `position` bigint NOT NULL, - `clock` bigint NOT NULL, - `output_block` bigint NOT NULL, - `event_id` bigint NOT NULL, + `id` bigint NOT NULL AUTO_INCREMENT, + `created_at` datetime DEFAULT CURRENT_TIMESTAMP, + `updated_at` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `game_contract` varchar(42) NOT NULL, + `data_index` bigint NOT NULL, + `parent_index` bigint NOT NULL, + `countered_by` varchar(42) NOT NULL, + `claimant` varchar(64) NOT NULL, + `bond` varchar(128) NOT NULL, + `claim` varchar(64) NOT NULL, + `position` varchar(128) NOT NULL, + `clock` varchar(128) NOT NULL, + `output_block` bigint NOT NULL, + `event_id` bigint NOT NULL, + `on_chain_status` varchar(32) NOT NULL DEFAULT 'valid', PRIMARY KEY (`id`), - KEY `credit_index` (`game_contract`, `data_index`) -); + KEY `claim_on_chain_status_index` (`on_chain_status`) +) ENGINE = InnoDB + AUTO_INCREMENT = 36273 + DEFAULT CHARSET = utf8mb3; -- ---------------------------- -- Table structure for game_credit -- ---------------------------- -DROP TABLE IF EXISTS game_credit; -CREATE TABLE IF NOT EXISTS game_credit +DROP TABLE IF EXISTS `game_credit`; +CREATE TABLE `game_credit` ( - `id` bigint NOT NULL AUTO_INCREMENT, - `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `updated_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `game_contract` varchar(42) NOT NULL, - `address` varchar(64) NOT NULL, - `credit` bigint NOT NULL, + `id` bigint NOT NULL AUTO_INCREMENT, + `created_at` datetime DEFAULT CURRENT_TIMESTAMP, + `updated_at` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `game_contract` varchar(42) NOT NULL, + `address` varchar(64) NOT NULL, + `credit` bigint NOT NULL, PRIMARY KEY (`id`) -); +) ENGINE = InnoDB + AUTO_INCREMENT = 36342 + DEFAULT CHARSET = utf8mb3; -- ---------------------------- -- Table structure for game_credit -- ---------------------------- -DROP TABLE IF EXISTS game_lost_bond; -CREATE TABLE IF NOT EXISTS game_lost_bond +DROP TABLE IF EXISTS `game_lost_bonds`; +CREATE TABLE `game_lost_bonds` ( - `id` bigint NOT NULL AUTO_INCREMENT, - `created_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `updated_at` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `game_contract` varchar(42) NOT NULL, - `address` varchar(64) NOT NULL, - `bond` bigint NOT NULL, + `id` bigint NOT NULL AUTO_INCREMENT, + `created_at` datetime DEFAULT CURRENT_TIMESTAMP, + `updated_at` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `game_contract` varchar(42) NOT NULL, + `address` varchar(64) NOT NULL, + `bond` varchar(128) NOT NULL, PRIMARY KEY (`id`) -) +) ENGINE = InnoDB + AUTO_INCREMENT = 14080 + DEFAULT CHARSET = utf8mb3; diff --git a/internal/handler/logFilter.go b/internal/handler/logFilter.go index 36371d0..95c75d4 100644 --- a/internal/handler/logFilter.go +++ b/internal/handler/logFilter.go @@ -46,11 +46,32 @@ func LogsToEvents(ctx *svc.ServiceContext, logs []types.Log, syncBlockID int64) blockTime := blockTimes[cast.ToInt64(vlog.BlockNumber)] if blockTime == 0 { - block, err := ctx.L1RPC.BlockByNumber(context.Background(), big.NewInt(cast.ToInt64(vlog.BlockNumber))) + blockNumber := cast.ToInt64(vlog.BlockNumber) + log.Infof("[LogsToEvents] Fetching block info for block number: %d, txHash: %s", blockNumber, vlog.TxHash.Hex()) + + // Try to get block using L1RPC client first + block, err := ctx.L1RPC.BlockByNumber(context.Background(), big.NewInt(blockNumber)) if err != nil { - return nil, errors.WithStack(err) + log.Errorf("[LogsToEvents] BlockByNumber failed for block %d, txHash: %s, error: %s", blockNumber, vlog.TxHash.Hex(), err.Error()) + + // If error contains "transaction type not supported", try alternative approach + if strings.Contains(err.Error(), "transaction type not supported") { + log.Infof("[LogsToEvents] Attempting to get block timestamp using header only for block %d", blockNumber) + header, headerErr := ctx.L1RPC.HeaderByNumber(context.Background(), big.NewInt(blockNumber)) + if headerErr != nil { + log.Errorf("[LogsToEvents] HeaderByNumber also failed for block %d: %s", blockNumber, headerErr.Error()) + return nil, errors.WithStack(err) + } + blockTime = cast.ToInt64(header.Time) + blockTimes[blockNumber] = blockTime + log.Infof("[LogsToEvents] Successfully got block timestamp %d for block %d using header", blockTime, blockNumber) + } else { + return nil, errors.WithStack(err) + } + } else { + blockTime = cast.ToInt64(block.Time()) + blockTimes[blockNumber] = blockTime } - blockTime = cast.ToInt64(block.Time()) } data, err := Event.Data(vlog) if err != nil { diff --git a/internal/handler/syncBlock.go b/internal/handler/syncBlock.go index 62286d4..84fb64e 100644 --- a/internal/handler/syncBlock.go +++ b/internal/handler/syncBlock.go @@ -34,6 +34,22 @@ func SyncBlock(ctx *svc.ServiceContext) { log.Infof("[Handler.SyncBlock]SyncedBlockHash:%s", ctx.SyncedBlockHash.String()) for { + // Check pending blocks count before syncing new blocks + var pendingCount int64 + err := ctx.DB.Model(&schema.SyncBlock{}).Where("status = ?", schema.BlockPending).Count(&pendingCount).Error + if err != nil { + log.Errorf("[Handler.SyncBlock] Check pending blocks count error: %s\n", errors.WithStack(err)) + time.Sleep(3 * time.Second) + continue + } + + // If pending blocks reach 40, pause block sync to save RPC traffic for other processing + if pendingCount >= 40 { + log.Infof("[Handler.SyncBlock] Pending blocks count (%d) reached limit (40), pausing block sync\n", pendingCount) + time.Sleep(10 * time.Second) + continue + } + syncingBlockNumber := ctx.SyncedBlockNumber + 1 log.Infof("[Handler.SyncBlock] Try to sync block number: %d\n", syncingBlockNumber) diff --git a/internal/handler/syncEvent.go b/internal/handler/syncEvent.go index 6e31c1c..acdfd2f 100644 --- a/internal/handler/syncEvent.go +++ b/internal/handler/syncEvent.go @@ -40,9 +40,10 @@ func SyncEvent(ctx *svc.ServiceContext) { defer _wg.Done() if block.Status == schema.BlockPending { // add events & block.status= valid + log.Infof("[Handler.SyncEvent] Processing pending block %d (hash: %s)", block.BlockNumber, block.BlockHash) err = HandlePendingBlock(ctx, block) if err != nil { - log.Errorf("[Handler.SyncEvent] HandlePendingBlock err: %s\n", errors.WithStack(err)) + log.Errorf("[Handler.SyncEvent] HandlePendingBlock err for block %d (hash: %s): %s\n", block.BlockNumber, block.BlockHash, errors.WithStack(err)) time.Sleep(500 * time.Millisecond) } } else if block.Status == schema.BlockRollback { @@ -63,10 +64,12 @@ func HandlePendingBlock(ctx *svc.ServiceContext, block schema.SyncBlock) error { log.Infof("[Handler.SyncEvent.PendingBlock]Start: %d, %s \n", block.BlockNumber, block.BlockHash) log.Infof("[Handler.SyncEvent.PendingBlock]GetContracts: %v\n", blockchain.GetContracts()) log.Infof("[Handler.SyncEvent.PendingBlock]GetEvents: %v\n", blockchain.GetEvents()) + + log.Infof("[Handler.SyncEvent.PendingBlock] About to call LogFilter for block %d", block.BlockNumber) events, err := LogFilter(ctx, block, blockchain.GetContracts(), [][]common.Hash{blockchain.GetEvents()}) log.Infof("[Handler.SyncEvent.PendingBlock] block %d, events number is %d:", block.BlockNumber, len(events)) if err != nil { - log.Errorf("[Handler.SyncEvent.PendingBlock] Log filter err: %s\n", err) + log.Errorf("[Handler.SyncEvent.PendingBlock] Log filter err for block %d (hash: %s): %s\n", block.BlockNumber, block.BlockHash, err) return errors.WithStack(err) } eventCount := len(events) diff --git a/internal/types/config.go b/internal/types/config.go index c0883dc..dc5249d 100644 --- a/internal/types/config.go +++ b/internal/types/config.go @@ -3,7 +3,7 @@ package types import ( "log" - "github.com/caarlos0/env/v6" + env "github.com/caarlos0/env/v6" ) type Config struct { diff --git a/migration/migrate/migration.go b/migration/migrate/migration.go index fcbdfa6..38be56f 100644 --- a/migration/migrate/migration.go +++ b/migration/migrate/migration.go @@ -3,7 +3,7 @@ package migrate import ( "log" - "github.com/go-gormigrate/gormigrate/v2" + gormigrate "github.com/go-gormigrate/gormigrate/v2" "github.com/optimism-java/dispute-explorer/migration/version" v0 "github.com/optimism-java/dispute-explorer/migration/version/v0" "gorm.io/gorm" diff --git a/migration/version/migration_version.go b/migration/version/migration_version.go index 75d4dfb..89ab971 100644 --- a/migration/version/migration_version.go +++ b/migration/version/migration_version.go @@ -1,7 +1,7 @@ package version import ( - "github.com/go-gormigrate/gormigrate/v2" + gormigrate "github.com/go-gormigrate/gormigrate/v2" v1 "github.com/optimism-java/dispute-explorer/migration/version/v1" v2 "github.com/optimism-java/dispute-explorer/migration/version/v2" v3 "github.com/optimism-java/dispute-explorer/migration/version/v3" diff --git a/migration/version/v1/add_game_lost_bond.go b/migration/version/v1/add_game_lost_bond.go index 485a12b..57db7e3 100644 --- a/migration/version/v1/add_game_lost_bond.go +++ b/migration/version/v1/add_game_lost_bond.go @@ -1,7 +1,7 @@ package v1 import ( - "github.com/go-gormigrate/gormigrate/v2" + gormigrate "github.com/go-gormigrate/gormigrate/v2" "github.com/optimism-java/dispute-explorer/migration" "gorm.io/gorm" ) diff --git a/migration/version/v2/add_calculate_lost_for_dispute_game.go b/migration/version/v2/add_calculate_lost_for_dispute_game.go index ed0e48d..f7de456 100644 --- a/migration/version/v2/add_calculate_lost_for_dispute_game.go +++ b/migration/version/v2/add_calculate_lost_for_dispute_game.go @@ -1,7 +1,7 @@ package v2 import ( - "github.com/go-gormigrate/gormigrate/v2" + gormigrate "github.com/go-gormigrate/gormigrate/v2" "gorm.io/gorm" )