Add MRT record debugging and raw byte export

CHANGELOG.md

@@ -4,8 +4,24 @@ All notable changes to this project will be documented in this file.
 
 ## Unreleased
 
+### Breaking changes
+
+* **`RawMrtRecord` field renamed**: `raw_bytes` is now `message_bytes` to clarify it contains only the message body
+  - Added new `header_bytes` field containing the raw header bytes as read from the wire
+  - The `raw_bytes()` method now returns complete MRT record bytes (header + body) without re-encoding
+
 ### New features
 
+* Add MRT record debugging and raw bytes export capabilities
+  - `RawMrtRecord` now stores both `header_bytes` and `message_bytes` to enable exact byte-for-byte export without re-encoding
+  - New methods on `RawMrtRecord`: `raw_bytes()`, `write_raw_bytes()`, `append_raw_bytes()`, `total_bytes_len()`
+  - Added `Display` implementations for `MrtRecord`, `CommonHeader`, and `MrtMessage` for human-readable debug output
+  - New examples: `mrt_debug.rs` and `extract_problematic_records.rs` demonstrating debug features
+* CLI now supports record-level output and multiple output formats
+  - New `--level` (`-L`) option: `elems` (default) or `records` to control output granularity
+  - New `--format` (`-F`) option: `default`, `json`, `json-pretty`, or `psv`
+  - Record-level output with `--level records` outputs MRT records instead of per-prefix elements
+  - JSON output for records provides full structured data for debugging
 * Add `UpdateIterator` and `FallibleUpdateIterator` for iterating over BGP announcements ([#250](https://github.com/bgpkit/bgpkit-parser/issues/250))
   - New `MrtUpdate` enum supporting both BGP4MP UPDATE messages and TableDumpV2 RIB entries
   - `Bgp4MpUpdate` struct for BGP4MP UPDATE messages with metadata (timestamp, peer_ip, peer_asn)

examples/README.md

@@ -38,5 +38,9 @@ This directory contains runnable examples for bgpkit_parser. They demonstrate ba
 ## Error Handling and Robustness
 - [fallible_parsing.rs](fallible_parsing.rs) — Demonstrate fallible record/element iterators that let you handle parse errors explicitly while continuing to process.
 
+## Debugging and Analysis
+- [mrt_debug.rs](mrt_debug.rs) — Demonstrate MRT debugging features: debug display for MRT records, raw byte export, and the new `Display` implementation.
+- [extract_problematic_records.rs](extract_problematic_records.rs) — Find and export MRT records that fail to parse for further analysis with other tools.
+
 ## Local-only and Misc
 - [local_only/src/main.rs](local_only/src/main.rs) — Minimal example that reads a local updates.bz2 file; intended for local experimentation (not network fetching).

examples/extract_problematic_records.rs

@@ -0,0 +1,116 @@
+//! Example demonstrating how to find and extract problematic MRT records.
+//!
+//! This example shows how to:
+//! 1. Iterate over raw MRT records
+//! 2. Attempt to parse each record
+//! 3. Export records that fail to parse for debugging
+//!
+//! This is useful for identifying malformed or unusual MRT records that
+//! cause parsing issues, allowing you to analyze them with other tools
+//! or report them for investigation.
+//!
+//! Run with: cargo run --example extract_problematic_records -- <mrt_file> [output_file]
+
+use bgpkit_parser::BgpkitParser;
+use std::env;
+
+fn main() {
+    let args: Vec<String> = env::args().collect();
+
+    if args.len() < 2 {
+        eprintln!("Usage: {} <mrt_file> [output_file]", args[0]);
+        eprintln!();
+        eprintln!("Arguments:");
+        eprintln!("  mrt_file    - Path or URL to the MRT file to analyze");
+        eprintln!("  output_file - Optional path to export problematic records (default: problematic_records.mrt)");
+        eprintln!();
+        eprintln!("Example:");
+        eprintln!(
+            "  {} https://data.ris.ripe.net/rrc00/latest-update.gz",
+            args[0]
+        );
+        std::process::exit(1);
+    }
+
+    let input_file = &args[1];
+    let output_file = args
+        .get(2)
+        .map(|s| s.as_str())
+        .unwrap_or("problematic_records.mrt");
+
+    println!("Analyzing MRT file: {}", input_file);
+    println!("Problematic records will be saved to: {}", output_file);
+    println!();
+
+    let parser = match BgpkitParser::new(input_file) {
+        Ok(p) => p,
+        Err(e) => {
+            eprintln!("Failed to open MRT file: {}", e);
+            std::process::exit(1);
+        }
+    };
+
+    let mut total_records = 0;
+    let mut parsed_ok = 0;
+    let mut parse_errors = 0;
+    let mut export_errors = 0;
+
+    for raw_record in parser.into_raw_record_iter() {
+        total_records += 1;
+
+        // Try to parse the record
+        match raw_record.clone().parse() {
+            Ok(_parsed) => {
+                parsed_ok += 1;
+            }
+            Err(e) => {
+                parse_errors += 1;
+                println!(
+                    "Record #{}: Parse error at timestamp {}",
+                    total_records, raw_record.common_header.timestamp
+                );
+                println!("  Error: {}", e);
+                println!("  Header: {}", raw_record.common_header);
+                println!("  Size: {} bytes", raw_record.total_bytes_len());
+
+                // Export the problematic record
+                if let Err(write_err) = raw_record.append_raw_bytes(output_file) {
+                    eprintln!("  Failed to export record: {}", write_err);
+                    export_errors += 1;
+                } else {
+                    println!("  -> Exported to {}", output_file);
+                }
+                println!();
+            }
+        }
+
+        // Progress indicator every 100,000 records
+        if total_records % 100_000 == 0 {
+            eprintln!(
+                "Progress: {} records processed ({} errors so far)",
+                total_records, parse_errors
+            );
+        }
+    }
+
+    println!("=== Summary ===");
+    println!("Total records processed: {}", total_records);
+    println!("Successfully parsed:     {}", parsed_ok);
+    println!("Parse errors:            {}", parse_errors);
+    if export_errors > 0 {
+        println!("Export errors:           {}", export_errors);
+    }
+
+    if parse_errors > 0 {
+        println!();
+        println!("Problematic records exported to: {}", output_file);
+        println!();
+        println!("You can analyze the exported records with:");
+        println!("  - bgpdump -m {}", output_file);
+        println!("  - This parser with verbose debugging");
+        println!("  - Hex editor for raw byte analysis");
+    } else {
+        println!();
+        println!("No problematic records found!");
+    }
+}

examples/mrt_debug.rs

@@ -0,0 +1,114 @@
+//! Example demonstrating MRT debug features.
+//!
+//! This example shows how to:
+//! 1. Display MRT records and BGP elements in JSON format for debugging
+//! 2. Display MRT records in a debug-friendly format
+//! 3. Export raw MRT record bytes to files for debugging
+//!
+//! Run with: cargo run --example mrt_debug --features serde
+
+use bgpkit_parser::BgpkitParser;
+
+fn main() {
+    let url = "https://spaces.bgpkit.org/parser/update-example.gz";
+    println!("Parsing: {}\n", url);
+
+    println!("=== MRT Record JSON Format Examples ===\n");
+
+    // Show first 3 MRT records in JSON format
+    for (idx, record) in BgpkitParser::new(url)
+        .unwrap()
+        .into_record_iter()
+        .take(3)
+        .enumerate()
+    {
+        println!("[Record {}]", idx + 1);
+        #[cfg(feature = "serde")]
+        println!("{}", serde_json::to_string_pretty(&record).unwrap());
+        #[cfg(not(feature = "serde"))]
+        println!("{:?}", record);
+        println!();
+    }
+
+    println!("=== BGP Element JSON Format Examples ===\n");
+
+    // Show first 5 elements in JSON format
+    for (idx, elem) in BgpkitParser::new(url)
+        .unwrap()
+        .into_iter()
+        .take(5)
+        .enumerate()
+    {
+        println!("[Element {}]", idx + 1);
+        #[cfg(feature = "serde")]
+        println!("{}", serde_json::to_string(&elem).unwrap());
+        #[cfg(not(feature = "serde"))]
+        println!("{:?}", elem);
+    }
+
+    println!("\n=== MRT Record Debug Display ===\n");
+
+    // Show first 5 MRT records with debug display
+    let parser = BgpkitParser::new(url).unwrap();
+    for (idx, record) in parser.into_record_iter().take(5).enumerate() {
+        println!("[{}] {}", idx + 1, record);
+    }
+
+    println!("\n=== Raw MRT Record Iteration ===\n");
+
+    // Demonstrate raw record iteration and byte export
+    let parser = BgpkitParser::new(url).unwrap();
+    for (idx, raw_record) in parser.into_raw_record_iter().take(3).enumerate() {
+        println!("[{}] Raw Record:", idx + 1);
+        println!("    Header: {}", raw_record.common_header);
+        println!("    Total bytes: {} bytes", raw_record.total_bytes_len());
+        println!("    Header size: {} bytes", raw_record.header_bytes.len());
+        println!(
+            "    Message body size: {} bytes",
+            raw_record.message_bytes.len()
+        );
+
+        // Demonstrate parsing the raw record
+        match raw_record.clone().parse() {
+            Ok(parsed) => {
+                println!("    Parsed: {}", parsed);
+            }
+            Err(e) => {
+                println!("    Parse error: {}", e);
+                // In case of error, you could export the problematic record:
+                // raw_record.write_raw_bytes(format!("problematic_{}.mrt", idx)).unwrap();
+            }
+        }
+        println!();
+    }
+
+    println!("=== Exporting Raw Bytes Example ===\n");
+
+    // Export a few records to demonstrate the functionality
+    let parser = BgpkitParser::new(url).unwrap();
+    let output_file = "/tmp/debug_records.mrt";
+
+    let mut count = 0;
+    for raw_record in parser.into_raw_record_iter().take(10) {
+        // Append each record to the same file
+        if let Err(e) = raw_record.append_raw_bytes(output_file) {
+            eprintln!("Failed to write record: {}", e);
+        }
+        count += 1;
+    }
+
+    println!("Exported {} records to {}", count, output_file);
+
+    // Verify by reading back
+    let verify_parser = BgpkitParser::new(output_file).unwrap();
+    let verify_count = verify_parser.into_record_iter().count();
+    println!(
+        "Verification: read back {} records from exported file",
+        verify_count
+    );
+
+    // Clean up
+    let _ = std::fs::remove_file(output_file);
+
+    println!("\nDone!");
+}