open-web3-stack · qiweiii · Jun 23, 2025 · Jun 23, 2025
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -17,7 +17,6 @@ jobs:
       matrix:
         validator:
           - codec
-          - traces
           - stf/accumulate
           - stf/assurances
           - stf/authorizations

diff --git a/README.md b/README.md
@@ -61,16 +61,3 @@ specific subsystem.
 Validation scripts are included to verify the JSON files against the expected
 ASN.1 syntax provided with the test vectors. These scripts currently rely on my
 [asn1tools](https://github.com/davxy/asn1tools) fork.
-
-## Binary To JSON
-
-The repository binary test vector files can be converted to their JSON
-equivalents for easier inspection and debugging. Use the provided conversion
-script to transform binary files into human-readable JSON format:
-
-```bash
-./scripts/convert-all.sh
-```
-
-This conversion script requires the [jam-types](https://github.com/davxy/jam-types)
-Python library to be installed.
diff --git a/lib/bin_to_json.py b/lib/bin_to_json.py
@@ -31,16 +31,13 @@ def decode(self):
         return decoded
 
 
-def convert_to_json(filename, subsystem_type, spec_name = None):
-    if spec_name in ("tiny", "full"):
-        spec.set_spec(spec_name)
-    print("* Converting:", filename)
+def convert_to_json(filename, subsystem_type):
     with open(filename, 'rb') as file:
         blob = file.read()
         scale_bytes = ScaleBytes(blob)
         dump = subsystem_type(data=scale_bytes)
         decoded = dump.decode()
-        json_filename = str(filename).replace('.bin', '.json')
+        json_filename = filename.replace('.bin', '.json')
         with open(json_filename, 'w') as json_file:
             json.dump(decoded, json_file, indent=4)
             json_file.write('\n')
@@ -50,4 +47,5 @@ def convert_group(group_name, spec_name, subsystem_type):
         spec.set_spec(spec_name)
     print(f"\n[Converting {group_name} ({spec_name})]")
     for file in glob.glob(f"{spec_name}/*.bin"):
+        print("* Converting:", file)
         convert_to_json(file, subsystem_type)
diff --git a/lib/validate_asn1.py b/lib/validate_asn1.py
@@ -76,30 +76,24 @@ def path_to_type_name(path):
     return name
 
 
-def validate(schema, json_file, root_type = None, json_tweaks_callback = None):
+def validate(schema, json_file, json_tweaks_callback=None):
     """Validate a JSON file against an ASN.1 schema.
 
+    The root type for decoding is determined as follows:
+    - If the schema contains "TestCase", use that type
+    - Otherwise, derive the type from the filename (e.g., "my_type.json" → "MyType")
+
     Args:
         schema: Compiled ASN.1 schema
         json_file: Path to JSON file to validate
-        root_type_name: Optional schema type name to be used to parse the json_file
         json_tweaks_callback: Optional callback to modify JSON before validation
-
-    If the `root_type_name` arg is None or is not present in the schema, then the root
-    type for decoding is determined as follows:
-    - If the schema contains "TestCase", use that type
-    - Otherwise, derive the type from the filename (e.g., "my_type.json" → "MyType")
-
     """
-    print("* Validating:", json_file)
-
     # Determine root type used for decoding
-    if root_type is None:
-        if "TestCase" in schema.types:
-            root_type = "TestCase"
-        else:
-            # Auto-detect root type from filename
-            root_type = path_to_type_name(json_file)
+    if "TestCase" in schema.types:
+        root_type = "TestCase"
+    else:
+        # Auto-detect root type from filename
+        root_type = path_to_type_name(json_file)
 
     # Read and prepare JSON
     with open(json_file, "rb") as f:
@@ -127,7 +121,7 @@ def validate_group(group_name, group_schema, spec_name, json_tweaks_callback=Non
     Args:
         group_name: Name of the validation group (for display)
         group_schema: ASN.1 schema file name (or None for base schema only)
-        spec_name: Specification name ("tiny", "full")
+        spec_name: Specification name ("tiny", "full", or "data")
         json_tweaks_callback: Optional callback to modify JSON before validation
     """
     print(f"\n[Validating {group_name} ({spec_name})]")
@@ -140,4 +134,5 @@ def validate_group(group_name, group_schema, spec_name, json_tweaks_callback=Non
     # Compile schema and validate all JSON files
     schema = asn1tools.compile_files(schema_files, codec="jer")
     for json_file in glob.glob(f"{spec_name}/*.json"):
-        validate(schema, json_file, None, json_tweaks_callback)
+        print("* Validating:", json_file)
+        validate(schema, json_file, json_tweaks_callback)
diff --git a/scripts/convert-all.sh b/scripts/convert-all.sh
diff --git a/scripts/validate-all.sh b/scripts/validate-all.sh
diff --git a/stf/accumulate/full/accumulate_ready_queued_reports-1.bin b/stf/accumulate/full/accumulate_ready_queued_reports-1.bin
diff --git a/stf/accumulate/full/accumulate_ready_queued_reports-1.json b/stf/accumulate/full/accumulate_ready_queued_reports-1.json
diff --git a/stf/accumulate/full/enqueue_and_unlock_chain-1.bin b/stf/accumulate/full/enqueue_and_unlock_chain-1.bin
diff --git a/stf/accumulate/full/enqueue_and_unlock_chain-1.json b/stf/accumulate/full/enqueue_and_unlock_chain-1.json
diff --git a/stf/accumulate/full/enqueue_and_unlock_chain-2.bin b/stf/accumulate/full/enqueue_and_unlock_chain-2.bin
diff --git a/stf/accumulate/full/enqueue_and_unlock_chain-2.json b/stf/accumulate/full/enqueue_and_unlock_chain-2.json
diff --git a/stf/accumulate/full/enqueue_and_unlock_chain-3.bin b/stf/accumulate/full/enqueue_and_unlock_chain-3.bin
diff --git a/stf/accumulate/full/enqueue_and_unlock_chain-3.json b/stf/accumulate/full/enqueue_and_unlock_chain-3.json
diff --git a/stf/accumulate/full/enqueue_and_unlock_chain-4.bin b/stf/accumulate/full/enqueue_and_unlock_chain-4.bin
diff --git a/stf/accumulate/full/enqueue_and_unlock_chain-4.json b/stf/accumulate/full/enqueue_and_unlock_chain-4.json
diff --git a/stf/accumulate/full/enqueue_and_unlock_chain_wraps-1.bin b/stf/accumulate/full/enqueue_and_unlock_chain_wraps-1.bin
diff --git a/stf/accumulate/full/enqueue_and_unlock_chain_wraps-1.json b/stf/accumulate/full/enqueue_and_unlock_chain_wraps-1.json
diff --git a/stf/accumulate/full/enqueue_and_unlock_chain_wraps-2.bin b/stf/accumulate/full/enqueue_and_unlock_chain_wraps-2.bin
diff --git a/stf/accumulate/full/enqueue_and_unlock_chain_wraps-2.json b/stf/accumulate/full/enqueue_and_unlock_chain_wraps-2.json
diff --git a/stf/accumulate/full/enqueue_and_unlock_chain_wraps-3.bin b/stf/accumulate/full/enqueue_and_unlock_chain_wraps-3.bin
diff --git a/stf/accumulate/full/enqueue_and_unlock_chain_wraps-3.json b/stf/accumulate/full/enqueue_and_unlock_chain_wraps-3.json
diff --git a/stf/accumulate/full/enqueue_and_unlock_chain_wraps-4.bin b/stf/accumulate/full/enqueue_and_unlock_chain_wraps-4.bin
diff --git a/stf/accumulate/full/enqueue_and_unlock_chain_wraps-4.json b/stf/accumulate/full/enqueue_and_unlock_chain_wraps-4.json
diff --git a/stf/accumulate/full/enqueue_and_unlock_chain_wraps-5.bin b/stf/accumulate/full/enqueue_and_unlock_chain_wraps-5.bin
diff --git a/stf/accumulate/full/enqueue_and_unlock_chain_wraps-5.json b/stf/accumulate/full/enqueue_and_unlock_chain_wraps-5.json
diff --git a/stf/accumulate/full/enqueue_and_unlock_simple-1.bin b/stf/accumulate/full/enqueue_and_unlock_simple-1.bin
diff --git a/stf/accumulate/full/enqueue_and_unlock_simple-1.json b/stf/accumulate/full/enqueue_and_unlock_simple-1.json
diff --git a/stf/accumulate/full/enqueue_and_unlock_simple-2.bin b/stf/accumulate/full/enqueue_and_unlock_simple-2.bin
diff --git a/stf/accumulate/full/enqueue_and_unlock_simple-2.json b/stf/accumulate/full/enqueue_and_unlock_simple-2.json
diff --git a/stf/accumulate/full/enqueue_and_unlock_with_sr_lookup-1.bin b/stf/accumulate/full/enqueue_and_unlock_with_sr_lookup-1.bin
diff --git a/stf/accumulate/full/enqueue_and_unlock_with_sr_lookup-1.json b/stf/accumulate/full/enqueue_and_unlock_with_sr_lookup-1.json
diff --git a/stf/accumulate/full/enqueue_and_unlock_with_sr_lookup-2.bin b/stf/accumulate/full/enqueue_and_unlock_with_sr_lookup-2.bin
diff --git a/stf/accumulate/full/enqueue_and_unlock_with_sr_lookup-2.json b/stf/accumulate/full/enqueue_and_unlock_with_sr_lookup-2.json
diff --git a/stf/accumulate/full/enqueue_self_referential-1.bin b/stf/accumulate/full/enqueue_self_referential-1.bin
diff --git a/stf/accumulate/full/enqueue_self_referential-1.json b/stf/accumulate/full/enqueue_self_referential-1.json
diff --git a/stf/accumulate/full/enqueue_self_referential-2.bin b/stf/accumulate/full/enqueue_self_referential-2.bin
diff --git a/stf/accumulate/full/enqueue_self_referential-2.json b/stf/accumulate/full/enqueue_self_referential-2.json
diff --git a/stf/accumulate/full/enqueue_self_referential-3.bin b/stf/accumulate/full/enqueue_self_referential-3.bin
diff --git a/stf/accumulate/full/enqueue_self_referential-3.json b/stf/accumulate/full/enqueue_self_referential-3.json
diff --git a/stf/accumulate/full/enqueue_self_referential-4.bin b/stf/accumulate/full/enqueue_self_referential-4.bin
diff --git a/stf/accumulate/full/enqueue_self_referential-4.json b/stf/accumulate/full/enqueue_self_referential-4.json
diff --git a/stf/accumulate/full/no_available_reports-1.bin b/stf/accumulate/full/no_available_reports-1.bin
diff --git a/stf/accumulate/full/no_available_reports-1.json b/stf/accumulate/full/no_available_reports-1.json
diff --git a/stf/accumulate/full/process_one_immediate_report-1.bin b/stf/accumulate/full/process_one_immediate_report-1.bin
diff --git a/stf/accumulate/full/process_one_immediate_report-1.json b/stf/accumulate/full/process_one_immediate_report-1.json
diff --git a/stf/accumulate/full/queues_are_shifted-1.bin b/stf/accumulate/full/queues_are_shifted-1.bin
diff --git a/stf/accumulate/full/queues_are_shifted-1.json b/stf/accumulate/full/queues_are_shifted-1.json
diff --git a/stf/accumulate/full/queues_are_shifted-2.bin b/stf/accumulate/full/queues_are_shifted-2.bin
diff --git a/stf/accumulate/full/queues_are_shifted-2.json b/stf/accumulate/full/queues_are_shifted-2.json
diff --git a/stf/accumulate/full/ready_queue_editing-1.bin b/stf/accumulate/full/ready_queue_editing-1.bin
diff --git a/stf/accumulate/full/ready_queue_editing-1.json b/stf/accumulate/full/ready_queue_editing-1.json
diff --git a/stf/accumulate/full/ready_queue_editing-2.bin b/stf/accumulate/full/ready_queue_editing-2.bin
diff --git a/stf/accumulate/full/ready_queue_editing-2.json b/stf/accumulate/full/ready_queue_editing-2.json
diff --git a/stf/accumulate/full/ready_queue_editing-3.bin b/stf/accumulate/full/ready_queue_editing-3.bin
diff --git a/stf/accumulate/full/ready_queue_editing-3.json b/stf/accumulate/full/ready_queue_editing-3.json
diff --git a/stf/accumulate/full/same_code_different_services-1.bin b/stf/accumulate/full/same_code_different_services-1.bin
diff --git a/stf/accumulate/full/same_code_different_services-1.json b/stf/accumulate/full/same_code_different_services-1.json
diff --git a/stf/accumulate/tiny/accumulate_ready_queued_reports-1.bin b/stf/accumulate/tiny/accumulate_ready_queued_reports-1.bin
diff --git a/stf/accumulate/tiny/accumulate_ready_queued_reports-1.json b/stf/accumulate/tiny/accumulate_ready_queued_reports-1.json
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_chain-1.bin b/stf/accumulate/tiny/enqueue_and_unlock_chain-1.bin
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_chain-1.json b/stf/accumulate/tiny/enqueue_and_unlock_chain-1.json
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_chain-2.bin b/stf/accumulate/tiny/enqueue_and_unlock_chain-2.bin
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_chain-2.json b/stf/accumulate/tiny/enqueue_and_unlock_chain-2.json
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_chain-3.bin b/stf/accumulate/tiny/enqueue_and_unlock_chain-3.bin
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_chain-3.json b/stf/accumulate/tiny/enqueue_and_unlock_chain-3.json
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_chain-4.bin b/stf/accumulate/tiny/enqueue_and_unlock_chain-4.bin
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_chain-4.json b/stf/accumulate/tiny/enqueue_and_unlock_chain-4.json
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_chain_wraps-1.bin b/stf/accumulate/tiny/enqueue_and_unlock_chain_wraps-1.bin
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_chain_wraps-1.json b/stf/accumulate/tiny/enqueue_and_unlock_chain_wraps-1.json
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_chain_wraps-2.bin b/stf/accumulate/tiny/enqueue_and_unlock_chain_wraps-2.bin
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_chain_wraps-2.json b/stf/accumulate/tiny/enqueue_and_unlock_chain_wraps-2.json
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_chain_wraps-3.bin b/stf/accumulate/tiny/enqueue_and_unlock_chain_wraps-3.bin
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_chain_wraps-3.json b/stf/accumulate/tiny/enqueue_and_unlock_chain_wraps-3.json
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_chain_wraps-4.bin b/stf/accumulate/tiny/enqueue_and_unlock_chain_wraps-4.bin
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_chain_wraps-4.json b/stf/accumulate/tiny/enqueue_and_unlock_chain_wraps-4.json
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_chain_wraps-5.bin b/stf/accumulate/tiny/enqueue_and_unlock_chain_wraps-5.bin
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_chain_wraps-5.json b/stf/accumulate/tiny/enqueue_and_unlock_chain_wraps-5.json
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_simple-1.bin b/stf/accumulate/tiny/enqueue_and_unlock_simple-1.bin
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_simple-1.json b/stf/accumulate/tiny/enqueue_and_unlock_simple-1.json
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_simple-2.bin b/stf/accumulate/tiny/enqueue_and_unlock_simple-2.bin
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_simple-2.json b/stf/accumulate/tiny/enqueue_and_unlock_simple-2.json
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_with_sr_lookup-1.bin b/stf/accumulate/tiny/enqueue_and_unlock_with_sr_lookup-1.bin
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_with_sr_lookup-1.json b/stf/accumulate/tiny/enqueue_and_unlock_with_sr_lookup-1.json
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_with_sr_lookup-2.bin b/stf/accumulate/tiny/enqueue_and_unlock_with_sr_lookup-2.bin
diff --git a/stf/accumulate/tiny/enqueue_and_unlock_with_sr_lookup-2.json b/stf/accumulate/tiny/enqueue_and_unlock_with_sr_lookup-2.json
diff --git a/stf/accumulate/tiny/enqueue_self_referential-1.bin b/stf/accumulate/tiny/enqueue_self_referential-1.bin
diff --git a/stf/accumulate/tiny/enqueue_self_referential-1.json b/stf/accumulate/tiny/enqueue_self_referential-1.json
diff --git a/stf/accumulate/tiny/enqueue_self_referential-2.bin b/stf/accumulate/tiny/enqueue_self_referential-2.bin
diff --git a/stf/accumulate/tiny/enqueue_self_referential-2.json b/stf/accumulate/tiny/enqueue_self_referential-2.json
diff --git a/stf/accumulate/tiny/enqueue_self_referential-3.bin b/stf/accumulate/tiny/enqueue_self_referential-3.bin
diff --git a/stf/accumulate/tiny/enqueue_self_referential-3.json b/stf/accumulate/tiny/enqueue_self_referential-3.json
diff --git a/stf/accumulate/tiny/enqueue_self_referential-4.bin b/stf/accumulate/tiny/enqueue_self_referential-4.bin
diff --git a/stf/accumulate/tiny/enqueue_self_referential-4.json b/stf/accumulate/tiny/enqueue_self_referential-4.json
diff --git a/stf/accumulate/tiny/no_available_reports-1.bin b/stf/accumulate/tiny/no_available_reports-1.bin
diff --git a/stf/accumulate/tiny/no_available_reports-1.json b/stf/accumulate/tiny/no_available_reports-1.json
diff --git a/stf/accumulate/tiny/process_one_immediate_report-1.bin b/stf/accumulate/tiny/process_one_immediate_report-1.bin
diff --git a/stf/accumulate/tiny/process_one_immediate_report-1.json b/stf/accumulate/tiny/process_one_immediate_report-1.json
diff --git a/stf/accumulate/tiny/queues_are_shifted-1.bin b/stf/accumulate/tiny/queues_are_shifted-1.bin
diff --git a/stf/accumulate/tiny/queues_are_shifted-1.json b/stf/accumulate/tiny/queues_are_shifted-1.json
diff --git a/stf/accumulate/tiny/queues_are_shifted-2.bin b/stf/accumulate/tiny/queues_are_shifted-2.bin
diff --git a/stf/accumulate/tiny/queues_are_shifted-2.json b/stf/accumulate/tiny/queues_are_shifted-2.json
diff --git a/stf/accumulate/tiny/ready_queue_editing-1.bin b/stf/accumulate/tiny/ready_queue_editing-1.bin
diff --git a/stf/accumulate/tiny/ready_queue_editing-1.json b/stf/accumulate/tiny/ready_queue_editing-1.json
diff --git a/stf/accumulate/tiny/ready_queue_editing-2.bin b/stf/accumulate/tiny/ready_queue_editing-2.bin
diff --git a/stf/accumulate/tiny/ready_queue_editing-2.json b/stf/accumulate/tiny/ready_queue_editing-2.json
diff --git a/stf/accumulate/tiny/ready_queue_editing-3.bin b/stf/accumulate/tiny/ready_queue_editing-3.bin
diff --git a/stf/accumulate/tiny/ready_queue_editing-3.json b/stf/accumulate/tiny/ready_queue_editing-3.json
diff --git a/stf/accumulate/tiny/same_code_different_services-1.bin b/stf/accumulate/tiny/same_code_different_services-1.bin
diff --git a/stf/accumulate/tiny/same_code_different_services-1.json b/stf/accumulate/tiny/same_code_different_services-1.json
diff --git a/stf/history/README.md b/stf/history/README.md
@@ -25,11 +25,11 @@ the input `accumulate_root`.
 
 ## Vectors
 
-- [progress_blocks_history-1](tiny/progress_blocks_history-1.json) 🟢 
+- [progress_blocks_history-1](data/progress_blocks_history-1.json) 🟢 
   - Empty history queue.
-- [progress_blocks_history-2](tiny/progress_blocks_history-2.json) 🟢 
+- [progress_blocks_history-2](data/progress_blocks_history-2.json) 🟢 
   - Not empty nor full history queue.
-- [progress_blocks_history-3](tiny/progress_blocks_history-3.json) 🟢 
+- [progress_blocks_history-3](data/progress_blocks_history-3.json) 🟢 
   - Fill the history queue.
-- [progress_blocks_history-4](tiny/progress_blocks_history-4.json) 🟢 
+- [progress_blocks_history-4](data/progress_blocks_history-4.json) 🟢 
   - Shift the history queue.
diff --git a/stf/history/convert.py b/stf/history/convert.py
@@ -11,5 +11,4 @@
 
 os.chdir(script_dir)
 
-for spec in ["tiny", "full"]:
-    convert_group("history", spec, HistoryTestVector)
+convert_group("history", "data", HistoryTestVector)
diff --git a/...istory/full/progress_blocks_history-1.bin → ...istory/data/progress_blocks_history-1.bin b/...istory/full/progress_blocks_history-1.bin → ...istory/data/progress_blocks_history-1.bin
diff --git a/...story/full/progress_blocks_history-1.json → ...story/data/progress_blocks_history-1.json b/...story/full/progress_blocks_history-1.json → ...story/data/progress_blocks_history-1.json
diff --git a/...istory/full/progress_blocks_history-2.bin → ...istory/data/progress_blocks_history-2.bin b/...istory/full/progress_blocks_history-2.bin → ...istory/data/progress_blocks_history-2.bin
diff --git a/...story/full/progress_blocks_history-2.json → ...story/data/progress_blocks_history-2.json b/...story/full/progress_blocks_history-2.json → ...story/data/progress_blocks_history-2.json
diff --git a/...istory/full/progress_blocks_history-3.bin → ...istory/data/progress_blocks_history-3.bin b/...istory/full/progress_blocks_history-3.bin → ...istory/data/progress_blocks_history-3.bin
diff --git a/...story/full/progress_blocks_history-3.json → ...story/data/progress_blocks_history-3.json b/...story/full/progress_blocks_history-3.json → ...story/data/progress_blocks_history-3.json
diff --git a/...istory/full/progress_blocks_history-4.bin → ...istory/data/progress_blocks_history-4.bin b/...istory/full/progress_blocks_history-4.bin → ...istory/data/progress_blocks_history-4.bin
diff --git a/...story/full/progress_blocks_history-4.json → ...story/data/progress_blocks_history-4.json b/...story/full/progress_blocks_history-4.json → ...story/data/progress_blocks_history-4.json
diff --git a/stf/history/tiny/progress_blocks_history-1.bin b/stf/history/tiny/progress_blocks_history-1.bin
diff --git a/stf/history/tiny/progress_blocks_history-1.json b/stf/history/tiny/progress_blocks_history-1.json
diff --git a/stf/history/tiny/progress_blocks_history-2.bin b/stf/history/tiny/progress_blocks_history-2.bin
diff --git a/stf/history/tiny/progress_blocks_history-2.json b/stf/history/tiny/progress_blocks_history-2.json
diff --git a/stf/history/tiny/progress_blocks_history-3.bin b/stf/history/tiny/progress_blocks_history-3.bin