Update JSON export tool to support binary meta cache files (#20096)

The test case needs to filter out quite a few things that are
unpredictable.

---------

Co-authored-by: Ivan Levkivskyi <levkivskyi@gmail.com>

Author: JukkaL

mypy/exportjson.py

@@ -1,4 +1,4 @@
-"""Tool to convert mypy cache file to a JSON format (print to stdout).
+"""Tool to convert binary mypy cache files (.ff) to JSON (.ff.json).
 
 Usage:
    python -m mypy.exportjson .mypy_cache/.../my_module.data.ff
@@ -18,8 +18,9 @@
 import sys
 from typing import Any, TypeAlias as _TypeAlias
 
-from librt.internal import ReadBuffer
+from librt.internal import ReadBuffer, cache_version
 
+from mypy.cache import CACHE_VERSION, CacheMeta
 from mypy.nodes import (
     FUNCBASE_FLAGS,
     FUNCDEF_FLAGS,
@@ -551,6 +552,33 @@ def convert_unbound_type(self: UnboundType) -> Json:
     }
 
 
+def convert_binary_cache_meta_to_json(data: bytes, data_file: str) -> Json:
+    assert (
+        data[0] == cache_version() and data[1] == CACHE_VERSION
+    ), "Cache file created by an incompatible mypy version"
+    meta = CacheMeta.read(ReadBuffer(data[2:]), data_file)
+    assert meta is not None, f"Error reading meta cache file associated with {data_file}"
+    return {
+        "id": meta.id,
+        "path": meta.path,
+        "mtime": meta.mtime,
+        "size": meta.size,
+        "hash": meta.hash,
+        "data_mtime": meta.data_mtime,
+        "dependencies": meta.dependencies,
+        "suppressed": meta.suppressed,
+        "options": meta.options,
+        "dep_prios": meta.dep_prios,
+        "dep_lines": meta.dep_lines,
+        "dep_hashes": [dep.hex() for dep in meta.dep_hashes],
+        "interface_hash": meta.interface_hash.hex(),
+        "error_lines": meta.error_lines,
+        "version_id": meta.version_id,
+        "ignore_all": meta.ignore_all,
+        "plugin_data": meta.plugin_data,
+    }
+
+
 def main() -> None:
     parser = argparse.ArgumentParser(
         description="Convert binary cache files to JSON. "
@@ -562,11 +590,19 @@ def main() -> None:
     args = parser.parse_args()
     fnams: list[str] = args.path
     for fnam in fnams:
-        if not fnam.endswith(".data.ff"):
-            sys.exit(f"error: Expected .data.ff extension, but got {fnam}")
+        if fnam.endswith(".data.ff"):
+            is_data = True
+        elif fnam.endswith(".meta.ff"):
+            is_data = False
+        else:
+            sys.exit(f"error: Expected .data.ff or .meta.ff extension, but got {fnam}")
         with open(fnam, "rb") as f:
             data = f.read()
-        json_data = convert_binary_cache_to_json(data)
+        if is_data:
+            json_data = convert_binary_cache_to_json(data)
+        else:
+            data_file = fnam.removesuffix(".meta.ff") + ".data.ff"
+            json_data = convert_binary_cache_meta_to_json(data, data_file)
         new_fnam = fnam + ".json"
         with open(new_fnam, "w") as f:
             json.dump(json_data, f)

mypy/test/testexportjson.py

@@ -9,7 +9,7 @@
 
 from mypy import build
 from mypy.errors import CompileError
-from mypy.exportjson import convert_binary_cache_to_json
+from mypy.exportjson import convert_binary_cache_meta_to_json, convert_binary_cache_to_json
 from mypy.modulefinder import BuildSource
 from mypy.options import Options
 from mypy.test.config import test_temp_dir
@@ -24,6 +24,7 @@ class TypeExportSuite(DataSuite):
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         error = False
         src = "\n".join(testcase.input)
+        is_meta = testcase.name.endswith("_meta")
         try:
             options = Options()
             options.use_builtins_fixtures = True
@@ -50,21 +51,46 @@ def run_case(self, testcase: DataDrivenTestCase) -> None:
                     "__future__",
                     "typing_extensions",
                     "sys",
+                    "collections",
                 ):
                     continue
                 fnam = os.path.join(cache_dir, f"{module}.data.ff")
-                with open(fnam, "rb") as f:
-                    json_data = convert_binary_cache_to_json(f.read(), implicit_names=False)
+                if not is_meta:
+                    with open(fnam, "rb") as f:
+                        json_data = convert_binary_cache_to_json(f.read(), implicit_names=False)
+                else:
+                    meta_fnam = os.path.join(cache_dir, f"{module}.meta.ff")
+                    with open(meta_fnam, "rb") as f:
+                        json_data = convert_binary_cache_meta_to_json(f.read(), fnam)
                 for line in json.dumps(json_data, indent=4).splitlines():
                     if '"path": ' in line:
-                        # We source file path is unpredictable, so filter it out
+                        # The source file path is unpredictable, so filter it out
                         line = re.sub(r'"[^"]+\.pyi?"', "...", line)
+                    if is_meta:
+                        if '"version_id"' in line:
+                            line = re.sub(r'"[0-9][^"]+"', "...", line)
+                        if '"mtime"' in line or '"data_mtime"' in line:
+                            line = re.sub(r": [0-9]+", ": ...", line)
+                        if '"platform"' in line:
+                            line = re.sub(': "[^"]+"', ": ...", line)
+                        if '"hash"' not in line:
+                            # Some hashes are unpredictable so filter them out
+                            line = re.sub(r'"[a-f0-9]{40}"', '"<hash>"', line)
                     assert "ERROR" not in line, line
                     a.append(line)
         except CompileError as e:
             a = e.messages
             error = True
         if error or "\n".join(testcase.output).strip() != "<not checked>":
+            if is_meta and sys.platform == "win32":
+                out = filter_platform_specific(testcase.output)
+                a = filter_platform_specific(a)
+            else:
+                out = testcase.output
             assert_string_arrays_equal(
-                testcase.output, a, f"Invalid output ({testcase.file}, line {testcase.line})"
+                out, a, f"Invalid output ({testcase.file}, line {testcase.line})"
             )
+
+
+def filter_platform_specific(lines: list[str]) -> list[str]:
+    return [l for l in lines if '"size":' not in l and '"hash":' not in l]

test-data/unit/exportjson.test

@@ -278,3 +278,47 @@ x: X = 2
 [builtins fixtures/tuple.pyi]
 [out]
 <not checked>
+
+[case testExportMetaBasic_meta]
+import typing
+from typing_extensions import Final
+[builtins fixtures/tuple.pyi]
+[out]
+{
+    "id": "main",
+    "path": ...,
+    "mtime": ...,
+    "size": 49,
+    "hash": "db2252f953c889e6b78dde8e30bd241a0c86b2d9",
+    "data_mtime": ...,
+    "dependencies": [
+        "typing",
+        "typing_extensions",
+        "builtins"
+    ],
+    "suppressed": [],
+    "options": {
+        "other_options": "<hash>",
+        "platform": ...
+    },
+    "dep_prios": [
+        10,
+        5,
+        5
+    ],
+    "dep_lines": [
+        1,
+        2,
+        1
+    ],
+    "dep_hashes": [
+        "<hash>",
+        "<hash>",
+        "<hash>"
+    ],
+    "interface_hash": "<hash>",
+    "error_lines": [],
+    "version_id": ...,
+    "ignore_all": false,
+    "plugin_data": null
+}