From 3c9bd6a981c11488a082378d22a70a293f5ca3fe Mon Sep 17 00:00:00 2001 From: Jukka Lehtosalo Date: Tue, 21 Oct 2025 11:06:34 +0100 Subject: [PATCH 1/4] Update JSON export tool to support binary meta cache files --- mypy/exportjson.py | 41 +++++++++++++++++++++++++++---- mypy/test/testexportjson.py | 24 +++++++++++++++---- test-data/unit/exportjson.test | 44 ++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 8 deletions(-) diff --git a/mypy/exportjson.py b/mypy/exportjson.py index 09945f0ef28f0..04bb1f2ec1372 100644 --- a/mypy/exportjson.py +++ b/mypy/exportjson.py @@ -1,4 +1,4 @@ -"""Tool to convert mypy cache file to a JSON format (print to stdout). +"""Tool to convert binary mypy cache files (.ff) to JSON (.ff.json). Usage: python -m mypy.exportjson .mypy_cache/.../my_module.data.ff @@ -21,6 +21,7 @@ from librt.internal import Buffer +from mypy.cache import CacheMeta from mypy.nodes import ( FUNCBASE_FLAGS, FUNCDEF_FLAGS, @@ -552,6 +553,30 @@ def convert_unbound_type(self: UnboundType) -> Json: } +def convert_binary_cache_meta_to_json(data: bytes, data_file: str) -> Json: + meta = CacheMeta.read(Buffer(data), data_file) + assert meta is not None, f"Error reading meta cache file associated with {data_file}" + return { + "id": meta.id, + "path": meta.path, + "mtime": meta.mtime, + "size": meta.size, + "hash": meta.hash, + "data_mtime": meta.data_mtime, + "dependencies": meta.dependencies, + "suppressed": meta.suppressed, + "options": meta.options, + "dep_prios": meta.dep_prios, + "dep_lines": meta.dep_lines, + "dep_hashes": [dep.hex() for dep in meta.dep_hashes], + "interface_hash": meta.interface_hash.hex(), + "error_lines": meta.error_lines, + "version_id": meta.version_id, + "ignore_all": meta.ignore_all, + "plugin_data": meta.plugin_data, + } + + def main() -> None: parser = argparse.ArgumentParser( description="Convert binary cache files to JSON. " @@ -563,11 +588,19 @@ def main() -> None: args = parser.parse_args() fnams: list[str] = args.path for fnam in fnams: - if not fnam.endswith(".data.ff"): - sys.exit(f"error: Expected .data.ff extension, but got {fnam}") + if fnam.endswith(".data.ff"): + is_data = True + elif fnam.endswith(".meta.ff"): + is_data = False + else: + sys.exit(f"error: Expected .data.ff or .meta.ff extension, but got {fnam}") with open(fnam, "rb") as f: data = f.read() - json_data = convert_binary_cache_to_json(data) + if is_data: + json_data = convert_binary_cache_to_json(data) + else: + data_file = fnam.removesuffix(".meta.ff") + ".data.ff" + json_data = convert_binary_cache_meta_to_json(data, data_file) new_fnam = fnam + ".json" with open(new_fnam, "w") as f: json.dump(json_data, f) diff --git a/mypy/test/testexportjson.py b/mypy/test/testexportjson.py index 13bd96d066427..908df9a16e16c 100644 --- a/mypy/test/testexportjson.py +++ b/mypy/test/testexportjson.py @@ -9,7 +9,7 @@ from mypy import build from mypy.errors import CompileError -from mypy.exportjson import convert_binary_cache_to_json +from mypy.exportjson import convert_binary_cache_meta_to_json, convert_binary_cache_to_json from mypy.modulefinder import BuildSource from mypy.options import Options from mypy.test.config import test_temp_dir @@ -53,12 +53,28 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: ): continue fnam = os.path.join(cache_dir, f"{module}.data.ff") - with open(fnam, "rb") as f: - json_data = convert_binary_cache_to_json(f.read(), implicit_names=False) + is_meta = testcase.name.endswith("_meta") + if not is_meta: + with open(fnam, "rb") as f: + json_data = convert_binary_cache_to_json(f.read(), implicit_names=False) + else: + meta_fnam = os.path.join(cache_dir, f"{module}.meta.ff") + with open(meta_fnam, "rb") as f: + json_data = convert_binary_cache_meta_to_json(f.read(), fnam) for line in json.dumps(json_data, indent=4).splitlines(): if '"path": ' in line: - # We source file path is unpredictable, so filter it out + # The source file path is unpredictable, so filter it out line = re.sub(r'"[^"]+\.pyi?"', "...", line) + if is_meta: + if '"version_id"' in line: + line = re.sub(r'"[0-9][^"]+"', "...", line) + if '"mtime"' in line or '"data_mtime"' in line: + line = re.sub(r": [0-9]+", ": ...", line) + if '"platform"' in line: + line = re.sub(': "[^"]+"', ": ...", line) + if '"hash"' not in line: + # Some hashes are unpredictable so filter them out + line = re.sub(r'"[a-f0-9]{40}"', '""', line) assert "ERROR" not in line, line a.append(line) except CompileError as e: diff --git a/test-data/unit/exportjson.test b/test-data/unit/exportjson.test index 14295281a48f8..43e061dfda322 100644 --- a/test-data/unit/exportjson.test +++ b/test-data/unit/exportjson.test @@ -278,3 +278,47 @@ x: X = 2 [builtins fixtures/tuple.pyi] [out] + +[case testExportMetaBasic_meta] +import typing +from typing_extensions import Final +[builtins fixtures/tuple.pyi] +[out] +{ + "id": "main", + "path": ..., + "mtime": ..., + "size": 49, + "hash": "db2252f953c889e6b78dde8e30bd241a0c86b2d9", + "data_mtime": ..., + "dependencies": [ + "typing", + "typing_extensions", + "builtins" + ], + "suppressed": [], + "options": { + "other_options": "", + "platform": ... + }, + "dep_prios": [ + 10, + 5, + 5 + ], + "dep_lines": [ + 1, + 2, + 1 + ], + "dep_hashes": [ + "", + "", + "" + ], + "interface_hash": "", + "error_lines": [], + "version_id": ..., + "ignore_all": false, + "plugin_data": null +} From c9f69401d5db3678e43400547c4e227d175d8795 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 30 Jan 2026 20:55:24 +0000 Subject: [PATCH 2/4] Fix new buffer name --- mypy/exportjson.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mypy/exportjson.py b/mypy/exportjson.py index e68a4d1f00e46..0514362a58448 100644 --- a/mypy/exportjson.py +++ b/mypy/exportjson.py @@ -553,7 +553,7 @@ def convert_unbound_type(self: UnboundType) -> Json: def convert_binary_cache_meta_to_json(data: bytes, data_file: str) -> Json: - meta = CacheMeta.read(Buffer(data), data_file) + meta = CacheMeta.read(ReadBuffer(data), data_file) assert meta is not None, f"Error reading meta cache file associated with {data_file}" return { "id": meta.id, From 5825e15ef4b19e037916b741341211a4a8138998 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 30 Jan 2026 21:12:48 +0000 Subject: [PATCH 3/4] Update meta format --- mypy/exportjson.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mypy/exportjson.py b/mypy/exportjson.py index 0514362a58448..922b3ecb30889 100644 --- a/mypy/exportjson.py +++ b/mypy/exportjson.py @@ -18,9 +18,9 @@ import sys from typing import Any, TypeAlias as _TypeAlias -from librt.internal import ReadBuffer +from librt.internal import ReadBuffer, cache_version -from mypy.cache import CacheMeta +from mypy.cache import CACHE_VERSION, CacheMeta from mypy.nodes import ( FUNCBASE_FLAGS, FUNCDEF_FLAGS, @@ -553,7 +553,10 @@ def convert_unbound_type(self: UnboundType) -> Json: def convert_binary_cache_meta_to_json(data: bytes, data_file: str) -> Json: - meta = CacheMeta.read(ReadBuffer(data), data_file) + assert ( + data[0] == cache_version() and data[1] == CACHE_VERSION + ), "Cache file created by an incompatible mypy version" + meta = CacheMeta.read(ReadBuffer(data[2:]), data_file) assert meta is not None, f"Error reading meta cache file associated with {data_file}" return { "id": meta.id, From 16d9bc55239918bb8f5e27e9b888d469b0beb101 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Fri, 30 Jan 2026 22:10:17 +0000 Subject: [PATCH 4/4] Skip few checks on Windows --- mypy/test/testexportjson.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/mypy/test/testexportjson.py b/mypy/test/testexportjson.py index 908df9a16e16c..ccd6726e9a60e 100644 --- a/mypy/test/testexportjson.py +++ b/mypy/test/testexportjson.py @@ -24,6 +24,7 @@ class TypeExportSuite(DataSuite): def run_case(self, testcase: DataDrivenTestCase) -> None: error = False src = "\n".join(testcase.input) + is_meta = testcase.name.endswith("_meta") try: options = Options() options.use_builtins_fixtures = True @@ -50,10 +51,10 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: "__future__", "typing_extensions", "sys", + "collections", ): continue fnam = os.path.join(cache_dir, f"{module}.data.ff") - is_meta = testcase.name.endswith("_meta") if not is_meta: with open(fnam, "rb") as f: json_data = convert_binary_cache_to_json(f.read(), implicit_names=False) @@ -81,6 +82,15 @@ def run_case(self, testcase: DataDrivenTestCase) -> None: a = e.messages error = True if error or "\n".join(testcase.output).strip() != "": + if is_meta and sys.platform == "win32": + out = filter_platform_specific(testcase.output) + a = filter_platform_specific(a) + else: + out = testcase.output assert_string_arrays_equal( - testcase.output, a, f"Invalid output ({testcase.file}, line {testcase.line})" + out, a, f"Invalid output ({testcase.file}, line {testcase.line})" ) + + +def filter_platform_specific(lines: list[str]) -> list[str]: + return [l for l in lines if '"size":' not in l and '"hash":' not in l]