From 4410c9bdc434e1a49f3464adef632cfa47815c25 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Mon, 29 Dec 2025 16:27:56 +0800 Subject: [PATCH 1/3] [python] Update PyPaimon Snapshot fields to not null --- paimon-python/pypaimon/snapshot/snapshot.py | 7 +++---- .../tests/rest/rest_catalog_commit_snapshot_test.py | 3 +-- paimon-python/pypaimon/tests/rest/rest_server.py | 3 +-- paimon-python/pypaimon/write/file_store_commit.py | 2 +- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/paimon-python/pypaimon/snapshot/snapshot.py b/paimon-python/pypaimon/snapshot/snapshot.py index d5f72dedcf46..eb18db67952d 100644 --- a/paimon-python/pypaimon/snapshot/snapshot.py +++ b/paimon-python/pypaimon/snapshot/snapshot.py @@ -27,21 +27,20 @@ @dataclass class Snapshot: # Required fields + version: int = json_field("version") id: int = json_field("id") schema_id: int = json_field("schemaId") base_manifest_list: str = json_field("baseManifestList") delta_manifest_list: str = json_field("deltaManifestList") + total_record_count: int = json_field("totalRecordCount") + delta_record_count: int = json_field("deltaRecordCount") commit_user: str = json_field("commitUser") commit_identifier: int = json_field("commitIdentifier") commit_kind: str = json_field("commitKind") time_millis: int = json_field("timeMillis") # Optional fields with defaults - version: Optional[int] = json_field("version", default=None) - log_offsets: Optional[Dict[int, int]] = json_field("logOffsets", default_factory=dict) changelog_manifest_list: Optional[str] = json_field("changelogManifestList", default=None) index_manifest: Optional[str] = json_field("indexManifest", default=None) - total_record_count: Optional[int] = json_field("totalRecordCount", default=None) - delta_record_count: Optional[int] = json_field("deltaRecordCount", default=None) changelog_record_count: Optional[int] = json_field("changelogRecordCount", default=None) watermark: Optional[int] = json_field("watermark", default=None) statistics: Optional[str] = json_field("statistics", default=None) diff --git a/paimon-python/pypaimon/tests/rest/rest_catalog_commit_snapshot_test.py b/paimon-python/pypaimon/tests/rest/rest_catalog_commit_snapshot_test.py index f038f3c0ae61..70b99fbe69ab 100644 --- a/paimon-python/pypaimon/tests/rest/rest_catalog_commit_snapshot_test.py +++ b/paimon-python/pypaimon/tests/rest/rest_catalog_commit_snapshot_test.py @@ -60,8 +60,7 @@ def setUp(self): commit_user="test_user", commit_identifier=12345, commit_kind="APPEND", - time_millis=int(time.time() * 1000), - log_offsets={} + time_millis=int(time.time() * 1000) ) # Create test statistics diff --git a/paimon-python/pypaimon/tests/rest/rest_server.py b/paimon-python/pypaimon/tests/rest/rest_server.py index 1970b79dd067..4510fc332329 100755 --- a/paimon-python/pypaimon/tests/rest/rest_server.py +++ b/paimon-python/pypaimon/tests/rest/rest_server.py @@ -544,8 +544,7 @@ def _write_snapshot_files(self, identifier: Identifier, snapshot, statistics): "commitUser": getattr(snapshot, 'commit_user', 'rest-server'), "commitIdentifier": getattr(snapshot, 'commit_identifier', 1), "commitKind": getattr(snapshot, 'commit_kind', 'APPEND'), - "timeMillis": getattr(snapshot, 'time_millis', 1703721600000), - "logOffsets": getattr(snapshot, 'log_offsets', {}) + "timeMillis": getattr(snapshot, 'time_millis', 1703721600000) } with open(snapshot_file, 'w') as f: diff --git a/paimon-python/pypaimon/write/file_store_commit.py b/paimon-python/pypaimon/write/file_store_commit.py index 42b1e0ca24e9..e30b272e26fc 100644 --- a/paimon-python/pypaimon/write/file_store_commit.py +++ b/paimon-python/pypaimon/write/file_store_commit.py @@ -199,7 +199,7 @@ def _try_commit(self, commit_kind, commit_entries, commit_identifier): new_snapshot_id = self._generate_snapshot_id() total_record_count += delta_record_count snapshot_data = Snapshot( - version=1, + version=3, id=new_snapshot_id, schema_id=self.table.table_schema.id, base_manifest_list=base_manifest_list, From 81989814f9398554ef5283032a419cd98488f2fd Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Mon, 29 Dec 2025 16:28:50 +0800 Subject: [PATCH 2/3] fix --- paimon-python/pypaimon/snapshot/snapshot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paimon-python/pypaimon/snapshot/snapshot.py b/paimon-python/pypaimon/snapshot/snapshot.py index eb18db67952d..b1fdbaafea4c 100644 --- a/paimon-python/pypaimon/snapshot/snapshot.py +++ b/paimon-python/pypaimon/snapshot/snapshot.py @@ -17,7 +17,7 @@ ################################################################################ from dataclasses import dataclass -from typing import Dict, Optional +from typing import Optional from pypaimon.common.json_util import json_field From 34eb80cec0377cd3d2b18f427dc04ed27274b188 Mon Sep 17 00:00:00 2001 From: JingsongLi Date: Mon, 29 Dec 2025 17:24:12 +0800 Subject: [PATCH 3/3] fix --- .../pypaimon/tests/rest/rest_catalog_commit_snapshot_test.py | 2 ++ paimon-python/pypaimon/tests/rest/rest_server.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/paimon-python/pypaimon/tests/rest/rest_catalog_commit_snapshot_test.py b/paimon-python/pypaimon/tests/rest/rest_catalog_commit_snapshot_test.py index 70b99fbe69ab..e9f95d3d0b0e 100644 --- a/paimon-python/pypaimon/tests/rest/rest_catalog_commit_snapshot_test.py +++ b/paimon-python/pypaimon/tests/rest/rest_catalog_commit_snapshot_test.py @@ -57,6 +57,8 @@ def setUp(self): schema_id=0, base_manifest_list="manifest-list-1", delta_manifest_list="manifest-list-1", + total_record_count=1, + delta_record_count=1, commit_user="test_user", commit_identifier=12345, commit_kind="APPEND", diff --git a/paimon-python/pypaimon/tests/rest/rest_server.py b/paimon-python/pypaimon/tests/rest/rest_server.py index 4510fc332329..b97d015dbeef 100755 --- a/paimon-python/pypaimon/tests/rest/rest_server.py +++ b/paimon-python/pypaimon/tests/rest/rest_server.py @@ -541,6 +541,8 @@ def _write_snapshot_files(self, identifier: Identifier, snapshot, statistics): "schemaId": getattr(snapshot, 'schema_id', 0), "baseManifestList": getattr(snapshot, 'base_manifest_list', f"manifest-list-{uuid.uuid4()}"), "deltaManifestList": getattr(snapshot, 'delta_manifest_list', f"manifest-list-{uuid.uuid4()}"), + "totalRecordCount": getattr(snapshot, 'total_record_count'), + "deltaRecordCount": getattr(snapshot, 'delta_record_count'), "commitUser": getattr(snapshot, 'commit_user', 'rest-server'), "commitIdentifier": getattr(snapshot, 'commit_identifier', 1), "commitKind": getattr(snapshot, 'commit_kind', 'APPEND'),