Add tests for schema immutability in table metadata

Author: ndrluis

tests/catalog/test_hive.py

@@ -626,6 +626,10 @@ def test_load_table(hive_table: HiveTable) -> None:
                 NestedField(field_id=1, name="x", field_type=LongType(), required=True),
                 NestedField(field_id=2, name="y", field_type=LongType(), required=True, doc="comment"),
                 NestedField(field_id=3, name="z", field_type=LongType(), required=True),
+                NestedField(field_id=4, name="a", field_type=DecimalType(precision=16, scale=2), required=True),
+                NestedField(field_id=5, name="b", field_type=DecimalType(precision=16, scale=8), required=True),
+                NestedField(field_id=6, name="c", field_type=FixedType(length=16), required=True),
+                NestedField(field_id=7, name="d", field_type=FixedType(length=18), required=True),
                 schema_id=1,
                 identifier_field_ids=[1, 2],
             ),

tests/conftest.py

@@ -821,6 +821,10 @@ def generate_snapshot(
                 {"id": 1, "name": "x", "required": True, "type": "long"},
                 {"id": 2, "name": "y", "required": True, "type": "long", "doc": "comment"},
                 {"id": 3, "name": "z", "required": True, "type": "long"},
+                {"id": 4, "name": "a", "required": True, "type": "decimal(16, 2)"},
+                {"id": 5, "name": "b", "required": True, "type": "decimal(16, 8)"},
+                {"id": 6, "name": "c", "required": True, "type": "fixed[16]"},
+                {"id": 7, "name": "d", "required": True, "type": "fixed[18]"},
             ],
         },
     ],

tests/table/test_metadata.py

@@ -174,7 +174,7 @@ def test_serialize_v1(example_table_metadata_v1: Dict[str, Any]) -> None:
 
 def test_serialize_v2(example_table_metadata_v2: Dict[str, Any]) -> None:
     table_metadata = TableMetadataV2(**example_table_metadata_v2).model_dump_json()
-    expected = """{"location":"s3://bucket/test/location","table-uuid":"9c12d441-03fe-4693-9a96-a0705ddf69c1","last-updated-ms":1602638573590,"last-column-id":3,"schemas":[{"type":"struct","fields":[{"id":1,"name":"x","type":"long","required":true}],"schema-id":0,"identifier-field-ids":[]},{"type":"struct","fields":[{"id":1,"name":"x","type":"long","required":true},{"id":2,"name":"y","type":"long","required":true,"doc":"comment"},{"id":3,"name":"z","type":"long","required":true}],"schema-id":1,"identifier-field-ids":[1,2]}],"current-schema-id":1,"partition-specs":[{"spec-id":0,"fields":[{"source-id":1,"field-id":1000,"transform":"identity","name":"x"}]}],"default-spec-id":0,"last-partition-id":1000,"properties":{"read.split.target.size":"134217728"},"current-snapshot-id":3055729675574597004,"snapshots":[{"snapshot-id":3051729675574597004,"sequence-number":0,"timestamp-ms":1515100955770,"manifest-list":"s3://a/b/1.avro","summary":{"operation":"append"}},{"snapshot-id":3055729675574597004,"parent-snapshot-id":3051729675574597004,"sequence-number":1,"timestamp-ms":1555100955770,"manifest-list":"s3://a/b/2.avro","summary":{"operation":"append"},"schema-id":1}],"snapshot-log":[{"snapshot-id":3051729675574597004,"timestamp-ms":1515100955770},{"snapshot-id":3055729675574597004,"timestamp-ms":1555100955770}],"metadata-log":[{"metadata-file":"s3://bucket/.../v1.json","timestamp-ms":1515100}],"sort-orders":[{"order-id":3,"fields":[{"source-id":2,"transform":"identity","direction":"asc","null-order":"nulls-first"},{"source-id":3,"transform":"bucket[4]","direction":"desc","null-order":"nulls-last"}]}],"default-sort-order-id":3,"refs":{"test":{"snapshot-id":3051729675574597004,"type":"tag","max-ref-age-ms":10000000},"main":{"snapshot-id":3055729675574597004,"type":"branch"}},"format-version":2,"last-sequence-number":34}"""
+    expected = """{"location":"s3://bucket/test/location","table-uuid":"9c12d441-03fe-4693-9a96-a0705ddf69c1","last-updated-ms":1602638573590,"last-column-id":3,"schemas":[{"type":"struct","fields":[{"id":1,"name":"x","type":"long","required":true}],"schema-id":0,"identifier-field-ids":[]},{"type":"struct","fields":[{"id":1,"name":"x","type":"long","required":true},{"id":2,"name":"y","type":"long","required":true,"doc":"comment"},{"id":3,"name":"z","type":"long","required":true},{"id":4,"name":"a","type":"decimal(16, 2)","required":true},{"id":5,"name":"b","type":"decimal(16, 8)","required":true},{"id":6,"name":"c","type":"fixed[16]","required":true},{"id":7,"name":"d","type":"fixed[18]","required":true}],"schema-id":1,"identifier-field-ids":[1,2]}],"current-schema-id":1,"partition-specs":[{"spec-id":0,"fields":[{"source-id":1,"field-id":1000,"transform":"identity","name":"x"}]}],"default-spec-id":0,"last-partition-id":1000,"properties":{"read.split.target.size":"134217728"},"current-snapshot-id":3055729675574597004,"snapshots":[{"snapshot-id":3051729675574597004,"sequence-number":0,"timestamp-ms":1515100955770,"manifest-list":"s3://a/b/1.avro","summary":{"operation":"append"}},{"snapshot-id":3055729675574597004,"parent-snapshot-id":3051729675574597004,"sequence-number":1,"timestamp-ms":1555100955770,"manifest-list":"s3://a/b/2.avro","summary":{"operation":"append"},"schema-id":1}],"snapshot-log":[{"snapshot-id":3051729675574597004,"timestamp-ms":1515100955770},{"snapshot-id":3055729675574597004,"timestamp-ms":1555100955770}],"metadata-log":[{"metadata-file":"s3://bucket/.../v1.json","timestamp-ms":1515100}],"sort-orders":[{"order-id":3,"fields":[{"source-id":2,"transform":"identity","direction":"asc","null-order":"nulls-first"},{"source-id":3,"transform":"bucket[4]","direction":"desc","null-order":"nulls-last"}]}],"default-sort-order-id":3,"refs":{"test":{"snapshot-id":3051729675574597004,"type":"tag","max-ref-age-ms":10000000},"main":{"snapshot-id":3055729675574597004,"type":"branch"}},"format-version":2,"last-sequence-number":34}"""
     assert table_metadata == expected
 
 
@@ -755,3 +755,14 @@ def test_make_metadata_fresh() -> None:
     )
 
     assert actual.model_dump() == expected.model_dump()
+
+
+def test_schema_immutability(example_table_metadata_v2: Dict[str, Any]) -> None:
+    table_metadata = TableMetadataV2(**example_table_metadata_v2)
+
+    # Schema id 1 has 2 columns of decimal and fixed columns
+    # to validate the schema immutability of Singleton types
+    original_schema = table_metadata.schemas[1]
+    copied_schema = table_metadata.model_copy(deep=True).schemas[1]
+
+    assert original_schema == copied_schema