Skip to content

Commit 969cb8b

Browse files
committed
positional delete does not decrement total-records
1 parent b440682 commit 969cb8b

File tree

1 file changed

+46
-3
lines changed

1 file changed

+46
-3
lines changed

tests/integration/test_deletes.py

Lines changed: 46 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -454,16 +454,59 @@ def test_partitioned_table_positional_deletes_sequence_number(spark: SparkSessio
454454
files = list(tbl.scan().plan_files())
455455
assert len(files) == 2
456456

457-
# Will rewrite a data file without a positional delete
457+
# Will rewrite a data file, not using positional delete
458458
tbl.delete(EqualTo("number", 201))
459459

460460
snapshots = tbl.snapshots()
461461
assert len(snapshots) == 3
462462

463463
# Snapshots produced by Spark
464-
assert [snapshot.summary.operation.value for snapshot in tbl.snapshots()[0:2]] == ["append", "delete"]
464+
assert [snapshot.summary.operation.value for snapshot in tbl.snapshots()] == ["append", "delete", "overwrite"]
465465

466-
# Will rewrite one parquet file
466+
# Insert 5 rows into 2 different partitions
467+
assert snapshots[0].summary == Summary(
468+
Operation.APPEND,
469+
**{
470+
"spark.app.id": snapshots[0].summary["spark.app.id"],
471+
"added-data-files": "2",
472+
"added-records": "5",
473+
"added-files-size": snapshots[0].summary["added-files-size"],
474+
"changed-partition-count": "2",
475+
"total-records": "5",
476+
"total-files-size": snapshots[0].summary["total-files-size"],
477+
"total-data-files": "2",
478+
"total-delete-files": "0",
479+
"total-position-deletes": "0",
480+
"total-equality-deletes": "0",
481+
"engine-version": snapshots[0].summary["engine-version"],
482+
"app-id": snapshots[0].summary["app-id"],
483+
"engine-name": snapshots[0].summary["engine-name"],
484+
"iceberg-version": snapshots[0].summary["iceberg-version"],
485+
},
486+
)
487+
# MoR delete 1 row, add 1 positional delete file
488+
assert snapshots[1].summary == Summary(
489+
Operation.DELETE,
490+
**{
491+
"added-position-delete-files": "1",
492+
"added-delete-files": "1",
493+
"added-files-size": snapshots[1].summary["added-files-size"],
494+
"added-position-deletes": "1",
495+
"changed-partition-count": "1",
496+
"total-records": "5", # ???
497+
"total-files-size": snapshots[1].summary["total-files-size"],
498+
"total-data-files": "2",
499+
"total-delete-files": "1",
500+
"total-position-deletes": "1",
501+
"total-equality-deletes": "0",
502+
"spark.app.id": snapshots[1].summary["spark.app.id"],
503+
"engine-version": snapshots[1].summary["engine-version"],
504+
"app-id": snapshots[1].summary["app-id"],
505+
"engine-name": snapshots[1].summary["engine-name"],
506+
"iceberg-version": snapshots[1].summary["iceberg-version"],
507+
},
508+
)
509+
# CoW delete 1 row, rewrite 1 parquet file
467510
assert snapshots[2].summary == Summary(
468511
Operation.OVERWRITE,
469512
**{

0 commit comments

Comments
 (0)