From 9b2ddd557e58c9a9e2b973ae495ec2a2cbffd02f Mon Sep 17 00:00:00 2001 From: Nikolay Volik Date: Fri, 23 Jan 2026 11:20:28 +0100 Subject: [PATCH 1/6] [iceberg] Allow partition spec if partition field index is not 0 (#7068) --- .../iceberg/IcebergRestMetadataCommitter.java | 31 ++++++-- .../IcebergRestMetadataCommitterTest.java | 70 +++++++++++++++++++ 2 files changed, 94 insertions(+), 7 deletions(-) diff --git a/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java b/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java index 119ddcd743e1..efb1176c3250 100644 --- a/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java +++ b/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java @@ -31,6 +31,7 @@ import org.apache.iceberg.BaseTable; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.MetadataUpdate; +import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.SnapshotRef; @@ -41,6 +42,7 @@ import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.types.Types.NestedField; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -141,7 +143,7 @@ private void commitMetadataImpl( try { if (!tableExists()) { LOG.info("Table {} does not exist, create it.", icebergTableIdentifier); - icebergTable = createTable(); + icebergTable = createTable(newMetadata); updatdeBuilder = updatesForCorrectBase( ((BaseTable) icebergTable).operations().current(), @@ -240,7 +242,7 @@ private TableMetadata.Builder updatesForCorrectBase( private TableMetadata.Builder updatesForIncorrectBase(TableMetadata newMetadata) { LOG.info("the base metadata is incorrect, we'll recreate the iceberg table."); - icebergTable = recreateTable(); + icebergTable = recreateTable(newMetadata); return updatesForCorrectBase( ((BaseTable) icebergTable).operations().current(), newMetadata, true); } @@ -267,15 +269,30 @@ private void createDatabase() { restCatalog.createNamespace(Namespace.of(icebergDatabaseName)); } - private Table createTable() { + private Table createTable(TableMetadata newMetadata) { /* Here we create iceberg table with an emptySchema. This is because: When creating table, fieldId in iceberg will be forced to start from 1, while fieldId in paimon usually start from 0. If we directly use the schema extracted from paimon to create iceberg table, the fieldId will be in disorder, and this may cause incorrectness when reading by iceberg reader. So we use an emptySchema here, and add the corresponding schemas later. */ - Schema emptySchema = new Schema(); - return restCatalog.createTable(icebergTableIdentifier, emptySchema); + PartitionSpec spec = newMetadata.spec(); + boolean isPartitionedWithZeroFieldId = + spec.fields().stream().anyMatch(f -> f.sourceId() == 0); + if (spec.isUnpartitioned() || isPartitionedWithZeroFieldId) { + if (isPartitionedWithZeroFieldId) { + LOG.warn("LOG MESSAGE HERE"); + } + Schema emptySchema = new Schema(); + return restCatalog.createTable(icebergTableIdentifier, emptySchema); + } else { + List columns = + newMetadata.schema().columns().stream() + .filter(nf -> nf.fieldId() != 0) + .collect(Collectors.toList()); + Schema dummySchema = new Schema(columns); + return restCatalog.createTable(icebergTableIdentifier, dummySchema, spec); + } } private Table getTable() { @@ -287,10 +304,10 @@ private void dropTable() { restCatalog.dropTable(icebergTableIdentifier, false); } - private Table recreateTable() { + private Table recreateTable(TableMetadata newMetadata) { try { dropTable(); - return createTable(); + return createTable(newMetadata); } catch (Exception e) { throw new RuntimeException("Fail to recreate iceberg table.", e); } diff --git a/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java b/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java index e52ac6e06b75..507b809176df 100644 --- a/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java +++ b/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java @@ -53,6 +53,7 @@ import org.apache.iceberg.rest.RESTCatalog; import org.apache.iceberg.rest.RESTCatalogServer; import org.apache.iceberg.rest.RESTServerExtension; +import org.junit.Ignore; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -208,6 +209,75 @@ public void testPartitionedPrimaryKeyTable() throws Exception { Record::toString); } + @Ignore + @Test + public void testPartitionedPrimaryKeyTableWithNonZeroFieldId() throws Exception { + RowType rowType = + RowType.of( + new DataType[] { + DataTypes.STRING(), + DataTypes.INT(), + DataTypes.STRING(), + DataTypes.INT(), + DataTypes.BIGINT() + }, + new String[] {"k", "pt1", "pt2", "v1", "v2"}); + + BiFunction binaryRow = + (pt1, pt2) -> { + BinaryRow b = new BinaryRow(2); + BinaryRowWriter writer = new BinaryRowWriter(b); + writer.writeInt(1, pt1); + writer.writeString(2, BinaryString.fromString(pt2)); + writer.complete(); + return b; + }; + + int numRounds = 20; + int numRecords = 500; + ThreadLocalRandom random = ThreadLocalRandom.current(); + boolean samePartitionEachRound = random.nextBoolean(); + + List> testRecords = new ArrayList<>(); + List> expected = new ArrayList<>(); + Map expectedMap = new LinkedHashMap<>(); + for (int r = 0; r < numRounds; r++) { + List round = new ArrayList<>(); + for (int i = 0; i < numRecords; i++) { + int pt1 = (random.nextInt(0, samePartitionEachRound ? 1 : 2) + r) % 3; + String pt2 = String.valueOf(random.nextInt(10, 12)); + String k = String.valueOf(random.nextInt(0, 100)); + int v1 = random.nextInt(); + long v2 = random.nextLong(); + round.add( + new TestRecord( + binaryRow.apply(pt1, pt2), + GenericRow.of( + BinaryString.fromString(k), + pt1, + BinaryString.fromString(pt2), + v1, + v2))); + expectedMap.put( + String.format("%s, %d, %s", k, pt1, pt2), String.format("%d, %d", v1, v2)); + } + testRecords.add(round); + expected.add( + expectedMap.entrySet().stream() + .map(e -> String.format("Record(%s, %s)", e.getKey(), e.getValue())) + .sorted() + .collect(Collectors.toList())); + } + + runCompatibilityTest( + rowType, + Arrays.asList("pt1", "pt2"), + Arrays.asList("k", "pt1", "pt2"), + testRecords, + expected, + Record::toString); + } + private void runCompatibilityTest( RowType rowType, List partitionKeys, From fb7995c7bb1569a2b0ee6139b1e5260de5c0476c Mon Sep 17 00:00:00 2001 From: Nikolay Volik Date: Sat, 24 Jan 2026 17:40:43 +0100 Subject: [PATCH 2/6] [iceberg] Fix unit test (#7068) --- .../apache/paimon/iceberg/IcebergRestMetadataCommitter.java | 5 ++++- .../paimon/iceberg/IcebergRestMetadataCommitterTest.java | 6 ++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java b/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java index efb1176c3250..d8ad5692ce9a 100644 --- a/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java +++ b/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java @@ -281,7 +281,8 @@ private Table createTable(TableMetadata newMetadata) { spec.fields().stream().anyMatch(f -> f.sourceId() == 0); if (spec.isUnpartitioned() || isPartitionedWithZeroFieldId) { if (isPartitionedWithZeroFieldId) { - LOG.warn("LOG MESSAGE HERE"); + LOG.warn( + "When partition field is 0 fieldId Icebert REST committer will use partition evolution in oreder to support Iceberg compatabilty with Paimon schema. If you want to avoid it uses non 0 field id as partition"); } Schema emptySchema = new Schema(); return restCatalog.createTable(icebergTableIdentifier, emptySchema); @@ -291,6 +292,8 @@ private Table createTable(TableMetadata newMetadata) { .filter(nf -> nf.fieldId() != 0) .collect(Collectors.toList()); Schema dummySchema = new Schema(columns); + LOG.info( + "In order to support schema compatability between Paimon and Iceberg REST dummy schema will be created first"); return restCatalog.createTable(icebergTableIdentifier, dummySchema, spec); } } diff --git a/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java b/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java index 507b809176df..95d99f8073de 100644 --- a/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java +++ b/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java @@ -53,7 +53,6 @@ import org.apache.iceberg.rest.RESTCatalog; import org.apache.iceberg.rest.RESTCatalogServer; import org.apache.iceberg.rest.RESTServerExtension; -import org.junit.Ignore; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -209,7 +208,6 @@ public void testPartitionedPrimaryKeyTable() throws Exception { Record::toString); } - @Ignore @Test public void testPartitionedPrimaryKeyTableWithNonZeroFieldId() throws Exception { RowType rowType = @@ -227,8 +225,8 @@ public void testPartitionedPrimaryKeyTableWithNonZeroFieldId() throws Exception (pt1, pt2) -> { BinaryRow b = new BinaryRow(2); BinaryRowWriter writer = new BinaryRowWriter(b); - writer.writeInt(1, pt1); - writer.writeString(2, BinaryString.fromString(pt2)); + writer.writeInt(0, pt1); + writer.writeString(1, BinaryString.fromString(pt2)); writer.complete(); return b; }; From e0608327817bea26ea6cde47441d9abccba8d2cb Mon Sep 17 00:00:00 2001 From: Nikolay Volik Date: Mon, 26 Jan 2026 16:44:34 -0800 Subject: [PATCH 3/6] [iceberg] offset nested fields (#7068) --- .../iceberg/IcebergRestMetadataCommitter.java | 26 +++++++++++++------ .../IcebergRestMetadataCommitterTest.java | 4 ++- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java b/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java index d8ad5692ce9a..71a14c408d43 100644 --- a/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java +++ b/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java @@ -31,6 +31,7 @@ import org.apache.iceberg.BaseTable; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.MetadataUpdate; +import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; @@ -42,6 +43,7 @@ import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.types.Types; import org.apache.iceberg.types.Types.NestedField; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -281,19 +283,27 @@ private Table createTable(TableMetadata newMetadata) { spec.fields().stream().anyMatch(f -> f.sourceId() == 0); if (spec.isUnpartitioned() || isPartitionedWithZeroFieldId) { if (isPartitionedWithZeroFieldId) { - LOG.warn( - "When partition field is 0 fieldId Icebert REST committer will use partition evolution in oreder to support Iceberg compatabilty with Paimon schema. If you want to avoid it uses non 0 field id as partition"); + LOG.info( + "When the partition field has a fieldId of 0, the Iceberg REST committer will use partition evolution in order to support Iceberg compatibility with the Paimon schema. If you want to avoid this, use a non-zero field ID as the partition"); } Schema emptySchema = new Schema(); return restCatalog.createTable(icebergTableIdentifier, emptySchema); } else { - List columns = - newMetadata.schema().columns().stream() - .filter(nf -> nf.fieldId() != 0) - .collect(Collectors.toList()); - Schema dummySchema = new Schema(columns); LOG.info( - "In order to support schema compatability between Paimon and Iceberg REST dummy schema will be created first"); + "In order to support schema compatibility between Paimon and Iceberg REST, a dummy schema will be created first"); + + int size = + spec.fields().stream().mapToInt(PartitionField::sourceId).max().orElseThrow(); + NestedField[] c = new NestedField[size]; + for (int idx = 0; idx < size; idx++) { + int fieldId = idx + 1; + c[idx] = NestedField.optional(fieldId, "f" + fieldId, Types.BooleanType.get()); + } + for (PartitionField f : spec.fields()) { + c[f.sourceId() - 1] = newMetadata.schema().findField(f.sourceId()); + } + + Schema dummySchema = new Schema(c); return restCatalog.createTable(icebergTableIdentifier, dummySchema, spec); } } diff --git a/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java b/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java index 95d99f8073de..4a8d8b24eb43 100644 --- a/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java +++ b/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java @@ -219,7 +219,9 @@ public void testPartitionedPrimaryKeyTableWithNonZeroFieldId() throws Exception DataTypes.INT(), DataTypes.BIGINT() }, - new String[] {"k", "pt1", "pt2", "v1", "v2"}); + new String[] { + "k", "pt1", "pt2", "v1", "v2" + }); // partition starts from fieldId 1 BiFunction binaryRow = (pt1, pt2) -> { From 8b8a0aebd8724f732de1b77a48ae2d4fea2591f8 Mon Sep 17 00:00:00 2001 From: Nikolay Volik Date: Mon, 2 Feb 2026 17:44:13 +0100 Subject: [PATCH 4/6] [iceberg] Added extra unit tests assert and correct comment (#7068) --- .../iceberg/IcebergRestMetadataCommitter.java | 27 ++++++++++++------- .../IcebergRestMetadataCommitterTest.java | 26 ++++++++++++++++++ 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java b/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java index 71a14c408d43..48a1f427c58c 100644 --- a/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java +++ b/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java @@ -272,11 +272,16 @@ private void createDatabase() { } private Table createTable(TableMetadata newMetadata) { - /* Here we create iceberg table with an emptySchema. This is because: - When creating table, fieldId in iceberg will be forced to start from 1, while fieldId in paimon usually start from 0. - If we directly use the schema extracted from paimon to create iceberg table, the fieldId will be in disorder, and this - may cause incorrectness when reading by iceberg reader. So we use an emptySchema here, and add the corresponding - schemas later. + /* + Handles fieldId incompatibility between Paimon (starts at 0) and Iceberg (starts at 1). + + Direct schema conversion shifts all fieldIds by +1, causing field disorder. While + schemas can be updated post-creation to start at fieldId 0, creating an empty schema + first triggers partition evolution issues that break some query engines. + + Strategy based on partition field position: + Position 0: Creates empty schema first (partition evolution unavoidable) + Position > 0: Creates dummy schema with offset fields and gap filling to preserve the partition spec */ PartitionSpec spec = newMetadata.spec(); boolean isPartitionedWithZeroFieldId = @@ -294,16 +299,20 @@ private Table createTable(TableMetadata newMetadata) { int size = spec.fields().stream().mapToInt(PartitionField::sourceId).max().orElseThrow(); - NestedField[] c = new NestedField[size]; + // prefill the schema with dummy fields + NestedField[] columns = new NestedField[size]; for (int idx = 0; idx < size; idx++) { int fieldId = idx + 1; - c[idx] = NestedField.optional(fieldId, "f" + fieldId, Types.BooleanType.get()); + columns[idx] = + NestedField.optional(fieldId, "f" + fieldId, Types.BooleanType.get()); } + // find and set partition fields with offset -1, so they align correctly after table + // creation for (PartitionField f : spec.fields()) { - c[f.sourceId() - 1] = newMetadata.schema().findField(f.sourceId()); + columns[f.sourceId() - 1] = newMetadata.schema().findField(f.sourceId()); } - Schema dummySchema = new Schema(c); + Schema dummySchema = new Schema(columns); return restCatalog.createTable(icebergTableIdentifier, dummySchema, spec); } } diff --git a/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java b/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java index 4a8d8b24eb43..7111726f3680 100644 --- a/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java +++ b/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java @@ -43,6 +43,8 @@ import org.apache.iceberg.BaseTable; import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.data.IcebergGenerics; @@ -53,6 +55,8 @@ import org.apache.iceberg.rest.RESTCatalog; import org.apache.iceberg.rest.RESTCatalogServer; import org.apache.iceberg.rest.RESTServerExtension; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.types.Types.NestedField; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -139,6 +143,9 @@ public void testUnPartitionedPrimaryKeyTable() throws Exception { testRecords, expected, Record::toString); + + PartitionSpec expectedPartitionSpec = PartitionSpec.builderFor(new Schema()).build(); + runPartitionSpecCompatibilityTest(expectedPartitionSpec); } @Test @@ -206,6 +213,9 @@ public void testPartitionedPrimaryKeyTable() throws Exception { testRecords, expected, Record::toString); + + PartitionSpec expectedPartitionSpec = PartitionSpec.builderFor(new Schema()).build(); + runPartitionSpecCompatibilityTest(expectedPartitionSpec); } @Test @@ -276,6 +286,16 @@ public void testPartitionedPrimaryKeyTableWithNonZeroFieldId() throws Exception testRecords, expected, Record::toString); + + PartitionSpec expectedPartitionSpec = + PartitionSpec.builderFor( + new Schema( + NestedField.required(1, "pt1", Types.IntegerType.get()), + NestedField.required(2, "pt2", Types.StringType.get()))) + .identity("pt1") + .identity("pt2") + .build(); + runPartitionSpecCompatibilityTest(expectedPartitionSpec); } private void runCompatibilityTest( @@ -326,6 +346,12 @@ private void runCompatibilityTest( commit.close(); } + private void runPartitionSpecCompatibilityTest(PartitionSpec expectedSpec) { + Table icebergTable = restCatalog.loadTable(TableIdentifier.of("mydb", "t")); + PartitionSpec spec = icebergTable.spec(); + assertThat(spec).isEqualTo(expectedSpec); + } + @Test public void testSchemaAndPropertiesChange() throws Exception { RowType rowType = From 17748d0c0141eb732cbeac4fc2869864ccda1c12 Mon Sep 17 00:00:00 2001 From: Nikolay Volik Date: Mon, 2 Feb 2026 18:37:40 +0100 Subject: [PATCH 5/6] [iceberg] Update documentation (#7068) --- docs/content/iceberg/rest-catalog.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/content/iceberg/rest-catalog.md b/docs/content/iceberg/rest-catalog.md index 62dbc3b00765..8acdd216ba24 100644 --- a/docs/content/iceberg/rest-catalog.md +++ b/docs/content/iceberg/rest-catalog.md @@ -100,6 +100,16 @@ the query results: 200, 20, 2 */ ``` + +**Schema compatabilty and Partition evolution:** + +There is a fundamental difference between Paimon and Iceberg regarding the starting fieldId. Paimon uses fieldId 0, while Iceberg uses fieldId 1. If we create an Iceberg table using a Paimon schema directly, it will shift all fieldIds by +1, causing field disorder. However, it is possible to update the schema after table creation and start the schema from fieldId 0. + +Table creation attempts to minimize issues with fieldId disorder and partition evolution by following a 2 option logic: + +- Partition fieldId = 0: Paimon creates an empty schema first and then updates the schema to the actual one. Partition evolution is unavoidable. +- Partition fieldId > 0: Paimon creates an initial dummy schema first, offsetting partition fields correctly, and then updates the schema to the actual one, avoiding partition evolution. + **Note:** Paimon will firstly write iceberg metadata in a separate directory like hadoop-catalog, and then commit metadata to iceberg rest catalog. @@ -108,5 +118,5 @@ If the two are incompatible, we take the metadata stored in the separate directo There are some cases when committing to iceberg rest catalog: 1. table not exists in iceberg rest-catalog. It'll create the table in rest catalog first, and commit metadata. 2. table exists in iceberg rest-catalog and is compatible with the base metadata stored in the separate directory. It'll directly get the table and commit metadata. -3. table exists, and isn't compatible with the base metadata stored in the separate directory. It'll **drop the table and recreate the table**, then commit metadata. +3. table exists, and isn't compatible with the base metadata stored in the separate directory. It'll **drop the table and recreate the table**, then commit metadata. From 501d00ac20201e3ac34e97bb1ee014270f37c27e Mon Sep 17 00:00:00 2001 From: Nikolay Volik Date: Tue, 3 Feb 2026 08:50:26 +0100 Subject: [PATCH 6/6] [iceberg] Update comments (#7068) --- .../paimon/iceberg/IcebergRestMetadataCommitter.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java b/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java index 48a1f427c58c..bb358512fe53 100644 --- a/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java +++ b/paimon-iceberg/src/main/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitter.java @@ -280,8 +280,8 @@ Handles fieldId incompatibility between Paimon (starts at 0) and Iceberg (starts first triggers partition evolution issues that break some query engines. Strategy based on partition field position: - Position 0: Creates empty schema first (partition evolution unavoidable) - Position > 0: Creates dummy schema with offset fields and gap filling to preserve the partition spec + - fieldId = 0: Creates empty schema first, partition evolution unavoidable + - fieldId > 0: Creates dummy schema with offset fields and gap filling to preserve the partition spec */ PartitionSpec spec = newMetadata.spec(); boolean isPartitionedWithZeroFieldId = @@ -289,13 +289,13 @@ Handles fieldId incompatibility between Paimon (starts at 0) and Iceberg (starts if (spec.isUnpartitioned() || isPartitionedWithZeroFieldId) { if (isPartitionedWithZeroFieldId) { LOG.info( - "When the partition field has a fieldId of 0, the Iceberg REST committer will use partition evolution in order to support Iceberg compatibility with the Paimon schema. If you want to avoid this, use a non-zero field ID as the partition"); + "Partition fieldId = 0. The Iceberg REST committer will use partition evolution to support Iceberg compatibility with the Paimon schema. If you want to avoid this, use a non-zero fieldId partition field"); } Schema emptySchema = new Schema(); return restCatalog.createTable(icebergTableIdentifier, emptySchema); } else { LOG.info( - "In order to support schema compatibility between Paimon and Iceberg REST, a dummy schema will be created first"); + "Partition fieldId > 0. In order to avoid partition evlolution, dummy schema will be created first"); int size = spec.fields().stream().mapToInt(PartitionField::sourceId).max().orElseThrow();