Skip to content

Commit 0e279ef

Browse files
wgtmacFokko
andauthored
Add comparator for UnknownLogicalType (#3292) (#3295)
This was missing, which caused exceptions. This PR adds them including a test. It takes the comparator of the underlying physical type, but it should always compare two null values: https://github.com/apache/parquet-format/blob/471d33a3c9d46e05b8a189437a9364f959f93f09/LogicalTypes.md?plain=1#L887-L892 Co-authored-by: Fokko Driesprong <fokko@apache.org>
1 parent f85f083 commit 0e279ef

File tree

2 files changed

+47
-0
lines changed

2 files changed

+47
-0
lines changed

parquet-column/src/main/java/org/apache/parquet/schema/PrimitiveType.java

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,12 @@ public Optional<PrimitiveComparator> visit(
119119
LogicalTypeAnnotation.TimestampLogicalTypeAnnotation timestampLogicalType) {
120120
return of(PrimitiveComparator.SIGNED_INT64_COMPARATOR);
121121
}
122+
123+
@Override
124+
public Optional<PrimitiveComparator> visit(
125+
LogicalTypeAnnotation.UnknownLogicalTypeAnnotation unknownLogicalTypeAnnotation) {
126+
return of(PrimitiveComparator.SIGNED_INT64_COMPARATOR);
127+
}
122128
})
123129
.orElseThrow(() -> new ShouldNeverHappenException(
124130
"No comparator logic implemented for INT64 logical type: " + logicalType));
@@ -183,6 +189,12 @@ public Optional<PrimitiveComparator> visit(
183189
}
184190
return empty();
185191
}
192+
193+
@Override
194+
public Optional<PrimitiveComparator> visit(
195+
LogicalTypeAnnotation.UnknownLogicalTypeAnnotation unknownLogicalTypeAnnotation) {
196+
return of(PrimitiveComparator.SIGNED_INT32_COMPARATOR);
197+
}
186198
})
187199
.orElseThrow(() -> new ShouldNeverHappenException(
188200
"No comparator logic implemented for INT32 logical type: " + logicalType));
@@ -283,6 +295,12 @@ public Optional<PrimitiveComparator> visit(
283295
LogicalTypeAnnotation.GeographyLogicalTypeAnnotation geographyLogicalType) {
284296
return of(PrimitiveComparator.UNSIGNED_LEXICOGRAPHICAL_BINARY_COMPARATOR);
285297
}
298+
299+
@Override
300+
public Optional<PrimitiveComparator> visit(
301+
LogicalTypeAnnotation.UnknownLogicalTypeAnnotation unknownLogicalTypeAnnotation) {
302+
return of(PrimitiveComparator.UNSIGNED_LEXICOGRAPHICAL_BINARY_COMPARATOR);
303+
}
286304
})
287305
.orElseThrow(() -> new ShouldNeverHappenException(
288306
"No comparator logic implemented for BINARY logical type: " + logicalType));
@@ -417,6 +435,12 @@ public Optional<PrimitiveComparator> visit(
417435
LogicalTypeAnnotation.Float16LogicalTypeAnnotation float16LogicalType) {
418436
return of(PrimitiveComparator.BINARY_AS_FLOAT16_COMPARATOR);
419437
}
438+
439+
@Override
440+
public Optional<PrimitiveComparator> visit(
441+
LogicalTypeAnnotation.UnknownLogicalTypeAnnotation unknownLogicalTypeAnnotation) {
442+
return of(PrimitiveComparator.UNSIGNED_LEXICOGRAPHICAL_BINARY_COMPARATOR);
443+
}
420444
})
421445
.orElseThrow(() -> new ShouldNeverHappenException(
422446
"No comparator logic implemented for FIXED_LEN_BYTE_ARRAY logical type: "

parquet-column/src/test/java/org/apache/parquet/schema/TestPrimitiveComparator.java

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,29 @@ private void testInt32Comparator(PrimitiveComparator<Integer> comparator, Intege
100100
checkThrowingUnsupportedException(comparator, Integer.TYPE);
101101
}
102102

103+
@Test
104+
public void testUnknownLogicalTypeComparator() {
105+
PrimitiveType.PrimitiveTypeName[] types = new PrimitiveType.PrimitiveTypeName[] {
106+
PrimitiveType.PrimitiveTypeName.BOOLEAN,
107+
PrimitiveType.PrimitiveTypeName.BINARY,
108+
PrimitiveType.PrimitiveTypeName.INT32,
109+
PrimitiveType.PrimitiveTypeName.INT64,
110+
PrimitiveType.PrimitiveTypeName.FLOAT,
111+
PrimitiveType.PrimitiveTypeName.DOUBLE,
112+
PrimitiveType.PrimitiveTypeName.INT96,
113+
PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY
114+
};
115+
116+
for (PrimitiveType.PrimitiveTypeName type : types) {
117+
assertEquals(
118+
new PrimitiveType(Type.Repetition.REQUIRED, type, "vo")
119+
.withLogicalTypeAnnotation(LogicalTypeAnnotation.unknownType())
120+
.comparator()
121+
.compare(null, null),
122+
0);
123+
}
124+
}
125+
103126
@Test
104127
public void testSignedInt64Comparator() {
105128
testInt64Comparator(

0 commit comments

Comments
 (0)