From 21b6196cb5a1dcb5ad8a2a212a0f89bea64f91a9 Mon Sep 17 00:00:00 2001 From: Ethan Knox Date: Mon, 31 Mar 2025 10:39:31 -0400 Subject: [PATCH 1/7] added docs for expressions --- mkdocs/docs/SUMMARY.md | 2 + mkdocs/docs/expression-dsl.md | 244 +++++++++++++++++++++++++++++++ mkdocs/docs/row-filter-syntax.md | 154 +++++++++++++++++++ 3 files changed, 400 insertions(+) create mode 100644 mkdocs/docs/expression-dsl.md create mode 100644 mkdocs/docs/row-filter-syntax.md diff --git a/mkdocs/docs/SUMMARY.md b/mkdocs/docs/SUMMARY.md index c344b2fdd2..d268bcc4b0 100644 --- a/mkdocs/docs/SUMMARY.md +++ b/mkdocs/docs/SUMMARY.md @@ -24,6 +24,8 @@ - [Configuration](configuration.md) - [CLI](cli.md) - [API](api.md) + - [Row Filter Syntax](row-filter-syntax.md) + - [Expression DSL](expression-dsl.md) - [Contributing](contributing.md) - [Community](community.md) - Releases diff --git a/mkdocs/docs/expression-dsl.md b/mkdocs/docs/expression-dsl.md new file mode 100644 index 0000000000..fa0a53f656 --- /dev/null +++ b/mkdocs/docs/expression-dsl.md @@ -0,0 +1,244 @@ +# Expression DSL + +The PyIceberg library provides a powerful expression DSL (Domain Specific Language) for building complex row filter expressions. This guide will help you understand how to use the expression DSL effectively. This DSL allows you to build type-safe expressions for use in the `row_filter` scan argument. + +They are composed of terms, predicates, and logical operators. + +## Basic Concepts + +### Terms + +Terms are the basic building blocks of expressions. They represent references to fields in your data: + +```python +from pyiceberg.expressions import Reference + +# Create a reference to a field named "age" +age_field = Reference("age") +``` + +### Predicates + +Predicates are expressions that evaluate to a boolean value. They can be combined using logical operators. + +#### Comparison Predicates + +```python +from pyiceberg.expressions import EqualTo, NotEqualTo, LessThan, LessThanOrEqual, GreaterThan, GreaterThanOrEqual + +# age equals 18 +age_equals_18 = EqualTo("age", 18) + +# age is not equal to 18 +age_not_equals_18 = NotEqualTo("age", 18) + +# age is less than 18 +age_less_than_18 = LessThan("age", 18) + +# Less than or equal to +age_less_than_or_equal_18 = LessThanOrEqual("age", 18) + +# Greater than +age_greater_than_18 = GreaterThan("age", 18) + +# Greater than or equal to +age_greater_than_or_equal_18 = GreaterThanOrEqual("age", 18) +``` + +#### Set Predicates + +```python +from pyiceberg.expressions import In, NotIn + +# age is one of 18, 19, 20 +age_in_set = In("age", [18, 19, 20]) + +# age is not 18, 19, oer 20 +age_not_in_set = NotIn("age", [18, 19, 20]) +``` + +#### Null Predicates + +```python +from pyiceberg.expressions import IsNull, NotNull + +# Is null +name_is_null = IsNull("name") + +# Is not null +name_is_not_null = NotNull("name") +``` + +#### String Predicates + +```python +from pyiceberg.expressions import StartsWith, NotStartsWith + +# TRUE for 'Johnathan', FALSE for 'Johan' +name_starts_with = StartsWith("name", "John") + +# FALSE for 'Johnathan', TRUE for 'Johan' +name_not_starts_with = NotStartsWith("name", "John") +``` + +### Logical Operators + +You can combine predicates using logical operators: + +```python +from pyiceberg.expressions import And, Or, Not + +# TRUE for 25, FALSE for 67 and 15 +age_between = And( + GreaterThanOrEqual("age", 18), + LessThanOrEqual("age", 65) +) + +# FALSE for 25, TRUE for 67 and 15 +age_outside = Or( + LessThan("age", 18), + GreaterThan("age", 65) +) + +# NOT operator +not_adult = Not(GreaterThanOrEqual("age", 18)) +``` + +## Advanced Usage + +### Complex Expressions + +You can build complex expressions by combining multiple predicates and operators: + +```python +from pyiceberg.expressions import And, Or, Not, EqualTo, GreaterThan, LessThan, In + +# (age >= 18 AND age <= 65) AND (status = 'active' OR status = 'pending') +complex_filter = And( + And( + GreaterThanOrEqual("age", 18), + LessThanOrEqual("age", 65) + ), + Or( + EqualTo("status", "active"), + EqualTo("status", "pending") + ) +) + +# NOT (age < 18 OR age > 65) +age_in_range = Not( + Or( + LessThan("age", 18), + GreaterThan("age", 65) + ) +) +``` + +### Type Safety + +The expression DSL provides type safety through Python's type system. When you create expressions, the types are checked at runtime: + +```python +from pyiceberg.expressions import EqualTo + +# This will work +age_equals_18 = EqualTo("age", 18) + +# This will raise a TypeError if the field type doesn't match +age_equals_18 = EqualTo("age", "18") # Will fail if age is an integer field +``` + +## Best Practices + +1. **Use Type Hints**: Always use type hints when working with expressions to catch type-related errors early. + +2. **Break Down Complex Expressions**: For complex expressions, break them down into smaller, more manageable parts: + +```python +# Instead of this: +complex_filter = And( + And( + GreaterThanOrEqual("age", 18), + LessThanOrEqual("age", 65) + ), + Or( + EqualTo("status", "active"), + EqualTo("status", "pending") + ) +) + +# Do this: +age_range = And( + GreaterThanOrEqual("age", 18), + LessThanOrEqual("age", 65) +) + +status_filter = Or( + EqualTo("status", "active"), + EqualTo("status", "pending") +) + +complex_filter = And(age_range, status_filter) +``` + +## Common Pitfalls + +1. **Type Mismatches**: Always ensure that the types of your literals match the field types in your schema. + +2. **Null Handling**: Be careful when using `IsNull` and `NotNull` predicates with required fields. The expression DSL will automatically optimize these cases: + - `IsNull` on a required field will always return `False` + - `NotNull` on a required field will always return `True` + +3. **String Comparisons**: When using string predicates like `StartsWith`, ensure that the field type is actually a string type. + +## Examples + +Here are some practical examples of using the expression DSL: + +### Basic Filtering + +```python + +from datetime import datetime +from pyiceberg.expressions import ( + And, + EqualTo, + GreaterThanOrEqual, + LessThanOrEqual, + GreaterThan, + In +) + +active_adult_users_filter = And( + EqualTo("status", "active"), + GreaterThanOrEqual("age", 18) +) + + +high_value_customers = And( + GreaterThan("total_spent", 1000), + In("membership_level", ["gold", "platinum"]) +) + +date_range_filter = And( + GreaterThanOrEqual("created_at", datetime(2024, 1, 1)), + LessThanOrEqual("created_at", datetime(2024, 12, 31)) +) +``` + +### Multi-Condition Filter + +```python +from pyiceberg.expressions import And, Or, Not, EqualTo, GreaterThan + +complex_filter = And( + Not(EqualTo("status", "deleted")), + Or( + And( + EqualTo("type", "premium"), + GreaterThan("subscription_months", 12) + ), + EqualTo("type", "enterprise") + ) +) +``` \ No newline at end of file diff --git a/mkdocs/docs/row-filter-syntax.md b/mkdocs/docs/row-filter-syntax.md new file mode 100644 index 0000000000..53844ec098 --- /dev/null +++ b/mkdocs/docs/row-filter-syntax.md @@ -0,0 +1,154 @@ +# Row Filter Syntax + +In addtion to the primary [Expression DSL](expression-dsl.md), PyIceberg provides a string-based statement interface for filtering rows in Iceberg tables. This guide explains the syntax and provides examples for supported operations. + +The row filter syntax is designed to be similar to SQL WHERE clauses. Here are the basic components: + +### Column References + +Columns can be referenced using either unquoted or quoted identifiers: + +```sql +column_name +"column.name" +``` + +### Literals + +The following literal types are supported: + +- Strings: `'hello world'` +- Numbers: `42`, `-42`, `3.14` +- Booleans: `true`, `false` (case insensitive) + +## Comparison Operations + +### Basic Comparisons + +```sql +column = 42 +column != 42 +column > 42 +column >= 42 +column < 42 +column <= 42 +``` + +!!! note + The `==` operator is an alias for `=` and `<>` is an alias for `!=` + +### String Comparisons + +```sql +column = 'hello' +column != 'world' +``` + +## NULL Checks + +Check for NULL values using the `IS NULL` and `IS NOT NULL` operators: + +```sql +column IS NULL +column IS NOT NULL +``` + +## NaN Checks + +For floating-point columns, you can check for NaN values: + +```sql +column IS NAN +column IS NOT NAN +``` + +## IN and NOT IN + +Check if a value is in a set of values: + +```sql +column IN ('a', 'b', 'c') +column NOT IN (1, 2, 3) +``` + +## LIKE Operations + +The LIKE operator supports pattern matching with a wildcard `%` at the end of the string: + +```sql +column LIKE 'prefix%' +column NOT LIKE 'prefix%' +``` + +!!! important + The `%` wildcard is only supported at the end of the pattern. Using it in the middle or beginning of the pattern will raise an error. + +## Logical Operations + +Combine multiple conditions using logical operators: + +```sql +column1 = 42 AND column2 = 'hello' +column1 > 0 OR column2 IS NULL +NOT (column1 = 42) +``` + +!!! tip + Parentheses can be used to group logical operations for clarity: + ```sql + (column1 = 42 AND column2 = 'hello') OR column3 IS NULL + ``` + +## Complete Examples + +Here are some complete examples showing how to combine different operations: + +```sql +-- Complex filter with multiple conditions +status = 'active' AND age > 18 AND NOT (country IN ('US', 'CA')) + +-- Filter with string pattern matching +name LIKE 'John%' AND age >= 21 + +-- Filter with NULL checks and numeric comparisons +price IS NOT NULL AND price > 100 AND quantity > 0 + +-- Filter with multiple logical operations +(status = 'pending' OR status = 'processing') AND NOT (priority = 'low') +``` + +## Common Pitfalls + +1. **String Quoting**: Always use single quotes for string literals. Double quotes are reserved for column identifiers. + ```sql + -- Correct + name = 'John' + + -- Incorrect + name = "John" + ``` + +2. **Wildcard Usage**: The `%` wildcard in LIKE patterns can only appear at the end. + ```sql + -- Correct + name LIKE 'John%' + + -- Incorrect (will raise an error) + name LIKE '%John%' + ``` + +3. **Case Sensitivity**: Boolean literals (`true`/`false`) are case insensitive, but string comparisons are case sensitive. + ```sql + -- All valid + is_active = true + is_active = TRUE + is_active = True + + -- Case sensitive + status = 'Active' -- Will not match 'active' + ``` + +## Best Practices +1. For complex use cases, use the primary [Expression DSL](expression-dsl.md) +2. When using multiple conditions, consider the order of operations (NOT > AND > OR) +3. For string comparisons, be consistent with case usage \ No newline at end of file From 033546344061a61b97f6d75c64e34ef72cd57ae4 Mon Sep 17 00:00:00 2001 From: Ethan Knox Date: Fri, 4 Apr 2025 15:03:09 -0400 Subject: [PATCH 2/7] Update mkdocs/docs/expression-dsl.md Co-authored-by: Fokko Driesprong --- mkdocs/docs/expression-dsl.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/mkdocs/docs/expression-dsl.md b/mkdocs/docs/expression-dsl.md index fa0a53f656..637d29383c 100644 --- a/mkdocs/docs/expression-dsl.md +++ b/mkdocs/docs/expression-dsl.md @@ -1,3 +1,20 @@ + + # Expression DSL The PyIceberg library provides a powerful expression DSL (Domain Specific Language) for building complex row filter expressions. This guide will help you understand how to use the expression DSL effectively. This DSL allows you to build type-safe expressions for use in the `row_filter` scan argument. From 4a6101964b918e8ac5b913aaaa3afa545c3aabdb Mon Sep 17 00:00:00 2001 From: Ethan Knox Date: Fri, 4 Apr 2025 15:03:15 -0400 Subject: [PATCH 3/7] Update mkdocs/docs/row-filter-syntax.md Co-authored-by: Fokko Driesprong --- mkdocs/docs/row-filter-syntax.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/mkdocs/docs/row-filter-syntax.md b/mkdocs/docs/row-filter-syntax.md index 53844ec098..7bbc98d160 100644 --- a/mkdocs/docs/row-filter-syntax.md +++ b/mkdocs/docs/row-filter-syntax.md @@ -1,3 +1,20 @@ + + # Row Filter Syntax In addtion to the primary [Expression DSL](expression-dsl.md), PyIceberg provides a string-based statement interface for filtering rows in Iceberg tables. This guide explains the syntax and provides examples for supported operations. From 5bb4be961161f5f671b56931e3e534a55439ce17 Mon Sep 17 00:00:00 2001 From: Ethan Knox Date: Tue, 15 Apr 2025 10:51:35 -0400 Subject: [PATCH 4/7] Update mkdocs/docs/expression-dsl.md Co-authored-by: Fokko Driesprong --- mkdocs/docs/expression-dsl.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/expression-dsl.md b/mkdocs/docs/expression-dsl.md index 637d29383c..333930cc38 100644 --- a/mkdocs/docs/expression-dsl.md +++ b/mkdocs/docs/expression-dsl.md @@ -74,7 +74,7 @@ age_in_set = In("age", [18, 19, 20]) age_not_in_set = NotIn("age", [18, 19, 20]) ``` -#### Null Predicates +#### Unary Predicates ```python from pyiceberg.expressions import IsNull, NotNull From c1eef4b47374514027fab1eb021b8ed34b195764 Mon Sep 17 00:00:00 2001 From: Ethan Knox Date: Tue, 15 Apr 2025 10:51:46 -0400 Subject: [PATCH 5/7] Update mkdocs/docs/expression-dsl.md Co-authored-by: Fokko Driesprong --- mkdocs/docs/expression-dsl.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs/docs/expression-dsl.md b/mkdocs/docs/expression-dsl.md index 333930cc38..f490bab99e 100644 --- a/mkdocs/docs/expression-dsl.md +++ b/mkdocs/docs/expression-dsl.md @@ -38,7 +38,7 @@ age_field = Reference("age") Predicates are expressions that evaluate to a boolean value. They can be combined using logical operators. -#### Comparison Predicates +#### Literal Predicates ```python from pyiceberg.expressions import EqualTo, NotEqualTo, LessThan, LessThanOrEqual, GreaterThan, GreaterThanOrEqual From 5b347e440fa74c50b9cf5f2fee7fd30a6c6b9430 Mon Sep 17 00:00:00 2001 From: Ethan Knox Date: Tue, 15 Apr 2025 10:52:39 -0400 Subject: [PATCH 6/7] Update mkdocs/docs/row-filter-syntax.md Co-authored-by: Fokko Driesprong --- mkdocs/docs/row-filter-syntax.md | 1 + 1 file changed, 1 insertion(+) diff --git a/mkdocs/docs/row-filter-syntax.md b/mkdocs/docs/row-filter-syntax.md index 7bbc98d160..45ce195e53 100644 --- a/mkdocs/docs/row-filter-syntax.md +++ b/mkdocs/docs/row-filter-syntax.md @@ -166,6 +166,7 @@ price IS NOT NULL AND price > 100 AND quantity > 0 ``` ## Best Practices + 1. For complex use cases, use the primary [Expression DSL](expression-dsl.md) 2. When using multiple conditions, consider the order of operations (NOT > AND > OR) 3. For string comparisons, be consistent with case usage \ No newline at end of file From f8ee2962aa5d276cee8f3d1d1c45668e9a7a72c0 Mon Sep 17 00:00:00 2001 From: Ethan Knox Date: Tue, 15 Apr 2025 11:07:38 -0400 Subject: [PATCH 7/7] added NaN --- mkdocs/docs/expression-dsl.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/mkdocs/docs/expression-dsl.md b/mkdocs/docs/expression-dsl.md index f490bab99e..e8d551d0e6 100644 --- a/mkdocs/docs/expression-dsl.md +++ b/mkdocs/docs/expression-dsl.md @@ -14,7 +14,7 @@ - See the License for the specific language governing permissions and - limitations under the License. --> - + # Expression DSL The PyIceberg library provides a powerful expression DSL (Domain Specific Language) for building complex row filter expressions. This guide will help you understand how to use the expression DSL effectively. This DSL allows you to build type-safe expressions for use in the `row_filter` scan argument. @@ -200,13 +200,11 @@ complex_filter = And(age_range, status_filter) ## Common Pitfalls -1. **Type Mismatches**: Always ensure that the types of your literals match the field types in your schema. - -2. **Null Handling**: Be careful when using `IsNull` and `NotNull` predicates with required fields. The expression DSL will automatically optimize these cases: - - `IsNull` on a required field will always return `False` - - `NotNull` on a required field will always return `True` +1. **Null Handling**: Be careful when using `IsNull` and `NotNull` predicates with required fields. The expression DSL will automatically optimize these cases: + - `IsNull` (and `IsNaN` for doubles/floats) on a required field will always return `False` + - `NotNull` (and `NotNaN` for doubles/floats) on a required field will always return `True` -3. **String Comparisons**: When using string predicates like `StartsWith`, ensure that the field type is actually a string type. +2. **String Comparisons**: When using string predicates like `StartsWith`, ensure that the field type is actually a string type. ## Examples