diff --git a/uv.lock b/uv.lock index a30ac7f366d..509da46d089 100644 --- a/uv.lock +++ b/uv.lock @@ -1640,14 +1640,27 @@ wheels = [ [[package]] name = "substrait" -version = "0.24.2" +version = "0.27.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "protobuf" }, + { name = "substrait-protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/40/99/48c601b6627609ffaea473f603006ff1ded88189ccd1260f58d9a340d88c/substrait-0.24.2.tar.gz", hash = "sha256:743cc352e96b0927b2cd37cd5a8fdac0a96a68df9600bd104fc36aebd222a836", size = 185019, upload-time = "2025-09-15T14:58:29.678Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4d/73/0f3d6a1df07c7868533e78467e8b61a1559844a988c2cd0565aafa7773c2/substrait-0.27.0.tar.gz", hash = "sha256:73d6f90465c8138a10fb912c669a840abb95b04060c7d31faf331e38901bfb0a", size = 173217, upload-time = "2026-01-29T22:51:11.743Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d3/9c/a67b8d3e10e0bb26d68325bbd3752422aa8dfcc324f2fa18adf0e6acfb15/substrait-0.24.2-py3-none-any.whl", hash = "sha256:d1d475833566fa9d67eed3273456883c0568486ccced92b524b31709d2817e19", size = 155429, upload-time = "2025-09-15T14:58:28.031Z" }, + { url = "https://files.pythonhosted.org/packages/c6/0b/d048fe689b39f35fac47bd5468b0c1f9deaa265ba5379015b8f1c3e908c3/substrait-0.27.0-py3-none-any.whl", hash = "sha256:09fbb748d61ab9d02149e518c8d282ca4db8caebfbe60c1660b23a28007bbc85", size = 90516, upload-time = "2026-01-29T22:51:10.705Z" }, +] + +[[package]] +name = "substrait-protobuf" +version = "0.79.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/8d/0d7ff0e56fdf341182729da4ce5754712ad3b950e0e404ec2fa53f46627b/substrait_protobuf-0.79.0.tar.gz", hash = "sha256:f1bf8641eb3ee785f4affd4f728568958deda2bb1ededa09f02f2118229616d4", size = 74840, upload-time = "2026-01-29T00:08:55.587Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/27/24e16471dc7b5986cd2af0d5aab9b12347fb835840fb2320b1b2375e53a8/substrait_protobuf-0.79.0-py3-none-any.whl", hash = "sha256:6f710459569be0b92661e16a2d180a9f2a03853c3ef0154a0c6c8a9aef365ffa", size = 83741, upload-time = "2026-01-29T00:08:54.03Z" }, ] [[package]] diff --git a/vortex-python/python/vortex/arrow/expression.py b/vortex-python/python/vortex/arrow/expression.py index 8a498d597ae..92c7e87638f 100644 --- a/vortex-python/python/vortex/arrow/expression.py +++ b/vortex-python/python/vortex/arrow/expression.py @@ -6,7 +6,7 @@ import pyarrow as pa import pyarrow.compute as pc from substrait.proto import ( # pyright: ignore[reportMissingTypeStubs] - ExtendedExpression, # pyright: ignore[reportAttributeAccessIssue, reportUnknownVariableType] + ExtendedExpression, ) from vortex._lib.expr import Expr # pyright: ignore[reportMissingModuleSource] @@ -46,10 +46,10 @@ def _schema_for_substrait(schema: pa.Schema) -> pa.Schema: def arrow_to_vortex(arrow_expression: pc.Expression, schema: pa.Schema) -> Expr: compat_schema = _schema_for_substrait(schema) - substrait_object = ExtendedExpression() # pyright: ignore[reportUnknownVariableType] - substrait_object.ParseFromString(arrow_expression.to_substrait(compat_schema)) # pyright: ignore[reportUnknownMemberType] + substrait_object = ExtendedExpression() + substrait_object.ParseFromString(bytes(arrow_expression.to_substrait(compat_schema))) # pyright: ignore[reportUnusedCallResult] - expressions = extended_expression(substrait_object) # pyright: ignore[reportUnknownArgumentType] + expressions = extended_expression(substrait_object) if len(expressions) < 0 or len(expressions) > 1: raise ValueError("arrow_to_vortex: extended expression must have exactly one child") diff --git a/vortex-python/python/vortex/substrait.py b/vortex-python/python/vortex/substrait.py index 6d72ce65d1f..59d9b9eb80a 100644 --- a/vortex-python/python/vortex/substrait.py +++ b/vortex-python/python/vortex/substrait.py @@ -3,12 +3,31 @@ import operator from collections.abc import Callable +from typing import TYPE_CHECKING from google.protobuf.internal.containers import RepeatedCompositeFieldContainer -from substrait.gen.proto.algebra_pb2 import Expression, FunctionArgument -from substrait.gen.proto.extended_expression_pb2 import ExpressionReference, ExtendedExpression -from substrait.gen.proto.extensions.extensions_pb2 import SimpleExtensionDeclaration, SimpleExtensionURI -from substrait.gen.proto.type_pb2 import NamedStruct + +if TYPE_CHECKING: + from substrait.algebra_pb2 import Expression, FunctionArgument + from substrait.extended_expression_pb2 import ExpressionReference, ExtendedExpression + from substrait.extensions.extensions_pb2 import ( + SimpleExtensionDeclaration, + SimpleExtensionURI, # pyright: ignore[reportDeprecated] + ) + from substrait.type_pb2 import NamedStruct +else: + try: + # substrait >= 0.27 + from substrait.algebra_pb2 import Expression, FunctionArgument + from substrait.extended_expression_pb2 import ExpressionReference, ExtendedExpression + from substrait.extensions.extensions_pb2 import SimpleExtensionDeclaration, SimpleExtensionURI + from substrait.type_pb2 import NamedStruct + except ImportError: + # substrait < 0.27 + from substrait.gen.proto.algebra_pb2 import Expression, FunctionArgument + from substrait.gen.proto.extended_expression_pb2 import ExpressionReference, ExtendedExpression + from substrait.gen.proto.extensions.extensions_pb2 import SimpleExtensionDeclaration, SimpleExtensionURI + from substrait.gen.proto.type_pb2 import NamedStruct from ._lib import dtype as _dtype # pyright: ignore[reportMissingModuleSource] from ._lib import expr as _expr # pyright: ignore[reportMissingModuleSource] @@ -150,7 +169,7 @@ def function_argument( def extension_function( substrait_object: SimpleExtensionDeclaration.ExtensionFunction, - extension_uris: RepeatedCompositeFieldContainer[SimpleExtensionURI], + extension_uris: RepeatedCompositeFieldContainer[SimpleExtensionURI], # pyright: ignore[reportDeprecated] ) -> Callable[..., _expr.Expr]: # https://github.com/substrait-io/substrait/blob/main/proto/substrait/extensions/extensions.proto#L57 match extension_uris[substrait_object.extension_uri_reference].uri: