Skip to content

Commit 248b78a

Browse files
committed
refactor(expressions): remove residual evaluator caching
1 parent cd25393 commit 248b78a

File tree

1 file changed

+7
-32
lines changed

1 file changed

+7
-32
lines changed

pyiceberg/expressions/visitors.py

Lines changed: 7 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,9 @@
1414
# KIND, either express or implied. See the License for the
1515
# specific language governing permissions and limitations
1616
# under the License.
17-
import copy
1817
import math
19-
import threading
2018
from abc import ABC, abstractmethod
21-
from collections.abc import Callable, Hashable
19+
from collections.abc import Callable
2220
from functools import singledispatch
2321
from typing import (
2422
Any,
@@ -27,9 +25,6 @@
2725
TypeVar,
2826
)
2927

30-
from cachetools import LRUCache, cached
31-
from cachetools.keys import hashkey
32-
3328
from pyiceberg.conversions import from_bytes
3429
from pyiceberg.expressions import (
3530
AlwaysFalse,
@@ -1975,37 +1970,17 @@ def residual_for(self, partition_data: Record) -> BooleanExpression:
19751970
return self.expr
19761971

19771972

1978-
_DEFAULT_RESIDUAL_EVALUATOR_CACHE_SIZE = 128
1979-
1980-
1981-
def _residual_evaluator_cache_key(
1982-
spec: PartitionSpec, expr: BooleanExpression, case_sensitive: bool, schema: Schema
1983-
) -> tuple[Hashable, ...]:
1984-
return hashkey(spec.spec_id, repr(expr), case_sensitive, schema.schema_id)
1985-
1986-
1987-
@cached(
1988-
cache=LRUCache(maxsize=_DEFAULT_RESIDUAL_EVALUATOR_CACHE_SIZE),
1989-
key=_residual_evaluator_cache_key,
1990-
lock=threading.RLock(),
1991-
)
1992-
def _cached_residual_evaluator_template(
1993-
spec: PartitionSpec, expr: BooleanExpression, case_sensitive: bool, schema: Schema
1994-
) -> ResidualEvaluator:
1995-
return (
1996-
UnpartitionedResidualEvaluator(schema=schema, expr=expr)
1997-
if spec.is_unpartitioned()
1998-
else ResidualEvaluator(spec=spec, expr=expr, schema=schema, case_sensitive=case_sensitive)
1999-
)
2000-
2001-
20021973
def residual_evaluator_of(
20031974
spec: PartitionSpec, expr: BooleanExpression, case_sensitive: bool, schema: Schema
20041975
) -> ResidualEvaluator:
20051976
"""Create a residual evaluator.
20061977
2007-
Always returns a fresh evaluator instance because evaluators are stateful
1978+
Returns a fresh evaluator instance because evaluators are stateful
20081979
(they set `self.struct` during evaluation) and may be used from multiple
20091980
threads.
20101981
"""
2011-
return copy.copy(_cached_residual_evaluator_template(spec=spec, expr=expr, case_sensitive=case_sensitive, schema=schema))
1982+
return (
1983+
UnpartitionedResidualEvaluator(schema=schema, expr=expr)
1984+
if spec.is_unpartitioned()
1985+
else ResidualEvaluator(spec=spec, expr=expr, schema=schema, case_sensitive=case_sensitive)
1986+
)

0 commit comments

Comments
 (0)