3131 bytes : "BYTES" ,
3232 bool : "BOOL" ,
3333}
34+ # UDFs older than this many days are considered stale and will be deleted
35+ # from the anonymous dataset before creating a new UDF.
3436_UDF_CLEANUP_THRESHOLD_DAYS = 3
3537
3638
@@ -80,13 +82,13 @@ def _cleanup_old_udfs(self):
8082 """Clean up old UDFs in the anonymous dataset."""
8183 dataset = self ._session ._anon_dataset_manager .dataset
8284 routines = list (self ._session .bqclient .list_routines (dataset ))
83- seven_days_ago = datetime .datetime .now (
85+ cleanup_cutoff_time = datetime .datetime .now (
8486 datetime .timezone .utc
8587 ) - datetime .timedelta (days = _UDF_CLEANUP_THRESHOLD_DAYS )
8688
8789 for routine in routines :
8890 if (
89- routine .created < seven_days_ago
91+ routine .created < cleanup_cutoff_time
9092 and routine ._properties ["routineType" ] == "SCALAR_FUNCTION"
9193 ):
9294 try :
@@ -100,15 +102,14 @@ def _create_udf(self):
100102 self ._session ._anon_dataset_manager .generate_unique_resource_id ()
101103 )
102104
103- # Try to clean up the old Python UDFs in the anonymous dataset. Failure
104- # to clean up is logged as a warning but does not halt execution.
105105 try :
106106 # Before creating a new UDF, attempt to clean up any uncollected,
107107 # old Python UDFs residing in the anonymous dataset. These UDFs
108108 # accumulate over time and can eventually exceed resource limits.
109109 # See more from b/450913424.
110110 self ._cleanup_old_udfs ()
111111 except Exception as e :
112+ # Log a warning on the failure, do not interrupt the workflow.
112113 msg = bfe .format_message (
113114 f"Failed to clean up the old Python UDFs before creating { udf_name } : { e } "
114115 )
0 commit comments