Skip to content

Commit 7b1f740

Browse files
fix batching logic
1 parent 6b9587a commit 7b1f740

File tree

1 file changed

+6
-2
lines changed

1 file changed

+6
-2
lines changed

bigframes/session/loader.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -402,7 +402,9 @@ def stream_data(
402402
SAFETY_MARGIN = (
403403
40 # Perf seems bad for large chunks, so do 40x smaller than max
404404
)
405-
batch_count = data.metadata.total_bytes // (MAX_BYTES // SAFETY_MARGIN)
405+
batch_count = math.ceil(
406+
data.metadata.total_bytes / (MAX_BYTES // SAFETY_MARGIN)
407+
)
406408
rows_per_batch = math.ceil(data.metadata.row_count / batch_count)
407409

408410
schema_w_offsets = data.schema.append(
@@ -454,7 +456,9 @@ def write_data(
454456
SAFETY_MARGIN = (
455457
4 # aim for 2.5mb to account for row variance, format differences, etc.
456458
)
457-
batch_count = data.metadata.total_bytes // (MAX_BYTES // SAFETY_MARGIN)
459+
batch_count = math.ceil(
460+
data.metadata.total_bytes / (MAX_BYTES // SAFETY_MARGIN)
461+
)
458462
rows_per_batch = math.ceil(data.metadata.row_count / batch_count)
459463

460464
schema_w_offsets = data.schema.append(

0 commit comments

Comments
 (0)