File tree Expand file tree Collapse file tree 1 file changed +6
-2
lines changed
Expand file tree Collapse file tree 1 file changed +6
-2
lines changed Original file line number Diff line number Diff line change @@ -402,7 +402,9 @@ def stream_data(
402402 SAFETY_MARGIN = (
403403 40 # Perf seems bad for large chunks, so do 40x smaller than max
404404 )
405- batch_count = data .metadata .total_bytes // (MAX_BYTES // SAFETY_MARGIN )
405+ batch_count = math .ceil (
406+ data .metadata .total_bytes / (MAX_BYTES // SAFETY_MARGIN )
407+ )
406408 rows_per_batch = math .ceil (data .metadata .row_count / batch_count )
407409
408410 schema_w_offsets = data .schema .append (
@@ -454,7 +456,9 @@ def write_data(
454456 SAFETY_MARGIN = (
455457 4 # aim for 2.5mb to account for row variance, format differences, etc.
456458 )
457- batch_count = data .metadata .total_bytes // (MAX_BYTES // SAFETY_MARGIN )
459+ batch_count = math .ceil (
460+ data .metadata .total_bytes / (MAX_BYTES // SAFETY_MARGIN )
461+ )
458462 rows_per_batch = math .ceil (data .metadata .row_count / batch_count )
459463
460464 schema_w_offsets = data .schema .append (
You can’t perform that action at this time.
0 commit comments