We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 7643931 commit e7290a7Copy full SHA for e7290a7
benchmarks/collect_gil_bench.py
@@ -23,13 +23,14 @@
23
24
def run(n_batches: int = 8, batch_size: int = 1_000_000) -> None:
25
ctx = SessionContext()
26
- batches = []
+ partitions = []
27
for i in range(n_batches):
28
start = i * batch_size
29
arr = pa.array(range(start, start + batch_size))
30
- batches.append(pa.record_batch([arr], names=["a"]))
+ batch = pa.record_batch([arr], names=["a"])
31
+ partitions.append([batch]) # Each batch in its own partition
32
- df = ctx.create_dataframe([batches])
33
+ df = ctx.create_dataframe(partitions)
34
35
start = time.perf_counter()
36
df.collect()
0 commit comments