From 7c2f5e8652ab6cf49bbfd9915934a70ad3268725 Mon Sep 17 00:00:00 2001 From: Fokko Driesprong Date: Tue, 4 Mar 2025 23:19:02 +0100 Subject: [PATCH] CI: Limit parallelism For the tests, we want to limit parallelism to avoid creating 1-row Parquet files. --- tests/conftest.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index a0e5e74522..e9abd9bf23 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2433,6 +2433,8 @@ def spark() -> "SparkSession": spark = ( SparkSession.builder.appName("PyIceberg integration test") .config("spark.sql.session.timeZone", "UTC") + .config("spark.sql.shuffle.partitions", "1") + .config("spark.default.parallelism", "1") .config("spark.sql.extensions", "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions") .config("spark.sql.catalog.integration", "org.apache.iceberg.spark.SparkCatalog") .config("spark.sql.catalog.integration.catalog-impl", "org.apache.iceberg.rest.RESTCatalog")