Merge branch 'main' into all_files_metadata_tables

Author: soumya-ghosh

.asf.yaml

@@ -34,6 +34,10 @@ github:
     rebase: true
   protected_branches:
     main:
+      required_status_checks:
+        # strict means "Require branches to be up to date before merging".
+        strict: true
+
       required_pull_request_reviews:
         required_approving_review_count: 1
 

.github/ISSUE_TEMPLATE/iceberg_bug_report.yml

@@ -28,7 +28,8 @@ body:
       description: What Apache Iceberg version are you using?
       multiple: false
       options:
-        - "0.8.1 (latest release)"
+        - "0.9.0 (latest release)"
+        - "0.8.1"
         - "0.8.0"
         - "0.7.1"
         - "0.7.0"

.github/dependabot.yml

@@ -22,9 +22,9 @@ updates:
   - package-ecosystem: "pip"
     directory: "/"
     schedule:
-      interval: "daily"
+      interval: "weekly"
     open-pull-requests-limit: 50
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: "daily"
+      interval: "weekly"

.github/pull_request_template.md

@@ -0,0 +1,14 @@
+<!--
+Thanks for opening a pull request!
+-->
+
+<!-- In the case this PR will resolve an issue, please replace ${GITHUB_ISSUE_ID} below with the actual Github issue id. -->
+<!-- Closes #${GITHUB_ISSUE_ID} -->
+
+# Rationale for this change
+
+# Are these changes tested?
+
+# Are there any user-facing changes?
+
+<!-- In the case of user-facing changes, please add the changelog label. -->

.github/workflows/pypi-build-artifacts.yml

@@ -62,7 +62,7 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.22.0
+        uses: pypa/cibuildwheel@v2.23.3
         with:
           output-dir: wheelhouse
           config-file: "pyproject.toml"

.github/workflows/python-ci.yml

@@ -58,9 +58,29 @@ jobs:
         python-version: ${{ matrix.python }}
         cache: poetry
         cache-dependency-path: ./poetry.lock
+    - name: Install system dependencies
+      run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
     - name: Install
       run: make install-dependencies
     - name: Linters
       run: make lint
     - name: Tests
-      run: make test-coverage
+      run: make test-coverage-unit
+
+  integration-test:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        python: ['3.9', '3.10', '3.11', '3.12']
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install system dependencies
+      run: sudo apt-get update && sudo apt-get install -y libkrb5-dev # for kerberos
+    - name: Install
+      run: make install
+    - name: Run integration tests
+      run: make test-coverage-integration
+    - name: Show debug logs
+      if: ${{ failure() }}
+      run: docker compose -f dev/docker-compose.yml logs

.github/workflows/python-integration.yml

@@ -57,7 +57,7 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.22.0
+        uses: pypa/cibuildwheel@v2.23.3
         with:
           output-dir: wheelhouse
           config-file: "pyproject.toml"

.github/workflows/svn-build-artifacts.yml

@@ -19,13 +19,20 @@
 help:  ## Display this help
 	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf "  \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
 
-install-poetry:  ## Install poetry if the user has not done that yet.
-	 @if ! command -v poetry &> /dev/null; then \
-         echo "Poetry could not be found. Installing..."; \
-         pip install --user poetry==2.0.1; \
-     else \
-         echo "Poetry is already installed."; \
-     fi
+POETRY_VERSION = 2.1.1
+install-poetry:  ## Ensure Poetry is installed and the correct version is being used.
+	@if ! command -v poetry &> /dev/null; then \
+		echo "Poetry could not be found. Installing..."; \
+		pip install --user poetry==$(POETRY_VERSION); \
+	else \
+		INSTALLED_VERSION=$$(pip show poetry | grep Version | awk '{print $$2}'); \
+		if [ "$$INSTALLED_VERSION" != "$(POETRY_VERSION)" ]; then \
+			echo "Poetry version $$INSTALLED_VERSION does not match required version $(POETRY_VERSION). Updating..."; \
+			pip install --user --upgrade poetry==$(POETRY_VERSION); \
+		else \
+			echo "Poetry version $$INSTALLED_VERSION is already installed."; \
+		fi \
+	fi
 
 install-dependencies: ## Install dependencies including dev, docs, and all extras
 	poetry install --all-extras

Makefile

@@ -1,3 +1,4 @@
+.github/*
 .rat-excludes
 build
 .git