Skip to content

Commit 85ebf3a

Browse files
FokkoFokko
committed
Merge branch 'main' of github.com:apache/iceberg-python into fd-try-azure-fs
2 parents ae16b3d + d1fea5c commit 85ebf3a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

79 files changed

+5310
-2968
lines changed

‎.asf.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,10 @@ github:
3434
rebase: true
3535
protected_branches:
3636
main:
37+
required_status_checks:
38+
# strict means "Require branches to be up to date before merging".
39+
strict: true
40+
3741
required_pull_request_reviews:
3842
required_approving_review_count: 1
3943

‎.github/dependabot.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ updates:
2323
directory: "/"
2424
schedule:
2525
interval: "daily"
26-
open-pull-requests-limit: 5
26+
open-pull-requests-limit: 50
2727
- package-ecosystem: "github-actions"
2828
directory: "/"
2929
schedule:

‎.github/workflows/nightly-pypi-build.yml‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: "Nightly PyPI Build"
21+
22+
on:
23+
schedule:
24+
- cron: "0 0 * * *" # Runs at midnight UTC every day
25+
workflow_dispatch: # Allows manual triggering
26+
27+
jobs:
28+
set-version:
29+
if: github.repository == 'apache/iceberg-python' # Only run for apache repo
30+
runs-on: ubuntu-latest
31+
outputs:
32+
VERSION: ${{ steps.set-version.outputs.VERSION }}
33+
steps:
34+
- uses: actions/checkout@v4
35+
with:
36+
fetch-depth: 1
37+
38+
- uses: actions/setup-python@v5
39+
with:
40+
python-version: 3.12
41+
42+
- name: Install Poetry
43+
run: make install-poetry
44+
45+
- name: Set version
46+
id: set-version
47+
run: |
48+
CURRENT_VERSION=$(poetry version --short)
49+
TIMESTAMP=$(date +%Y%m%d%H%M%S)
50+
echo "VERSION=${CURRENT_VERSION}.dev${TIMESTAMP}" >> "$GITHUB_OUTPUT"
51+
52+
- name: Debug version
53+
run: echo "Publishing version ${{ steps.set-version.outputs.VERSION }}"
54+
55+
nightly-build:
56+
needs: set-version
57+
uses: ./.github/workflows/pypi-build-artifacts.yml
58+
with:
59+
version: ${{ needs.set-version.outputs.VERSION }}
60+
testpypi-publish:
61+
name: Publish to TestPypi
62+
needs:
63+
- nightly-build
64+
runs-on: ubuntu-latest
65+
environment:
66+
name: testpypi
67+
url: https://test.pypi.org/p/pyiceberg
68+
69+
permissions:
70+
id-token: write # IMPORTANT: mandatory for trusted publishing
71+
72+
steps:
73+
- name: Download all the artifacts
74+
uses: actions/download-artifact@v4
75+
with:
76+
merge-multiple: true
77+
path: dist/
78+
- name: List downloaded artifacts
79+
run: ls -R dist/
80+
- name: Publish to TestPyPI
81+
uses: pypa/gh-action-pypi-publish@release/v1
82+
with:
83+
repository-url: https://test.pypi.org/legacy/
84+
skip-existing: true
85+
verbose: true

‎.github/workflows/pypi-build-artifacts.yml‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
#
2+
# Licensed to the Apache Software Foundation (ASF) under one
3+
# or more contributor license agreements. See the NOTICE file
4+
# distributed with this work for additional information
5+
# regarding copyright ownership. The ASF licenses this file
6+
# to you under the Apache License, Version 2.0 (the
7+
# "License"); you may not use this file except in compliance
8+
# with the License. You may obtain a copy of the License at
9+
#
10+
# http://www.apache.org/licenses/LICENSE-2.0
11+
#
12+
# Unless required by applicable law or agreed to in writing,
13+
# software distributed under the License is distributed on an
14+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
# KIND, either express or implied. See the License for the
16+
# specific language governing permissions and limitations
17+
# under the License.
18+
#
19+
20+
name: "Build PyPI Artifacts"
21+
22+
on:
23+
workflow_call:
24+
inputs:
25+
VERSION:
26+
required: true
27+
type: string
28+
29+
jobs:
30+
pypi-build-artifacts:
31+
name: Build artifacts for PyPi on ${{ matrix.os }}
32+
runs-on: ${{ matrix.os }}
33+
strategy:
34+
matrix:
35+
os: [ ubuntu-22.04, windows-2022, macos-13, macos-14 ]
36+
37+
steps:
38+
- uses: actions/checkout@v4
39+
with:
40+
fetch-depth: 1
41+
42+
- uses: actions/setup-python@v5
43+
with:
44+
python-version: |
45+
3.9
46+
3.10
47+
3.11
48+
3.12
49+
50+
- name: Install poetry
51+
run: make install-poetry
52+
53+
- name: Set version with RC
54+
env:
55+
VERSION: ${{ inputs.VERSION }}
56+
run: python -m poetry version "${{ env.VERSION }}"
57+
58+
# Publish the source distribution with the version that's in
59+
# the repository, otherwise the tests will fail
60+
- name: Compile source distribution
61+
run: python3 -m poetry build --format=sdist
62+
if: startsWith(matrix.os, 'ubuntu')
63+
64+
- name: Build wheels
65+
uses: pypa/cibuildwheel@v2.22.0
66+
with:
67+
output-dir: wheelhouse
68+
config-file: "pyproject.toml"
69+
env:
70+
# Ignore 32 bit architectures
71+
CIBW_ARCHS: "auto64"
72+
CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9,<3.13"
73+
CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1"
74+
CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py"
75+
# Ignore tests for pypy since not all dependencies are compiled for it
76+
# and would require a local rust build chain
77+
CIBW_TEST_SKIP: "pp*"
78+
79+
- name: Add source distribution
80+
if: startsWith(matrix.os, 'ubuntu')
81+
run: ls -lah dist/* && cp dist/* wheelhouse/
82+
83+
- uses: actions/upload-artifact@v4
84+
with:
85+
name: "pypi-release-candidate-${{ matrix.os }}"
86+
path: ./wheelhouse/*
87+
88+
pypi-merge-artifacts:
89+
runs-on: ubuntu-latest
90+
needs:
91+
- pypi-build-artifacts
92+
steps:
93+
- name: Merge Artifacts
94+
uses: actions/upload-artifact/merge@v4
95+
with:
96+
name: "pypi-release-candidate-${{ inputs.VERSION }}"
97+
pattern: pypi-release-candidate*
98+
delete-merged: true

‎.github/workflows/python-release.yml‎

Lines changed: 94 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -17,84 +17,117 @@
1717
# under the License.
1818
#
1919

20-
name: "Python Release"
20+
name: "Python Build Release Candidate"
2121

2222
on:
23+
push:
24+
tags:
25+
# Trigger this workflow when tag follows the versioning format: pyiceberg-<major>.<minor>.<patch>rc<release_candidate>
26+
# Example valid tags: pyiceberg-0.8.0rc2, pyiceberg-1.0.0rc1
27+
- 'pyiceberg-[0-9]+.[0-9]+.[0-9]+rc[0-9]+'
2328
workflow_dispatch:
2429
inputs:
2530
version:
26-
description: 'Version'
31+
description: 'Version (e.g., 0.8.0)'
2732
type: string
28-
default: 'main'
29-
33+
required: true
34+
rc:
35+
description: 'Release Candidate (RC) (e.g., 1)'
36+
type: number
37+
required: true
3038

3139
jobs:
32-
build_wheels:
33-
name: Build wheels on ${{ matrix.os }}
34-
runs-on: ${{ matrix.os }}
35-
strategy:
36-
matrix:
37-
os: [ ubuntu-22.04, windows-2022, macos-13, macos-14, macos-15 ]
40+
validate-inputs:
41+
runs-on: ubuntu-latest
42+
outputs:
43+
VERSION: ${{ steps.validate-inputs.outputs.VERSION }} # e.g. 0.8.0
44+
RC: ${{ steps.validate-inputs.outputs.RC }} # e.g. 1
45+
steps:
46+
- name: Validate and Extract Version and RC
47+
id: validate-inputs
48+
run: |
49+
if [ "$GITHUB_EVENT_NAME" = "push" ]; then
50+
echo "Workflow triggered by tag push."
51+
TAG=${GITHUB_REF#refs/tags/} # Extract the tag name
52+
VERSION_AND_RC=${TAG#pyiceberg-} # Remove the 'pyiceberg-' prefix
53+
VERSION=${VERSION_AND_RC%rc*} # Extract VERSION by removing everything after 'rc'
54+
RC=${VERSION_AND_RC#*rc} # Extract RC by keeping everything after 'rc'
55+
56+
if [[ -z "$VERSION" || -z "$RC" ]]; then
57+
echo "Error: Unable to parse VERSION or RC from tag ($TAG). Ensure the tag format is correct."
58+
exit 1
59+
fi
60+
else
61+
echo "Workflow triggered manually via workflow_dispatch."
62+
VERSION="${{ github.event.inputs.version }}"
63+
RC="${{ github.event.inputs.rc }}"
64+
65+
# Validate version (e.g., 1.0.0)
66+
if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
67+
echo "Error: version ($VERSION) must be in the format: <number>.<number>.<number>"
68+
exit 1
69+
fi
3870
71+
# Validate rc (e.g., 1)
72+
if [[ ! "$RC" =~ ^[0-9]+$ ]]; then
73+
echo "Error: rc ($RC) must be in the format: <number>"
74+
exit 1
75+
fi
76+
fi
77+
78+
# Export variables for future steps
79+
echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
80+
echo "RC=$RC" >> $GITHUB_OUTPUT
81+
82+
- name: Display Extracted Version and RC
83+
run: |
84+
echo "Using Version: ${{ steps.validate-inputs.outputs.VERSION }}"
85+
echo "Using RC: ${{ steps.validate-inputs.outputs.RC }}"
86+
87+
validate-library-version:
88+
runs-on: ubuntu-latest
89+
needs:
90+
- validate-inputs
3991
steps:
4092
- uses: actions/checkout@v4
4193
with:
42-
fetch-depth: 0
94+
fetch-depth: 1
4395

4496
- uses: actions/setup-python@v5
4597
with:
46-
python-version: |
47-
3.9
48-
3.10
49-
3.11
50-
3.12
98+
python-version: 3.12
5199

52-
- name: Install poetry
53-
run: pip install poetry
100+
- name: Install Poetry
101+
run: make install-poetry
54102

55-
- name: Set version
56-
run: python -m poetry version "${{ inputs.version }}"
57-
if: "${{ github.event.inputs.version != 'main' }}"
58-
59-
# Publish the source distribution with the version that's in
60-
# the repository, otherwise the tests will fail
61-
- name: Compile source distribution
62-
run: python3 -m poetry build --format=sdist
63-
if: startsWith(matrix.os, 'ubuntu')
64-
65-
- name: Build wheels
66-
uses: pypa/cibuildwheel@v2.22.0
67-
with:
68-
output-dir: wheelhouse
69-
config-file: "pyproject.toml"
103+
- name: Validate current pyiceberg version
70104
env:
71-
# Ignore 32 bit architectures
72-
CIBW_ARCHS: "auto64"
73-
CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9,<3.13"
74-
CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1"
75-
CIBW_TEST_EXTRAS: "s3fs,glue"
76-
CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py"
77-
# There is an upstream issue with installing on MacOSX
78-
# https://github.com/pypa/cibuildwheel/issues/1603
79-
# Ignore tests for pypy since not all dependencies are compiled for it
80-
# and would require a local rust build chain
81-
CIBW_TEST_SKIP: "pp* *macosx*"
105+
VERSION: ${{ needs.validate-inputs.outputs.VERSION }}
106+
run: |
107+
# Extract the current version from Poetry
108+
current_pyiceberg_version=$(poetry version --short)
109+
echo "Detected Poetry version: $current_pyiceberg_version"
82110
83-
- name: Add source distribution
84-
if: startsWith(matrix.os, 'ubuntu')
85-
run: ls -lah dist/* && cp dist/* wheelhouse/
111+
# Compare the input version with the Poetry version
112+
if [[ "$VERSION" != "$current_pyiceberg_version" ]]; then
113+
echo "Error: Input version ($VERSION) does not match the Poetry version ($current_pyiceberg_version)"
114+
exit 1
115+
fi
86116
87-
- uses: actions/upload-artifact@v4
88-
with:
89-
name: "release-${{ matrix.os }}"
90-
path: ./wheelhouse/*
91-
merge:
92-
runs-on: ubuntu-latest
93-
needs: build_wheels
94-
steps:
95-
- name: Merge Artifacts
96-
uses: actions/upload-artifact/merge@v4
97-
with:
98-
name: "release-${{ github.event.inputs.version }}"
99-
pattern: release-*
100-
delete-merged: true
117+
# SVN
118+
svn-build-artifacts:
119+
needs:
120+
- validate-inputs
121+
- validate-library-version
122+
uses: ./.github/workflows/svn-build-artifacts.yml
123+
with:
124+
version: ${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }}
125+
126+
# PyPi
127+
pypi-build-artifacts:
128+
needs:
129+
- validate-inputs
130+
- validate-library-version
131+
uses: ./.github/workflows/pypi-build-artifacts.yml
132+
with:
133+
version: ${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }}

‎.github/workflows/stale.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ jobs:
3131
if: github.repository_owner == 'apache'
3232
runs-on: ubuntu-22.04
3333
steps:
34-
- uses: actions/stale@v9.0.0
34+
- uses: actions/stale@v9.1.0
3535
with:
3636
stale-issue-label: 'stale'
3737
exempt-issue-labels: 'not-stale'

0 commit comments

Comments
 (0)