From b6bdf2bcbb8f0650dfa863e8062bfb167fb91593 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 27 Nov 2024 20:33:54 -0500 Subject: [PATCH 01/10] GH-7: Add macOS, Windows CI Fixes #7. Fixes #8. --- .github/workflows/test.yml | 63 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b47404ee43..cb6a816e8c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -72,3 +72,66 @@ jobs: -e CI=true \ -e "DEVELOCITY_ACCESS_KEY=$DEVELOCITY_ACCESS_KEY" \ ${{ matrix.image }} + + macos: + name: AMD64 macOS ${{ matrix.macos }} Java JDK ${{ matrix.jdk }} + runs-on: macos-${{ matrix.macos }} + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + jdk: [11] + macos: [13, latest] + steps: + - name: Set up Java + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: ${{ matrix.jdk }} + - name: Checkout Arrow + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + - name: Build + shell: bash + env: + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + run: ci/scripts/java_build.sh $(pwd) $(pwd)/build + - name: Test + shell: bash + env: + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + run: ci/scripts/java_test.sh $(pwd) $(pwd)/build + + windows: + name: AMD64 Windows Server 2022 Java JDK ${{ matrix.jdk }} + runs-on: windows-latest + if: ${{ !contains(github.event.pull_request.title, 'WIP') }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + jdk: [11] + steps: + - name: Set up Java + uses: actions/setup-java@v4 + with: + java-version: ${{ matrix.jdk }} + distribution: 'temurin' + - name: Checkout Arrow + uses: actions/checkout@v4 + with: + fetch-depth: 0 + submodules: recursive + - name: Build + shell: bash + env: + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + run: ci/scripts/java_build.sh $(pwd) $(pwd)/build + - name: Test + shell: bash + env: + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + run: ci/scripts/java_test.sh $(pwd) $(pwd)/build \ No newline at end of file From 599ebc4120fc3ce6d1a934458677c65f34807378 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 27 Nov 2024 20:48:04 -0500 Subject: [PATCH 02/10] macOS --- ci/scripts/java_build.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh index 8441e00cc7..8c56cccb55 100755 --- a/ci/scripts/java_build.sh +++ b/ci/scripts/java_build.sh @@ -44,7 +44,10 @@ mkdir -p "${build_dir}/arrow-format" cp -r "${source_dir}/arrow-format" "${build_dir}" cp -r "${source_dir}/dev" "${build_dir}" -for source_root in $(find "${source_dir}" -not \( -path "${source_dir}"/build -prune \) -type f -name pom.xml -exec realpath -s --relative-to="${source_dir}" '{}' \; | +# Instead of hardcoding the list of directories to copy, find pom.xml and then +# crawl back up to the top. GNU realpath has --relative-to but this does not +# work on macOS +for source_root in $(find "${source_dir}" -not \( -path "${source_dir}"/build -prune \) -type f -name pom.xml -exec python -c "import os.path; print(os.path.relpath('{}', '${source_dir}'))" \; | awk -F/ '{print $1}' | sort -u); do cp -r "${source_dir}/${source_root}" "${build_dir}" From 986636bd96d020b64fff5a64ddac798f9ca03831 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 27 Nov 2024 20:59:19 -0500 Subject: [PATCH 03/10] macOS (2) --- ci/scripts/java_build.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh index 8c56cccb55..430cf38678 100755 --- a/ci/scripts/java_build.sh +++ b/ci/scripts/java_build.sh @@ -47,9 +47,15 @@ cp -r "${source_dir}/dev" "${build_dir}" # Instead of hardcoding the list of directories to copy, find pom.xml and then # crawl back up to the top. GNU realpath has --relative-to but this does not # work on macOS -for source_root in $(find "${source_dir}" -not \( -path "${source_dir}"/build -prune \) -type f -name pom.xml -exec python -c "import os.path; print(os.path.relpath('{}', '${source_dir}'))" \; | - awk -F/ '{print $1}' | - sort -u); do + +poms=$(find "${source_dir}" -not \( -path "${source_dir}"/build -prune \) -type f -name pom.xml) +if [[ "$OSTYPE" == "darwin"* ]]; then + poms=$(echo "$poms" | xargs -n1 python -c "import os.path; print(os.path.relpath('{}', '${source_dir}'))") +else + poms=$(echo "$poms" | xargs -n1 realpath -s --relative-to="${source_dir}") +fi + +for source_root in $(echo "${poms}" | awk -F/ '{print $1}' | sort -u); do cp -r "${source_dir}/${source_root}" "${build_dir}" done From c0ab13133b73e0bded06b1015b6cc6b8201d249b Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 27 Nov 2024 21:12:37 -0500 Subject: [PATCH 04/10] macOS (3) --- ci/scripts/java_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh index 430cf38678..b5a12d9171 100755 --- a/ci/scripts/java_build.sh +++ b/ci/scripts/java_build.sh @@ -50,7 +50,7 @@ cp -r "${source_dir}/dev" "${build_dir}" poms=$(find "${source_dir}" -not \( -path "${source_dir}"/build -prune \) -type f -name pom.xml) if [[ "$OSTYPE" == "darwin"* ]]; then - poms=$(echo "$poms" | xargs -n1 python -c "import os.path; print(os.path.relpath('{}', '${source_dir}'))") + poms=$(echo "$poms" | xargs -n1 python -c "import sys; import os.path; print(os.path.relpath(sys.argv[1], '${source_dir}'))") else poms=$(echo "$poms" | xargs -n1 realpath -s --relative-to="${source_dir}") fi From 76ecac2924ac2e42d65b3b2999434bb6a9ed7955 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 27 Nov 2024 21:13:25 -0500 Subject: [PATCH 05/10] macOS (4) --- .github/workflows/test.yml | 2 +- dev/release/rat_exclude_files.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cb6a816e8c..e2d0fbec77 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -134,4 +134,4 @@ jobs: shell: bash env: DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - run: ci/scripts/java_test.sh $(pwd) $(pwd)/build \ No newline at end of file + run: ci/scripts/java_test.sh $(pwd) $(pwd)/build diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt index 76cde3070b..8efd379a73 100644 --- a/dev/release/rat_exclude_files.txt +++ b/dev/release/rat_exclude_files.txt @@ -15,4 +15,5 @@ # specific language governing permissions and limitations # under the License. +.gitmodules dataset/src/test/resources/data/student.csv From 5836b623797256c5c651a8facb64edd60d6995b9 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 27 Nov 2024 21:16:33 -0500 Subject: [PATCH 06/10] pre-commit --- .gitignore | 1 - arrow-format/FlightSql.proto | 2 +- arrow-format/Schema.fbs | 4 ++-- arrow-format/substrait/extension_types.yaml | 3 +-- 4 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 3ac6ed5f88..ca0ac32463 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,3 @@ cmake_install.cmake dependency-reduced-pom.xml install_manifest.txt target/ -/.mvn/.develocity/ diff --git a/arrow-format/FlightSql.proto b/arrow-format/FlightSql.proto index ef1ae7513d..3568d851cb 100644 --- a/arrow-format/FlightSql.proto +++ b/arrow-format/FlightSql.proto @@ -1857,7 +1857,7 @@ message DoPutPreparedStatementResult { // statement should be considered invalid, and all subsequent requests for this prepared // statement must use this new handle. // The updated handle allows implementing query parameters with stateless services. - // + // // When an updated handle is not provided by the server, clients should contiue // using the previous handle provided by `ActionCreatePreparedStatementResonse`. optional bytes prepared_statement_handle = 1; diff --git a/arrow-format/Schema.fbs b/arrow-format/Schema.fbs index e8e14b112a..7ba9aaf7a8 100644 --- a/arrow-format/Schema.fbs +++ b/arrow-format/Schema.fbs @@ -216,14 +216,14 @@ table Bool { /// Contains two child arrays, run_ends and values. /// The run_ends child array must be a 16/32/64-bit integer array -/// which encodes the indices at which the run with the value in +/// which encodes the indices at which the run with the value in /// each corresponding index in the values child array ends. /// Like list/struct types, the value array can be of any type. table RunEndEncoded { } /// Exact decimal value represented as an integer value in two's -/// complement. Currently 32-bit (4-byte), 64-bit (8-byte), +/// complement. Currently 32-bit (4-byte), 64-bit (8-byte), /// 128-bit (16-byte) and 256-bit (32-byte) integers are used. /// The representation uses the endianness indicated in the Schema. table Decimal { diff --git a/arrow-format/substrait/extension_types.yaml b/arrow-format/substrait/extension_types.yaml index 0073da1acc..4c0ef8ee2f 100644 --- a/arrow-format/substrait/extension_types.yaml +++ b/arrow-format/substrait/extension_types.yaml @@ -48,7 +48,7 @@ # * Functions have the same meaning when applied to the encoded type # # Note: if two types have a different range (e.g. string and large_string) then -# they do not satisfy the above criteria and are not encodings. +# they do not satisfy the above criteria and are not encodings. # # These types will never have a Substrait equivalent. In the Substrait point # of view these are execution details. @@ -167,4 +167,3 @@ types: parameters: - name: unit type: string - From 3682d2fe0abc8b100bb7668f0986ca864a915f03 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 27 Nov 2024 21:42:36 -0500 Subject: [PATCH 07/10] Update hashFiles --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e2d0fbec77..0d4469bc5e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,7 +61,7 @@ jobs: uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: path: .docker - key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('**/docker-compose.yml') }} + key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('**/docker-compose.yml', '**'pom.xml') }} restore-keys: maven-${{ matrix.jdk }}-${{ matrix.maven }}- - name: Execute Docker Build env: From 17fdc310a44608b25123d683c1020e8009605f80 Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 27 Nov 2024 21:46:18 -0500 Subject: [PATCH 08/10] typo --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0d4469bc5e..cf8a5e7a94 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,7 +61,7 @@ jobs: uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2 with: path: .docker - key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('**/docker-compose.yml', '**'pom.xml') }} + key: maven-${{ matrix.jdk }}-${{ matrix.maven }}-${{ hashFiles('**/docker-compose.yml', '**/pom.xml') }} restore-keys: maven-${{ matrix.jdk }}-${{ matrix.maven }}- - name: Execute Docker Build env: From 193419d1aadadc8d80da6ed7cf53d5ad96cee628 Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 28 Nov 2024 06:27:09 -0500 Subject: [PATCH 09/10] tweak --- .github/workflows/test.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cf8a5e7a94..5e69aa3906 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,15 +74,19 @@ jobs: ${{ matrix.image }} macos: - name: AMD64 macOS ${{ matrix.macos }} Java JDK ${{ matrix.jdk }} + name: ${{ matrix.arch }} macOS ${{ matrix.macos }} Java JDK ${{ matrix.jdk }} runs-on: macos-${{ matrix.macos }} if: ${{ !contains(github.event.pull_request.title, 'WIP') }} timeout-minutes: 30 strategy: fail-fast: false matrix: - jdk: [11] - macos: [13, latest] + - arch: AMD64 + jdk: 11 + macos: 13 + - arch: AArch64 + jdk: 11 + macos: latest steps: - name: Set up Java uses: actions/setup-java@v4 From 5da41acbcde6f044189bf6c96cd8cb2fe691de74 Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 28 Nov 2024 06:29:16 -0500 Subject: [PATCH 10/10] syntax --- .github/workflows/test.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5e69aa3906..473ce84ca7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -81,12 +81,13 @@ jobs: strategy: fail-fast: false matrix: - - arch: AMD64 - jdk: 11 - macos: 13 - - arch: AArch64 - jdk: 11 - macos: latest + include: + - arch: AMD64 + jdk: 11 + macos: 13 + - arch: AArch64 + jdk: 11 + macos: latest steps: - name: Set up Java uses: actions/setup-java@v4