diff --git a/.github/workflows/cuda_extra.yml b/.github/workflows/cuda_extra.yml new file mode 100644 index 00000000000..1700d6a8456 --- /dev/null +++ b/.github/workflows/cuda_extra.yml @@ -0,0 +1,136 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: CUDA Extra + +on: + push: + tags: + - '**' + pull_request: + types: + - labeled + - opened + - reopened + - synchronize + schedule: + - cron: | + 0 6 * * * + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +permissions: + actions: read + contents: read + pull-requests: read + +jobs: + check-labels: + if: github.event_name != 'schedule' || github.repository == 'apache/arrow' + uses: ./.github/workflows/check_labels.yml + secrets: inherit + with: + parent-workflow: cuda_extra + + docker: + needs: check-labels + name: ${{ matrix.title }} + runs-on: "runs-on=${{ github.run_id }}/family=g4dn.xlarge/image=ubuntu24-gpu-x64/spot=capacity-optimized" + if: >- + needs.check-labels.outputs.force == 'true' || + contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra') || + contains(fromJSON(needs.check-labels.outputs.ci-extra-labels || '[]'), 'CI: Extra: CUDA') + timeout-minutes: 75 + strategy: + fail-fast: false + matrix: + include: + - cuda: 12.9.0 + ubuntu: 24.04 + image: ubuntu-cuda-cpp + title: AMD64 Ubuntu 24 CUDA 12.9.0 + - cuda: 11.7.1 + ubuntu: 22.04 + image: ubuntu-cuda-cpp + title: AMD64 Ubuntu 22 CUDA 11.7.1 + - cuda: 12.9.0 + ubuntu: 24.04 + image: ubuntu-cuda-python + title: AMD64 Ubuntu 24 CUDA 12.9.0 Python + - cuda: 11.7.1 + ubuntu: 22.04 + image: ubuntu-cuda-python + title: AMD64 Ubuntu 22 CUDA 11.7.1 Python + env: + ARCHERY_DEBUG: 1 + ARROW_ENABLE_TIMING_TESTS: OFF + DOCKER_VOLUME_PREFIX: ".docker/" + steps: + - name: Checkout Arrow + uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: recursive + - name: Cache Docker Volumes + uses: actions/cache@v5 + with: + path: .docker + key: extra-${{ matrix.image }}-${{ hashFiles('cpp/**') }} + restore-keys: extra-${{ matrix.image }}- + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: 3 + - name: Setup Archery + run: python3 -m pip install -e dev/archery[docker] + - name: Display NVIDIA SMI details + run: | + nvidia-smi + nvidia-smi -L + nvidia-smi -q -d Memory + - name: Execute Docker Build + continue-on-error: ${{ matrix.continue-on-error || false }} + env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + CUDA: ${{ matrix.cuda }} + UBUNTU: ${{ matrix.ubuntu }} + run: | + # GH-40558: reduce ASLR to avoid ASAN/LSAN crashes + sudo sysctl -w vm.mmap_rnd_bits=28 + source ci/scripts/util_enable_core_dumps.sh + archery docker run ${{ matrix.run-options || '' }} ${{ matrix.image }} + - name: Docker Push + if: >- + success() && + github.event_name == 'push' && + github.repository == 'apache/arrow' && + github.ref_name == 'main' + env: + ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} + ARCHERY_DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + continue-on-error: true + run: archery docker push ${{ matrix.image }} + + report-extra-cpp: + if: github.event_name == 'schedule' && always() + needs: + - docker + uses: ./.github/workflows/report_ci.yml + secrets: inherit diff --git a/dev/tasks/docker-tests/github.cuda.yml b/dev/tasks/docker-tests/github.cuda.yml deleted file mode 100644 index e65ac457b2e..00000000000 --- a/dev/tasks/docker-tests/github.cuda.yml +++ /dev/null @@ -1,52 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -{% import 'macros.jinja' as macros with context %} - -{{ macros.github_header() }} - -jobs: - test: - name: | - Docker Test {{ flags|default("") }} {{ image }} {{ command|default("") }} - runs-on: ['self-hosted', 'cuda'] -{{ macros.github_set_env(env) }} - timeout-minutes: {{ timeout|default(60) }} - steps: - {{ macros.github_checkout_arrow(fetch_depth=fetch_depth|default(1))|indent }} - # python 3.10 is installed on the runner, no need to install - - name: Install pip - run: sudo apt update && sudo apt install python3-pip -y - - name: Install archery - run: python3 -m pip install -e arrow/dev/archery[docker] - - name: Execute Docker Build - shell: bash - env: - {{ macros.github_set_sccache_envvars()|indent(8) }} - run: | - source arrow/ci/scripts/util_enable_core_dumps.sh - archery docker run \ - -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" \ - {{ flags|default("") }} \ - {{ image }} \ - {{ command|default("") }} - {% if arrow.is_default_branch() %} - {{ macros.github_login_dockerhub()|indent }} - - name: Push Docker Image - shell: bash - run: archery docker push {{ image }} - {% endif %} diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml index 266073daff6..2667aa1fb5e 100644 --- a/dev/tasks/tasks.yml +++ b/dev/tasks/tasks.yml @@ -41,9 +41,6 @@ groups: {############################# Testing tasks #################################} - cuda: - - test-cuda-* - test: - test-* @@ -762,35 +759,12 @@ tasks: artifacts: - docs.tar.gz - ############################## CUDA tests ################################# - -{% for ubuntu, cuda in [("22.04", "11.7.1"), ("24.04", "13.0.2")] %} - test-cuda-cpp-ubuntu-{{ ubuntu }}-cuda-{{ cuda }}: - ci: github - template: docker-tests/github.cuda.yml - params: - env: - CUDA: {{ cuda }} - UBUNTU: {{ ubuntu }} - image: ubuntu-cuda-cpp - - test-cuda-python-ubuntu-{{ ubuntu }}-cuda-{{ cuda }}: - ci: github - template: docker-tests/github.cuda.yml - params: - env: - CUDA: {{ cuda }} - UBUNTU: {{ ubuntu }} - image: ubuntu-cuda-python -{% endfor %} - ############################## Fuzz tests ################################# test-build-cpp-fuzz: ci: github template: fuzz-tests/github.oss-fuzz.yml - ############################## vcpkg tests ################################## test-build-vcpkg-win: