diff --git a/.github/workflows/yetus-general-check.yml b/.github/workflows/yetus-general-check.yml new file mode 100644 index 000000000000..3a664ea8270a --- /dev/null +++ b/.github/workflows/yetus-general-check.yml @@ -0,0 +1,142 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# yamllint disable rule:line-length +--- +name: Yetus General Check + +"on": + pull_request: + types: [opened, synchronize, reopened] + +permissions: + checks: write + contents: read + pull-requests: write + statuses: write + +jobs: + general-check: + runs-on: ubuntu-latest + timeout-minutes: 600 + + env: + YETUS_VERSION: '0.15.0' + + steps: + - name: Checkout HBase + uses: actions/checkout@v4 + with: + path: src + fetch-depth: 0 + + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + + - name: Maven cache + uses: actions/cache@v4 + with: + path: ~/.m2 + key: hbase-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: | + hbase-m2- + + - name: Download Yetus + run: | + mkdir -p yetus + cd yetus + bash "${{ github.workspace }}/src/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \ + --keys 'https://downloads.apache.org/yetus/KEYS' \ + --verify-tar-gz \ + ./apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz \ + yetus/${{ env.YETUS_VERSION }}/apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz + + - name: Run Yetus General Check + env: + ARCHIVE_PATTERN_LIST: "TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump" + DOCKERFILE: "${{ github.workspace }}/src/dev-support/docker/Dockerfile" + GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USER: ${{ github.actor }} + PATCHDIR: "${{ github.workspace }}/yetus-general-check/output" + PLUGINS: "all,-javadoc,-jira,-shadedjars,-unit" + SET_JAVA_HOME: "/usr/lib/jvm/java-11" + SOURCEDIR: "${{ github.workspace }}/src" + TESTS_FILTER: "checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop" + YETUSDIR: "${{ github.workspace }}/yetus" + AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc" + BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt" + BLANKS_TABS_IGNORE_FILE: "dev-support/blanks-tabs-ignore.txt" + EXCLUDE_TESTS_URL: "https://ci-hbase.apache.org/job/HBase-Find-Flaky-Tests/job/${{ github.base_ref }}/lastSuccessfulBuild/artifact/output/excludes" + BUILD_THREAD: "4" + SUREFIRE_FIRST_PART_FORK_COUNT: "1.0C" + SUREFIRE_SECOND_PART_FORK_COUNT: "0.5C" + BRANCH_NAME: "${{ github.base_ref }}" + DEBUG: 'true' + run: | + cd "${{ github.workspace }}" + bash src/dev-support/jenkins_precommit_github_yetus.sh + + - name: Hide Old Comments + if: always() + env: + DEBUG: 'true' + run: | + # github-actions[bot] has a known user ID: 41898282 + # Verify: curl -s "https://api.github.com/users/github-actions%5Bbot%5D" | jq .id + export DEBUG="${DEBUG:-false}" + export GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" + export BUILD_BOT_USER_ID="41898282" + export JOB_NAME="HBase-PreCommit-GH-Actions-PR" + export REPO="${{ github.repository }}" + bash src/dev-support/gh_hide_old_comments.sh ${{ github.event.pull_request.number }} || true + + - name: Create Nightlies Directory Structure + # this version of rsync doesn't support --mkdirs so we do this step manually + if: always() + run: | + eval $(ssh-agent -s) + ssh-add - <<< "${{ secrets.NIGHTLIES_RSYNC_KEY }}" + ssh -p ${{ secrets.NIGHTLIES_RSYNC_PORT }} \ + -o StrictHostKeyChecking=no \ + ${{ secrets.NIGHTLIES_RSYNC_USER }}@${{ secrets.NIGHTLIES_RSYNC_HOST }} \ + "mkdir -p ${{ secrets.NIGHTLIES_RSYNC_PATH }}/hbase/HBase-PreCommit-GH-Actions-PR/PR-${{ github.event.pull_request.number }}/${{ github.run_number }}" + ssh-agent -k + + - name: Publish to Nightlies + if: always() + uses: burnett01/rsync-deployments@0dc935cdecc5f5e571865e60d2a6cdc673704823 + with: + switches: -avzr --no-o + path: yetus-general-check + remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/hbase/HBase-PreCommit-GH-Actions-PR/PR-${{ github.event.pull_request.number }}/${{ github.run_number }}/ + remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} + remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} + remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} + remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} + + - name: Publish Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: yetus-general-check-output + path: ${{ github.workspace }}/yetus-general-check/output + retention-days: 7 diff --git a/dev-support/gh_hide_old_comments.sh b/dev-support/gh_hide_old_comments.sh index 61217cfa241b..1e96918db979 100755 --- a/dev-support/gh_hide_old_comments.sh +++ b/dev-support/gh_hide_old_comments.sh @@ -104,11 +104,16 @@ function identify_most_recent_build_number { local pr="$1" local comments_file="$2" local jq_filter + local url_pattern="${JOB_NAME}/job/PR-${pr}/(?[0-9]+)/" + # GitHub Actions URLs don't have /job/ in them + if [[ "${JOB_NAME}" == *"GH-Actions"* ]]; then + url_pattern="${JOB_NAME}/PR-${pr}/(?[0-9]+)/" + fi read -r -d '' jq_filter << EOF || : .[] \ | select(.user.id == ${BUILD_BOT_USER_ID}) \ | .body \ -| capture("${JOB_NAME}/job/PR-${pr}/(?[0-9]+)/") \ +| capture("${url_pattern}") \ | .buildnum EOF @@ -122,10 +127,15 @@ function identify_old_comment_ids { local comments_file="$2" local most_recent_build_number="$3" local jq_filter + local url_pattern="${JOB_NAME}/job/PR-${pr}/(?[0-9]+)/" + # GitHub Actions URLs don't have /job/ in them + if [[ "${JOB_NAME}" == *"GH-Actions"* ]]; then + url_pattern="${JOB_NAME}/PR-${pr}/(?[0-9]+)/" + fi read -r -d '' jq_filter << EOF || : .[] \ | select(.user.id == ${BUILD_BOT_USER_ID}) \ -| { node_id, buildnum: (.body | capture("${JOB_NAME}/job/PR-${pr}/(?[0-9]+)/") | .buildnum | tonumber) } \ +| { node_id, buildnum: (.body | capture("${url_pattern}") | .buildnum | tonumber) } \ | select(.buildnum < (${most_recent_build_number} | tonumber)) \ | .node_id EOF diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index 9a5d34cc2138..da19e1b6924b 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -88,6 +88,20 @@ function personality_globals # See HBASE-19902 for how we arrived at 20g. #shellcheck disable=SC2034 DOCKERMEMLIMIT=20g + + # Override BUILD_URL for GitHub Actions to point to nightlies + # instead of the GitHub Actions run page + # The robot sets BUILD_URL to the GHA run page, but we want nightlies URLs + if [[ "${GITHUB_ACTIONS}" == "true" ]] && [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then + local pr_number + # GITHUB_REF is a standard GitHub Actions environment variable + # shellcheck disable=SC2153 + pr_number=$(echo "${GITHUB_REF}" | cut -f3 -d/) + #shellcheck disable=SC2034 + BUILD_URL="https://nightlies.apache.org/hbase/HBase-PreCommit-GH-Actions-PR/PR-${pr_number}/${GITHUB_RUN_NUMBER}/" + #shellcheck disable=SC2034 + BUILD_URL_ARTIFACTS="yetus-general-check/output" + fi } ## @description Parse extra arguments required by personalities, if any. @@ -835,6 +849,19 @@ function hbaseanti_patchfile add_test_type spotless +## @description Build artifact URL for GitHub Actions +## @audience private +## @stability evolving +## @replaceable no +if ! declare -f githubactions_artifact_url >/dev/null; then + function githubactions_artifact_url + { + if [[ -n "${BUILD_URL}" ]] && [[ -n "${BUILD_URL_ARTIFACTS}" ]]; then + echo "${BUILD_URL}${BUILD_URL_ARTIFACTS}" + fi + } +fi + ## @description spotless file filter ## @audience private ## @stability evolving @@ -869,12 +896,13 @@ function spotless_rebuild count=$(${GREP} -c '\[ERROR\]' "${logfile}") if [[ ${count} -gt 0 ]]; then - add_vote_table -1 spotless "${repostatus} has ${count} errors when running spotless:check, run spotless:apply to fix." - add_footer_table spotless "@@BASE@@/${repostatus}-spotless.txt" + add_vote_table_v2 -1 spotless \ + "@@BASE@@/${repostatus}-spotless.txt" \ + "${repostatus} has ${count} errors when running spotless:check, run spotless:apply to fix." return 1 fi - add_vote_table +1 spotless "${repostatus} has no errors when running spotless:check." + add_vote_table_v2 +1 spotless "" "${repostatus} has no errors when running spotless:check." return 0 } diff --git a/dev-support/jenkins_precommit_github_yetus.sh b/dev-support/jenkins_precommit_github_yetus.sh index 2d8421fd756d..314780d441ef 100755 --- a/dev-support/jenkins_precommit_github_yetus.sh +++ b/dev-support/jenkins_precommit_github_yetus.sh @@ -31,7 +31,6 @@ declare -i missing_env=0 declare -a required_envs=( # these ENV variables define the required API with Jenkinsfile_GitHub "ARCHIVE_PATTERN_LIST" - "BUILD_URL_ARTIFACTS" "DOCKERFILE" "GITHUB_PASSWORD" "GITHUB_USER" @@ -53,6 +52,12 @@ for required_env in "${required_envs[@]}"; do fi done +# BUILD_URL_ARTIFACTS is required for Jenkins but set in personality for GitHub Actions +if [[ "${GITHUB_ACTIONS}" != "true" ]] && [[ -z "${BUILD_URL_ARTIFACTS}" ]]; then + echo "[ERROR] Required environment variable 'BUILD_URL_ARTIFACTS' is not set." + missing_env=${missing_env}+1 +fi + if [ ${missing_env} -gt 0 ]; then echo "[ERROR] Please set the required environment variables before invoking. If this error is " \ "on Jenkins, then please file a JIRA about the error."