diff --git a/.github/workflows/cleanup-pr-comments.yml b/.github/workflows/cleanup-pr-comments.yml new file mode 100644 index 000000000000..378dae44c101 --- /dev/null +++ b/.github/workflows/cleanup-pr-comments.yml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# yamllint disable rule:line-length +--- +name: Cleanup Old PR Comments + +"on": + schedule: + - cron: '0 */6 * * *' + workflow_dispatch: + +permissions: + pull-requests: write + issues: write + +jobs: + cleanup: + runs-on: ubuntu-latest + steps: + - name: Checkout HBase + uses: actions/checkout@v4 + + - name: Clean up old comments on recently active PRs + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUILD_BOT_USER_ID: "41898282" + JOB_NAME: "HBase-PreCommit-GH-Actions-PR" + REPO: ${{ github.repository }} + DEBUG: 'false' + run: | + # Find open PRs updated in last 7 hours (buffer beyond 6hr cron interval) + prs=$(gh pr list \ + --state open \ + --search "updated:>$(date -u -d '7 hours ago' +%Y-%m-%dT%H:%M:%SZ)" \ + --json number \ + --jq '.[].number') + + if [ -z "$prs" ]; then + echo "No recently active PRs found" + exit 0 + fi + + echo "Processing $(echo "$prs" | wc -w) recently active PRs" + for pr in $prs; do + echo "Cleaning PR #$pr" + bash dev-support/gh_hide_old_comments.sh "$pr" || true + sleep 2 + done diff --git a/.github/workflows/yetus-general-check.yml b/.github/workflows/yetus-general-check.yml index 664ef27624e4..fa2e95be3a87 100644 --- a/.github/workflows/yetus-general-check.yml +++ b/.github/workflows/yetus-general-check.yml @@ -95,44 +95,6 @@ jobs: cd "${{ github.workspace }}" bash src/dev-support/jenkins_precommit_github_yetus.sh - - name: Hide Old Comments - if: always() - env: - DEBUG: 'true' - run: | - # github-actions[bot] has a known user ID: 41898282 - # Verify: curl -s "https://api.github.com/users/github-actions%5Bbot%5D" | jq .id - export DEBUG="${DEBUG:-false}" - export GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" - export BUILD_BOT_USER_ID="41898282" - export JOB_NAME="HBase-PreCommit-GH-Actions-PR" - export REPO="${{ github.repository }}" - bash src/dev-support/gh_hide_old_comments.sh ${{ github.event.pull_request.number }} || true - - - name: Create Nightlies Directory Structure - # this version of rsync doesn't support --mkdirs so we do this step manually - if: always() - run: | - eval $(ssh-agent -s) - ssh-add - <<< "${{ secrets.NIGHTLIES_RSYNC_KEY }}" - ssh -p ${{ secrets.NIGHTLIES_RSYNC_PORT }} \ - -o StrictHostKeyChecking=no \ - ${{ secrets.NIGHTLIES_RSYNC_USER }}@${{ secrets.NIGHTLIES_RSYNC_HOST }} \ - "mkdir -p ${{ secrets.NIGHTLIES_RSYNC_PATH }}/hbase/HBase-PreCommit-GH-Actions-PR/PR-${{ github.event.pull_request.number }}/${{ github.run_number }}" - ssh-agent -k - - - name: Publish to Nightlies - if: always() - uses: burnett01/rsync-deployments@0dc935cdecc5f5e571865e60d2a6cdc673704823 - with: - switches: -avzr --no-o - path: yetus-general-check - remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/hbase/HBase-PreCommit-GH-Actions-PR/PR-${{ github.event.pull_request.number }}/${{ github.run_number }}/ - remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }} - remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }} - remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }} - remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }} - - name: Publish Test Results if: always() uses: actions/upload-artifact@v4 diff --git a/dev-support/hbase-personality.sh b/dev-support/hbase-personality.sh index 168443384dc6..9ff53a463d98 100755 --- a/dev-support/hbase-personality.sh +++ b/dev-support/hbase-personality.sh @@ -88,20 +88,6 @@ function personality_globals # See HBASE-19902 for how we arrived at 20g. #shellcheck disable=SC2034 DOCKERMEMLIMIT=20g - - # Override BUILD_URL for GitHub Actions to point to nightlies - # instead of the GitHub Actions run page - # The robot sets BUILD_URL to the GHA run page, but we want nightlies URLs - if [[ "${GITHUB_ACTIONS}" == "true" ]] && [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then - local pr_number - # GITHUB_REF is a standard GitHub Actions environment variable - # shellcheck disable=SC2153 - pr_number=$(echo "${GITHUB_REF}" | cut -f3 -d/) - #shellcheck disable=SC2034 - BUILD_URL="https://nightlies.apache.org/hbase/HBase-PreCommit-GH-Actions-PR/PR-${pr_number}/${GITHUB_RUN_NUMBER}/" - #shellcheck disable=SC2034 - BUILD_URL_ARTIFACTS="yetus-general-check/output" - fi } ## @description Parse extra arguments required by personalities, if any. @@ -849,19 +835,6 @@ function hbaseanti_patchfile add_test_type spotless -## @description Build artifact URL for GitHub Actions -## @audience private -## @stability evolving -## @replaceable no -if ! declare -f githubactions_artifact_url >/dev/null; then - function githubactions_artifact_url - { - if [[ -n "${BUILD_URL}" ]] && [[ -n "${BUILD_URL_ARTIFACTS}" ]]; then - echo "${BUILD_URL}${BUILD_URL_ARTIFACTS}" - fi - } -fi - ## @description spotless file filter ## @audience private ## @stability evolving @@ -879,6 +852,7 @@ function spotless_rebuild { local repostatus=$1 local logfile="${PATCH_DIR}/${repostatus}-spotless.txt" + local linecommentsfile="${PATCH_DIR}/${repostatus}-spotless-linecomments.txt" if ! verify_needed_test spotless; then return 0 @@ -896,6 +870,20 @@ function spotless_rebuild count=$(${GREP} -c '\[ERROR\]' "${logfile}") if [[ ${count} -gt 0 ]]; then + # Generate file-level annotations for GitHub Actions + if [[ -n "${BUGLINECOMMENTS}" ]]; then + # Extract files with violations: lines like "[ERROR] src/path/to/file.java" + # with leading whitespace after [ERROR] + ${GREP} '^\[ERROR\][[:space:]]\+[^[:space:]]' "${logfile}" \ + | ${SED} 's/^\[ERROR\][[:space:]]*//g' \ + | while read -r file; do + echo "${file}:1:Spotless formatting required, run mvn spotless:apply" + done > "${linecommentsfile}" + if [[ -s "${linecommentsfile}" ]]; then + bugsystem_linecomments_queue spotless "${linecommentsfile}" + fi + fi + add_vote_table_v2 -1 spotless \ "@@BASE@@/${repostatus}-spotless.txt" \ "${repostatus} has ${count} errors when running spotless:check, run spotless:apply to fix." @@ -926,6 +914,160 @@ function hbase_javac_logfilter ${GREP} -E '\[(ERROR|WARNING)\] /.*\.java:' "${input}" | sort > "${output}" } +## @description Override github comment generation to avoid broken links +## @description when artifact URLs are not available (fork PRs without nightlies) +## @audience private +## @stability evolving +## @replaceable no +## @param result +function github_finalreport_as_comment +{ + declare result=$1 + declare i + declare commentfile=${PATCH_DIR}/gitcommentfile.$$ + declare comment + declare url + declare ela + declare subs + declare logfile + declare calctime + declare vote + declare emoji + declare t1 + declare t2 + declare ourstring + + rm "${commentfile}" 2>/dev/null + + if [[ ${ROBOT} = "false" + || -z ${GITHUB_ISSUE} ]] ; then + return 0 + fi + + url=$(get_artifact_url) + + big_console_header "Adding comment to Github" + + if [[ ${result} == 0 ]]; then + echo ":confetti_ball: **+1 overall**" >> "${commentfile}" + else + echo ":broken_heart: **-1 overall**" >> "${commentfile}" + fi + printf '\n\n\n\n' >> "${commentfile}" + + i=0 + until [[ ${i} -ge ${#TP_HEADER[@]} ]]; do + printf '%s\n\n' "${TP_HEADER[i]}" >> "${commentfile}" + ((i=i+1)) + done + + { + printf '\n\n' + echo "| Vote | Subsystem | Runtime | Logfile | Comment |" + echo "|:----:|----------:|--------:|:--------:|:-------:|" + } >> "${commentfile}" + + i=0 + until [[ ${i} -ge ${#TP_VOTE_TABLE[@]} ]]; do + ourstring=$(echo "${TP_VOTE_TABLE[i]}" | tr -s ' ') + vote=$(echo "${ourstring}" | cut -f2 -d\| | tr -d ' ') + subs=$(echo "${ourstring}" | cut -f3 -d\|) + ela=$(echo "${ourstring}" | cut -f4 -d\|) + calctime=$(clock_display "${ela}") + logfile=$(echo "${ourstring}" | cut -f5 -d\| | tr -d ' ') + comment=$(echo "${ourstring}" | cut -f6 -d\|) + + if [[ "${vote}" = "H" ]]; then + echo "|||| _${comment}_ |" >> "${commentfile}" + ((i=i+1)) + continue + fi + + if [[ ${GITHUB_USE_EMOJI_VOTE} == true ]]; then + emoji="" + case ${vote} in + 1|"+1") + emoji="+1 :green_heart:" + ;; + -1) + emoji="-1 :x:" + ;; + 0) + emoji="+0 :ok:" + ;; + -0) + emoji="-0 :warning:" + ;; + H) + # this never gets called (see above) but this is here so others know the color is taken + emoji="" + ;; + *) + # usually this should not happen but let's keep the old vote result if it happens + emoji=${vote} + ;; + esac + else + emoji="${vote}" + fi + + # HBase modification: Only create markdown links for valid URLs + # to avoid broken links when artifact URLs are not available + if [[ -n "${logfile}" ]]; then + t1=${logfile/@@BASE@@/} + t2=$(echo "${logfile}" | "${SED}" -e "s,@@BASE@@,${url},g") + # Only create markdown link if t2 is a valid URL (starts with http) + if [[ "${t2}" =~ ^https?:// ]]; then + t2="[${t1}](${t2})" + else + # Just show filename without link for filesystem paths + t2="${t1}" + fi + else + t2="" + fi + + printf '| %s | %s | %s | %s | %s |\n' \ + "${emoji}" \ + "${subs}" \ + "${calctime}" \ + "${t2}" \ + "${comment}" \ + >> "${commentfile}" + + ((i=i+1)) + done + + if [[ ${#TP_TEST_TABLE[@]} -gt 0 ]]; then + { + printf '\n\n' + echo "| Reason | Tests |" + echo "|-------:|:------|" + } >> "${commentfile}" + i=0 + until [[ ${i} -ge ${#TP_TEST_TABLE[@]} ]]; do + echo "${TP_TEST_TABLE[i]}" >> "${commentfile}" + ((i=i+1)) + done + fi + + { + printf '\n\n' + echo "| Subsystem | Report/Notes |" + echo "|----------:|:-------------|" + } >> "${commentfile}" + + i=0 + until [[ $i -ge ${#TP_FOOTER_TABLE[@]} ]]; do + comment=$(echo "${TP_FOOTER_TABLE[i]}" | "${SED}" -e "s,@@BASE@@,${url},g") + printf '%s\n' "${comment}" >> "${commentfile}" + ((i=i+1)) + done + printf '\n\nThis message was automatically generated.\n\n' >> "${commentfile}" + + github_write_comment "${commentfile}" +} + ## This is named so that yetus will check us right after running tests. ## Essentially, we check for normal failures and then we look for zombies. #function hbase_unit_logfilter diff --git a/dev-support/jenkins_precommit_github_yetus.sh b/dev-support/jenkins_precommit_github_yetus.sh index 685ae3f186b8..3869a462ce8f 100755 --- a/dev-support/jenkins_precommit_github_yetus.sh +++ b/dev-support/jenkins_precommit_github_yetus.sh @@ -108,7 +108,9 @@ YETUS_ARGS+=("--spotbugs-strict-precheck") # rsync these files back into the archive dir YETUS_ARGS+=("--archive-list=${ARCHIVE_PATTERN_LIST}") # URL for user-side presentation in reports and such to our artifacts -YETUS_ARGS+=("--build-url-artifacts=${BUILD_URL_ARTIFACTS}") +if [[ -n "${BUILD_URL_ARTIFACTS}" ]]; then + YETUS_ARGS+=("--build-url-artifacts=${BUILD_URL_ARTIFACTS}") +fi # plugins to enable YETUS_ARGS+=("--plugins=${PLUGINS},-findbugs") # run in docker mode and specifically point to our diff --git a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ClientTests.java b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ClientTests.java index b0e259e1f9e2..ff98ebbc81fa 100644 --- a/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ClientTests.java +++ b/hbase-annotations/src/test/java/org/apache/hadoop/hbase/testclassification/ClientTests.java @@ -1,25 +1,9 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ package org.apache.hadoop.hbase.testclassification; /** * Tag a test as related to the client. This tests the hbase-client package and all of the client * tests in hbase-server. + * @author ndimiduk * @see org.apache.hadoop.hbase.testclassification.ClientTests * @see org.apache.hadoop.hbase.testclassification.CoprocessorTests * @see org.apache.hadoop.hbase.testclassification.FilterTests @@ -36,5 +20,5 @@ * @see org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests */ public interface ClientTests { - public static final String TAG = "org.apache.hadoop.hbase.testclassification.ClientTests"; + public static final String TAG = "org.apache.hadoop.hbase.testclassification.ClientTests"; }