Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 142 additions & 0 deletions .github/workflows/yetus-general-check.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

# yamllint disable rule:line-length
---
name: Yetus General Check

"on":
pull_request:
types: [opened, synchronize, reopened]

permissions:
checks: write
contents: read
pull-requests: write
statuses: write

jobs:
general-check:
runs-on: ubuntu-latest
timeout-minutes: 600

env:
YETUS_VERSION: '0.15.0'

steps:
- name: Checkout HBase
uses: actions/checkout@v4
with:
path: src
fetch-depth: 0

- name: Set up JDK 17
uses: actions/setup-java@v4
with:
java-version: '17'
distribution: 'temurin'

- name: Maven cache
uses: actions/cache@v4
with:
path: ~/.m2
key: hbase-m2-${{ hashFiles('**/pom.xml') }}
restore-keys: |
hbase-m2-

- name: Download Yetus
run: |
mkdir -p yetus
cd yetus
bash "${{ github.workspace }}/src/dev-support/jenkins-scripts/cache-apache-project-artifact.sh" \
--keys 'https://downloads.apache.org/yetus/KEYS' \
--verify-tar-gz \
./apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz \
yetus/${{ env.YETUS_VERSION }}/apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz
tar --strip-components=1 -xzf apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz
rm apache-yetus-${{ env.YETUS_VERSION }}-bin.tar.gz

- name: Run Yetus General Check
env:
ARCHIVE_PATTERN_LIST: "TEST-*.xml,org.apache.h*.txt,*.dumpstream,*.dump"
DOCKERFILE: "${{ github.workspace }}/src/dev-support/docker/Dockerfile"
GITHUB_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
GITHUB_USER: ${{ github.actor }}
PATCHDIR: "${{ github.workspace }}/yetus-general-check/output"
PLUGINS: "all,-javadoc,-jira,-shadedjars,-unit"
SET_JAVA_HOME: "/usr/lib/jvm/java-17"
SOURCEDIR: "${{ github.workspace }}/src"
TESTS_FILTER: "checkstyle,javac,javadoc,pylint,shellcheck,shelldocs,blanks,perlcritic,ruby-lint,rubocop"
YETUSDIR: "${{ github.workspace }}/yetus"
AUTHOR_IGNORE_LIST: "src/main/asciidoc/_chapters/developer.adoc"
BLANKS_EOL_IGNORE_FILE: "dev-support/blanks-eol-ignore.txt"
BLANKS_TABS_IGNORE_FILE: "dev-support/blanks-tabs-ignore.txt"
EXCLUDE_TESTS_URL: "https://ci-hbase.apache.org/job/HBase-Find-Flaky-Tests/job/${{ github.base_ref }}/lastSuccessfulBuild/artifact/output/excludes"
BUILD_THREAD: "4"
SUREFIRE_FIRST_PART_FORK_COUNT: "1.0C"
SUREFIRE_SECOND_PART_FORK_COUNT: "0.5C"
BRANCH_NAME: "${{ github.base_ref }}"
DEBUG: 'true'
run: |
cd "${{ github.workspace }}"
bash src/dev-support/jenkins_precommit_github_yetus.sh

- name: Hide Old Comments
if: always()
env:
DEBUG: 'true'
run: |
# github-actions[bot] has a known user ID: 41898282
# Verify: curl -s "https://api.github.com/users/github-actions%5Bbot%5D" | jq .id
export DEBUG="${DEBUG:-false}"
export GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}"
export BUILD_BOT_USER_ID="41898282"
export JOB_NAME="HBase-PreCommit-GH-Actions-PR"
export REPO="${{ github.repository }}"
bash src/dev-support/gh_hide_old_comments.sh ${{ github.event.pull_request.number }} || true

- name: Create Nightlies Directory Structure
# this version of rsync doesn't support --mkdirs so we do this step manually
if: always()
run: |
eval $(ssh-agent -s)
ssh-add - <<< "${{ secrets.NIGHTLIES_RSYNC_KEY }}"
ssh -p ${{ secrets.NIGHTLIES_RSYNC_PORT }} \
-o StrictHostKeyChecking=no \
${{ secrets.NIGHTLIES_RSYNC_USER }}@${{ secrets.NIGHTLIES_RSYNC_HOST }} \
"mkdir -p ${{ secrets.NIGHTLIES_RSYNC_PATH }}/hbase/HBase-PreCommit-GH-Actions-PR/PR-${{ github.event.pull_request.number }}/${{ github.run_number }}"
ssh-agent -k

- name: Publish to Nightlies
if: always()
uses: burnett01/rsync-deployments@0dc935cdecc5f5e571865e60d2a6cdc673704823
with:
switches: -avzr --no-o
path: yetus-general-check
remote_path: ${{ secrets.NIGHTLIES_RSYNC_PATH }}/hbase/HBase-PreCommit-GH-Actions-PR/PR-${{ github.event.pull_request.number }}/${{ github.run_number }}/
remote_host: ${{ secrets.NIGHTLIES_RSYNC_HOST }}
remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}

- name: Publish Test Results
if: always()
uses: actions/upload-artifact@v4
with:
name: yetus-general-check-output
path: ${{ github.workspace }}/yetus-general-check/output
retention-days: 7
14 changes: 12 additions & 2 deletions dev-support/gh_hide_old_comments.sh
Original file line number Diff line number Diff line change
Expand Up @@ -104,11 +104,16 @@ function identify_most_recent_build_number {
local pr="$1"
local comments_file="$2"
local jq_filter
local url_pattern="${JOB_NAME}/job/PR-${pr}/(?<buildnum>[0-9]+)/"
# GitHub Actions URLs don't have /job/ in them
if [[ "${JOB_NAME}" == *"GH-Actions"* ]]; then
url_pattern="${JOB_NAME}/PR-${pr}/(?<buildnum>[0-9]+)/"
fi
read -r -d '' jq_filter << EOF || :
.[] \
| select(.user.id == ${BUILD_BOT_USER_ID}) \
| .body \
| capture("${JOB_NAME}/job/PR-${pr}/(?<buildnum>[0-9]+)/") \
| capture("${url_pattern}") \
| .buildnum
EOF

Expand All @@ -122,10 +127,15 @@ function identify_old_comment_ids {
local comments_file="$2"
local most_recent_build_number="$3"
local jq_filter
local url_pattern="${JOB_NAME}/job/PR-${pr}/(?<buildnum>[0-9]+)/"
# GitHub Actions URLs don't have /job/ in them
if [[ "${JOB_NAME}" == *"GH-Actions"* ]]; then
url_pattern="${JOB_NAME}/PR-${pr}/(?<buildnum>[0-9]+)/"
fi
read -r -d '' jq_filter << EOF || :
.[] \
| select(.user.id == ${BUILD_BOT_USER_ID}) \
| { node_id, buildnum: (.body | capture("${JOB_NAME}/job/PR-${pr}/(?<buildnum>[0-9]+)/") | .buildnum | tonumber) } \
| { node_id, buildnum: (.body | capture("${url_pattern}") | .buildnum | tonumber) } \
| select(.buildnum < (${most_recent_build_number} | tonumber)) \
| .node_id
EOF
Expand Down
34 changes: 31 additions & 3 deletions dev-support/hbase-personality.sh
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,20 @@ function personality_globals
# See HBASE-19902 for how we arrived at 20g.
#shellcheck disable=SC2034
DOCKERMEMLIMIT=20g

# Override BUILD_URL for GitHub Actions to point to nightlies
# instead of the GitHub Actions run page
# The robot sets BUILD_URL to the GHA run page, but we want nightlies URLs
if [[ "${GITHUB_ACTIONS}" == "true" ]] && [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
local pr_number
# GITHUB_REF is a standard GitHub Actions environment variable
# shellcheck disable=SC2153
pr_number=$(echo "${GITHUB_REF}" | cut -f3 -d/)
#shellcheck disable=SC2034
BUILD_URL="https://nightlies.apache.org/hbase/HBase-PreCommit-GH-Actions-PR/PR-${pr_number}/${GITHUB_RUN_NUMBER}/"
#shellcheck disable=SC2034
BUILD_URL_ARTIFACTS="yetus-general-check/output"
fi
}

## @description Parse extra arguments required by personalities, if any.
Expand Down Expand Up @@ -835,6 +849,19 @@ function hbaseanti_patchfile

add_test_type spotless

## @description Build artifact URL for GitHub Actions
## @audience private
## @stability evolving
## @replaceable no
if ! declare -f githubactions_artifact_url >/dev/null; then
function githubactions_artifact_url
{
if [[ -n "${BUILD_URL}" ]] && [[ -n "${BUILD_URL_ARTIFACTS}" ]]; then
echo "${BUILD_URL}${BUILD_URL_ARTIFACTS}"
fi
}
fi

## @description spotless file filter
## @audience private
## @stability evolving
Expand Down Expand Up @@ -869,12 +896,13 @@ function spotless_rebuild

count=$(${GREP} -c '\[ERROR\]' "${logfile}")
if [[ ${count} -gt 0 ]]; then
add_vote_table -1 spotless "${repostatus} has ${count} errors when running spotless:check, run spotless:apply to fix."
add_footer_table spotless "@@BASE@@/${repostatus}-spotless.txt"
add_vote_table_v2 -1 spotless \
"@@BASE@@/${repostatus}-spotless.txt" \
"${repostatus} has ${count} errors when running spotless:check, run spotless:apply to fix."
return 1
fi

add_vote_table +1 spotless "${repostatus} has no errors when running spotless:check."
add_vote_table_v2 +1 spotless "" "${repostatus} has no errors when running spotless:check."
return 0
}

Expand Down
7 changes: 6 additions & 1 deletion dev-support/jenkins_precommit_github_yetus.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ declare -i missing_env=0
declare -a required_envs=(
# these ENV variables define the required API with Jenkinsfile_GitHub
"ARCHIVE_PATTERN_LIST"
"BUILD_URL_ARTIFACTS"
"DOCKERFILE"
"GITHUB_PASSWORD"
"GITHUB_USER"
Expand All @@ -53,6 +52,12 @@ for required_env in "${required_envs[@]}"; do
fi
done

# BUILD_URL_ARTIFACTS is required for Jenkins but set in personality for GitHub Actions
if [[ "${GITHUB_ACTIONS}" != "true" ]] && [[ -z "${BUILD_URL_ARTIFACTS}" ]]; then
echo "[ERROR] Required environment variable 'BUILD_URL_ARTIFACTS' is not set."
missing_env=${missing_env}+1
fi

if [ ${missing_env} -gt 0 ]; then
echo "[ERROR] Please set the required environment variables before invoking. If this error is " \
"on Jenkins, then please file a JIRA about the error."
Expand Down