diff --git a/.asf.yaml b/.asf.yaml index 8bd6ce570b5..c7e617cd9c9 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -66,6 +66,16 @@ github: # strict means "Require branches to be up to date before merging". strict: true # contexts are the names of checks that must pass + contexts: + - frontend (ubuntu-latest, 18) + - frontend (windows-latest, 18) + - frontend (macos-latest, 18) + - scala (ubuntu-22.04, 11) + - python (ubuntu-latest, 3.10) + - python (ubuntu-latest, 3.11) + - python (ubuntu-latest, 3.12) + - Check License Headers + - Validate PR title required_pull_request_reviews: dismiss_stale_reviews: false require_code_owner_reviews: false @@ -74,10 +84,7 @@ github: notifications: commits: commits@texera.apache.org - issues: dev@texera.apache.org - pullrequests: dev@texera.apache.org - discussions: dev@texera.apache.org + issues: notifications@texera.apache.org + pullrequests: notifications@texera.apache.org + discussions: notifications@texera.apache.org jobs: commits@texera.apache.org - -publish: - whoami: asf-site diff --git a/.github/ISSUE_TEMPLATE/bug-template.yaml b/.github/ISSUE_TEMPLATE/bug-template.yaml new file mode 100644 index 00000000000..7a3ebc92022 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-template.yaml @@ -0,0 +1,81 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Bug Report +description: File a bug report. +labels: ["triage"] +type: "Bug" +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + - type: textarea + id: what-happened + attributes: + label: What happened? + description: Also tell us, what did you expect to happen? + placeholder: Tell us what you see! + value: "A bug happened!" + validations: + required: true + - type: textarea + id: reproduce + attributes: + label: How to reproduce? + description: Please include steps for a repro. + validations: + required: true + - type: dropdown + id: version + attributes: + label: Version + description: What version of Texera are you running? + options: + - 1.1.0-incubating (Pre-release/Master) + - 1.0.0 + default: 0 + validations: + required: true + - type: input + id: commit-hash + attributes: + label: Commit Hash (Optional) + description: If you know the specific commit that has the issue, please provide the commit hash here. + placeholder: e.g., a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6q7r8s9t0 + validations: + required: false + - type: dropdown + id: browsers + attributes: + label: What browsers are you seeing the problem on? + multiple: true + options: + - Chrome + - Safari + - Firefox + - Microsoft Edge + - type: textarea + id: logs + attributes: + label: Relevant log output + description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + render: shell + + - type: markdown + attributes: + value: | + By submitting this issue, you agree to follow the [Apache Code of Conduct](https://www.apache.org/foundation/policies/conduct). diff --git a/.github/ISSUE_TEMPLATE/feature-template.yaml b/.github/ISSUE_TEMPLATE/feature-template.yaml new file mode 100644 index 00000000000..f88d85b0f67 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-template.yaml @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Feature Request +description: Suggest a new feature or improvement. +labels: ["triage"] +type: "Feature" +body: + - type: markdown + attributes: + value: | + Thanks for suggesting a feature! Please provide as much detail as possible to help us evaluate your idea. + + - type: textarea + id: summary + attributes: + label: Feature Summary + description: Clearly describe what the feature is and the problem it solves. + placeholder: Describe your feature idea and what problem it addresses. + validations: + required: true + + - type: textarea + id: proposal + attributes: + label: Proposed Solution or Design + description: Explain how you imagine this feature working. Include examples, diagrams, or pseudo-code if relevant. + placeholder: Describe your proposed solution or design approach. + validations: + required: true + + - type: dropdown + id: impact + attributes: + label: Impact / Priority + description: How important is this feature? + options: + - (P0)Critical – blocks existing use cases + - (P1)High – significantly improves user experience + - (P2)Medium – useful enhancement + - (P3)Low – nice to have + default: 2 + validations: + required: true + + - type: dropdown + id: affected-area + attributes: + label: Affected Area + description: Which part of the system does this feature relate to? + multiple: true + options: + - Workflow Engine (Amber) + - Workflow UI + - Hub + - Storage / Metadata + - Deployment / Infrastructure + - Other + + - type: markdown + attributes: + value: | + By submitting this issue, you agree to follow the [Apache Code of Conduct](https://www.apache.org/foundation/policies/conduct). \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/task-template.yaml b/.github/ISSUE_TEMPLATE/task-template.yaml new file mode 100644 index 00000000000..1e299fef7c6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/task-template.yaml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Task +description: Create a new development or maintenance task. +labels: ["triage"] +type: "Task" +body: + - type: markdown + attributes: + value: | + Thanks for creating a task! Please describe what needs to be done and why. + + - type: textarea + id: task-summary + attributes: + label: Task Summary + description: Briefly describe what needs to be done, try to do a single step in a task. + placeholder: Example — Refactor workflow scheduler module for better modularity. + validations: + required: true + + - type: dropdown + id: priority + attributes: + label: Priority + description: How urgent or important is this task? + options: + - P0 – Critical + - P1 – High + - P2 – Medium + - P3 – Low + default: 2 + + - type: checkboxes + id: checklist + attributes: + label: Task Type + description: Select the type of work involved. + options: + - label: Code Implementation + - label: Documentation + - label: Refactor / Cleanup + - label: Testing / QA + - label: DevOps / Deployment + + - type: markdown + attributes: + value: | + By submitting this issue, you agree to follow the [Apache Code of Conduct](https://www.apache.org/foundation/policies/conduct). diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE new file mode 100644 index 00000000000..41287564ff3 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE @@ -0,0 +1,53 @@ + + +### What changes were proposed in this PR? + + + +### Any related issues, documentation, discussions? + + + +### How was this PR tested? + + + +### Was this PR authored or co-authored using generative AI tooling? + diff --git a/.github/labeler.yml b/.github/labeler.yml index 9d3d7a68fa2..72ff0ffd4a0 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -14,30 +14,30 @@ # See the License for the specific language governing permissions and # limitations under the License. -gui: +frontend: - changed-files: - any-glob-to-any-file: - - 'core/gui/**' + - 'frontend/**' -backend: - - all: - - changed-files: - - any-glob-to-any-file: - - 'core/**' - - all-globs-to-all-files: - - '!core/gui/**' - - '!core/log/**' - - '!core/scripts/**' +common: + - changed-files: + - any-glob-to-any-file: + - 'common/**' + +service: + - changed-files: + - any-glob-to-any-file: + - '*-service/**' engine: - changed-files: - any-glob-to-any-file: - - 'core/amber/**' + - 'amber/**' python: - changed-files: - any-glob-to-any-file: - - 'core/amber/src/main/python/**' + - 'amber/src/main/python/**' - '**/*.py' docs: @@ -56,7 +56,7 @@ ci: build: - changed-files: - any-glob-to-any-file: - - 'core/scripts/**' + - 'bin/**' - 'deployment/**' dependencies: @@ -74,8 +74,8 @@ ddl-change: feature: - head-branch: - - '^feat' - - 'feature' + - '^feat' + - 'feature' fix: - head-branch: '^fix' diff --git a/.github/release/vote-email-template.md b/.github/release/vote-email-template.md new file mode 100644 index 00000000000..8fd6cd471f5 --- /dev/null +++ b/.github/release/vote-email-template.md @@ -0,0 +1,85 @@ +Subject: [VOTE] Release Apache Texera (incubating) ${VERSION} RC${RC_NUM} + +Hi Texera Community, + +This is a call for vote to release Apache Texera (incubating) ${VERSION}. + +== Release Candidate Artifacts == + +The release candidate artifacts can be found at: +https://dist.apache.org/repos/dist/dev/incubator/texera/${RC_DIR}/ + +The artifacts include: +- apache-texera-${VERSION}-rc${RC_NUM}-src.tar.gz (source tarball) +- apache-texera-${VERSION}-rc${RC_NUM}-src.tar.gz.asc (GPG signature) +- apache-texera-${VERSION}-rc${RC_NUM}-src.tar.gz.sha512 (SHA512 checksum) + +== Git Tag == + +The Git tag for this release candidate: +https://github.com/apache/incubator-texera/releases/tag/${TAG_NAME} + +The commit hash for this tag: +${COMMIT_HASH} + +== Release Notes == + +Release notes can be found at: +https://github.com/apache/incubator-texera/releases/tag/${TAG_NAME} + +== Keys == + +The artifacts have been signed with Key [${GPG_KEY_ID}], corresponding to [${GPG_EMAIL}]. + +The KEYS file containing the public keys can be found at: +https://dist.apache.org/repos/dist/dev/incubator/texera/KEYS + +== How to Verify == + +1. Download the release artifacts: + + wget https://dist.apache.org/repos/dist/dev/incubator/texera/${RC_DIR}/apache-texera-${VERSION}-rc${RC_NUM}-src.tar.gz + wget https://dist.apache.org/repos/dist/dev/incubator/texera/${RC_DIR}/apache-texera-${VERSION}-rc${RC_NUM}-src.tar.gz.asc + wget https://dist.apache.org/repos/dist/dev/incubator/texera/${RC_DIR}/apache-texera-${VERSION}-rc${RC_NUM}-src.tar.gz.sha512 + +2. Import the KEYS file and verify the GPG signature: + + wget https://dist.apache.org/repos/dist/dev/incubator/texera/KEYS + gpg --import KEYS + gpg --verify apache-texera-${VERSION}-rc${RC_NUM}-src.tar.gz.asc apache-texera-${VERSION}-rc${RC_NUM}-src.tar.gz + +3. Verify the SHA512 checksum: + + sha512sum -c apache-texera-${VERSION}-rc${RC_NUM}-src.tar.gz.sha512 + +4. Extract and build from source: + + tar -xzf apache-texera-${VERSION}-rc${RC_NUM}-src.tar.gz + cd apache-texera-${VERSION}-rc${RC_NUM}-src + # Follow build instructions in README + +== How to Vote == + +The vote will be open for at least 72 hours. + +Please vote accordingly: + +[ ] +1 Approve the release +[ ] 0 No opinion +[ ] -1 Disapprove the release (please provide the reason) + +== Checklist for Reference == + +When reviewing, please check: + +[ ] Download links are valid +[ ] Checksums and PGP signatures are valid +[ ] LICENSE and NOTICE files are correct +[ ] All files have ASF license headers where appropriate +[ ] No unexpected binary files +[ ] Source tarball matches the Git tag +[ ] Can compile from source successfully + +Thanks, +[Your Name] +Apache Texera (incubating) PPMC diff --git a/.github/workflows/build-and-push-images.yml b/.github/workflows/build-and-push-images.yml new file mode 100644 index 00000000000..478ce28ad2e --- /dev/null +++ b/.github/workflows/build-and-push-images.yml @@ -0,0 +1,513 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build and push images + +on: + workflow_dispatch: + inputs: + branch: + description: 'Branch to checkout and build from' + required: false + default: 'main' + type: string + image_tag: + description: 'Docker image tag to use (e.g., latest, v1.0.0)' + required: true + default: 'latest' + type: string + docker_registry: + description: 'Docker registry namespace (e.g., apache, myorg)' + required: false + default: 'apache' + type: string + services: + description: 'Services to build (comma-separated, "*" for all)' + required: false + default: '*' + type: string + platforms: + description: 'Target platforms to build' + required: false + default: 'both' + type: choice + options: + - both + - amd64 + - arm64 + schedule: + # Run nightly at 2:00 AM UTC + - cron: '0 2 * * *' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.inputs.image_tag || 'nightly' }} + cancel-in-progress: false + +env: + DOCKER_REGISTRY: apache # Will be overridden by job-level env if needed + +jobs: + # Step 0: Set runtime parameters (handles both manual and scheduled runs) + set-parameters: + runs-on: ubuntu-latest + outputs: + branch: ${{ steps.set-params.outputs.branch }} + image_tag: ${{ steps.set-params.outputs.image_tag }} + docker_registry: ${{ steps.set-params.outputs.docker_registry }} + services: ${{ steps.set-params.outputs.services }} + platforms: ${{ steps.set-params.outputs.platforms }} + steps: + - name: Set build parameters + id: set-params + run: | + # Detect if this is a scheduled run + if [[ "${{ github.event_name }}" == "schedule" ]]; then + echo "Nightly build detected - using nightly defaults" + echo "branch=main" >> $GITHUB_OUTPUT + echo "image_tag=nightly" >> $GITHUB_OUTPUT + echo "docker_registry=apache" >> $GITHUB_OUTPUT + echo "services=*" >> $GITHUB_OUTPUT + echo "platforms=both" >> $GITHUB_OUTPUT + else + echo "Manual workflow_dispatch - using user inputs" + echo "branch=${{ github.event.inputs.branch || 'main' }}" >> $GITHUB_OUTPUT + echo "image_tag=${{ github.event.inputs.image_tag }}" >> $GITHUB_OUTPUT + echo "docker_registry=${{ github.event.inputs.docker_registry || 'apache' }}" >> $GITHUB_OUTPUT + echo "services=${{ github.event.inputs.services || '*' }}" >> $GITHUB_OUTPUT + echo "platforms=${{ github.event.inputs.platforms || 'both' }}" >> $GITHUB_OUTPUT + fi + + # Step 1: Generate JOOQ code once and share it + generate-jooq: + needs: [set-parameters] + runs-on: ubuntu-latest + env: + JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + + steps: + - name: Checkout Texera + uses: actions/checkout@v5 + with: + ref: ${{ needs.set-parameters.outputs.branch }} + + - name: Setup JDK + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: 11 + + - name: Setup sbt launcher + uses: sbt/setup-sbt@3e125ece5c3e5248e18da9ed8d2cce3d335ec8dd # v1.1.14 + + - uses: coursier/cache-action@4e2615869d13561d626ed48655e1a39e5b192b3c # v6.4.9 + with: + extraSbtFiles: '["*.sbt", "project/**.{scala,sbt}", "project/build.properties" ]' + + - name: Install PostgreSQL + run: sudo apt-get update && sudo apt-get install -y postgresql + + - name: Start PostgreSQL Service + run: sudo systemctl start postgresql + + - name: Configure PostgreSQL authentication + run: | + sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';" + sudo sed -i 's/local all postgres peer/local all postgres md5/' /etc/postgresql/*/main/pg_hba.conf + sudo sed -i 's/host all all 127.0.0.1\/32 scram-sha-256/host all all 127.0.0.1\/32 md5/' /etc/postgresql/*/main/pg_hba.conf + sudo systemctl restart postgresql + sleep 2 + + - name: Create Databases + run: | + PGPASSWORD=postgres psql -h localhost -U postgres -f sql/texera_ddl.sql + PGPASSWORD=postgres psql -h localhost -U postgres -f sql/iceberg_postgres_catalog.sql + PGPASSWORD=postgres psql -h localhost -U postgres -f sql/texera_lakefs.sql + + - name: Generate JOOQ code + run: sbt "DAO/runMain org.apache.texera.dao.JooqCodeGenerator" + + - name: Upload JOOQ generated code + uses: actions/upload-artifact@v4 + with: + name: jooq-code + path: | + common/dao/src/main/scala/org/apache/texera/dao/jooq/generated/ + retention-days: 1 + + # Step 2: Parse services and prepare build matrix + prepare-matrix: + needs: [set-parameters] + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + build_amd64: ${{ steps.set-platforms.outputs.build_amd64 }} + build_arm64: ${{ steps.set-platforms.outputs.build_arm64 }} + need_manifest: ${{ steps.set-platforms.outputs.need_manifest }} + steps: + - name: Checkout Texera + uses: actions/checkout@v5 + with: + ref: ${{ needs.set-parameters.outputs.branch }} + + - name: Set target platforms + id: set-platforms + run: | + PLATFORM_INPUT="${{ needs.set-parameters.outputs.platforms }}" + + case "$PLATFORM_INPUT" in + both) + echo "build_amd64=true" >> $GITHUB_OUTPUT + echo "build_arm64=true" >> $GITHUB_OUTPUT + echo "need_manifest=true" >> $GITHUB_OUTPUT + echo "Building for both platforms (parallel jobs)" + ;; + amd64) + echo "build_amd64=true" >> $GITHUB_OUTPUT + echo "build_arm64=false" >> $GITHUB_OUTPUT + echo "need_manifest=false" >> $GITHUB_OUTPUT + echo "Building for AMD64 only" + ;; + arm64) + echo "build_amd64=false" >> $GITHUB_OUTPUT + echo "build_arm64=true" >> $GITHUB_OUTPUT + echo "need_manifest=false" >> $GITHUB_OUTPUT + echo "Building for ARM64 only" + ;; + esac + + - name: Discover and parse services + id: set-matrix + run: | + SERVICES="${{ needs.set-parameters.outputs.services }}" + + # Discover all Dockerfiles in bin/ directory + echo "Discovering services from Dockerfiles..." + cd bin + + # Standard services from *.dockerfile pattern (excluding postgres17-pgroonga) + STANDARD_SERVICES=() + for dockerfile in *.dockerfile; do + if [[ -f "$dockerfile" ]]; then + service_name=$(basename "$dockerfile" .dockerfile) + # Skip postgres17-pgroonga + if [[ "$service_name" != "postgres17-pgroonga" ]]; then + STANDARD_SERVICES+=("$service_name") + fi + fi + done + + # All services are standard services only + ALL_SERVICES=("${STANDARD_SERVICES[@]}") + + echo "Found ${#ALL_SERVICES[@]} services: ${ALL_SERVICES[*]}" + + # Filter based on user input + if [[ "$SERVICES" == "*" ]]; then + SERVICES_LIST=("${ALL_SERVICES[@]}") + else + IFS=',' read -ra SERVICES_LIST <<< "$SERVICES" + # Trim whitespace + for i in "${!SERVICES_LIST[@]}"; do + SERVICES_LIST[$i]=$(echo "${SERVICES_LIST[$i]}" | xargs) + done + fi + + # Create JSON matrix with dockerfile info + JSON="[" + FIRST=true + for service in "${SERVICES_LIST[@]}"; do + # Determine dockerfile path and context + if [[ " ${STANDARD_SERVICES[@]} " =~ " ${service} " ]]; then + dockerfile="bin/${service}.dockerfile" + context="." + + # Map dockerfile service names to Docker image names + case "$service" in + "texera-web-application") + image_name="texera-dashboard-service" + ;; + "computing-unit-master") + image_name="texera-workflow-execution-coordinator" + ;; + "computing-unit-worker") + image_name="texera-workflow-execution-runner" + ;; + "access-control-service") + image_name="texera-access-control-service" + ;; + "config-service") + image_name="texera-config-service" + ;; + "file-service") + image_name="texera-file-service" + ;; + "workflow-compiling-service") + image_name="texera-workflow-compiling-service" + ;; + "workflow-computing-unit-managing-service") + image_name="texera-workflow-computing-unit-managing-service" + ;; + *) + # Default: use service name as-is + image_name="$service" + ;; + esac + else + echo "WARNING: Unknown service: $service, skipping" + continue + fi + + if [[ "$FIRST" == "true" ]]; then + FIRST=false + else + JSON+="," + fi + JSON+="{\"service\":\"$service\",\"image_name\":\"$image_name\",\"dockerfile\":\"$dockerfile\",\"context\":\"$context\"}" + done + JSON+="]" + + echo "Generated matrix: $JSON" + echo "matrix={\"include\":$JSON}" >> $GITHUB_OUTPUT + + # Step 3a: Build AMD64 images (runs in parallel with ARM64) + build-amd64: + runs-on: ubuntu-latest + needs: [set-parameters, generate-jooq, prepare-matrix] + if: needs.prepare-matrix.outputs.build_amd64 == 'true' + strategy: + matrix: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }} + fail-fast: false + max-parallel: 8 # Higher parallelism for native builds + env: + DOCKER_REGISTRY: ${{ needs.set-parameters.outputs.docker_registry }} + JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + + steps: + - name: Checkout Texera + uses: actions/checkout@v5 + with: + ref: ${{ needs.set-parameters.outputs.branch }} + + - name: Setup JDK + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: 11 + + - name: Setup sbt launcher + uses: sbt/setup-sbt@3e125ece5c3e5248e18da9ed8d2cce3d335ec8dd # v1.1.14 + + - uses: coursier/cache-action@4e2615869d13561d626ed48655e1a39e5b192b3c # v6.4.9 + with: + extraSbtFiles: '["*.sbt", "project/**.{scala,sbt}", "project/build.properties" ]' + + - name: Download JOOQ generated code + uses: actions/download-artifact@v4 + with: + name: jooq-code + path: common/dao/src/main/scala/org/apache/texera/dao/jooq/generated/ + + - name: Free up disk space + run: | + sudo apt-get clean + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/share/boost + df -h + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Log in to Docker Hub + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build and push AMD64 image + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ matrix.context }} + file: ${{ matrix.dockerfile }} + platforms: linux/amd64 + push: true + tags: ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }}-amd64 + cache-from: type=gha,scope=${{ matrix.image_name }}-amd64 + cache-to: type=gha,mode=max,scope=${{ matrix.image_name }}-amd64 + labels: | + org.opencontainers.image.title=${{ matrix.image_name }} + org.opencontainers.image.description=Apache Texera ${{ matrix.image_name }} (AMD64) + org.opencontainers.image.vendor=Apache Texera + + # Step 3b: Build ARM64 images (runs in parallel with AMD64) + build-arm64: + runs-on: ubuntu-latest + needs: [set-parameters, generate-jooq, prepare-matrix] + if: needs.prepare-matrix.outputs.build_arm64 == 'true' + strategy: + matrix: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }} + fail-fast: false + max-parallel: 4 # Lower for QEMU builds + env: + DOCKER_REGISTRY: ${{ needs.set-parameters.outputs.docker_registry }} + JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8 + + steps: + - name: Checkout Texera + uses: actions/checkout@v5 + with: + ref: ${{ needs.set-parameters.outputs.branch }} + + - name: Setup JDK + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: 11 + + - name: Setup sbt launcher + uses: sbt/setup-sbt@3e125ece5c3e5248e18da9ed8d2cce3d335ec8dd # v1.1.14 + + - uses: coursier/cache-action@4e2615869d13561d626ed48655e1a39e5b192b3c # v6.4.9 + with: + extraSbtFiles: '["*.sbt", "project/**.{scala,sbt}", "project/build.properties" ]' + + - name: Download JOOQ generated code + uses: actions/download-artifact@v4 + with: + name: jooq-code + path: common/dao/src/main/scala/org/apache/texera/dao/jooq/generated/ + + - name: Free up disk space + run: | + sudo apt-get clean + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf /usr/local/share/boost + df -h + + # Set up QEMU for ARM64 emulation + - name: Set up QEMU + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 + with: + platforms: linux/arm64 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 + + - name: Log in to Docker Hub + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build and push ARM64 image + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 + with: + context: ${{ matrix.context }} + file: ${{ matrix.dockerfile }} + platforms: linux/arm64 + push: true + tags: ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }}-arm64 + cache-from: type=gha,scope=${{ matrix.image_name }}-arm64 + cache-to: type=gha,mode=max,scope=${{ matrix.image_name }}-arm64 + labels: | + org.opencontainers.image.title=${{ matrix.image_name }} + org.opencontainers.image.description=Apache Texera ${{ matrix.image_name }} (ARM64) + org.opencontainers.image.vendor=Apache Texera + + # Step 4: Create multi-arch manifests (only if building both platforms) + create-manifests: + runs-on: ubuntu-latest + needs: [set-parameters, prepare-matrix, build-amd64, build-arm64] + if: always() && needs.prepare-matrix.outputs.need_manifest == 'true' + strategy: + matrix: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }} + fail-fast: false + env: + DOCKER_REGISTRY: ${{ needs.set-parameters.outputs.docker_registry }} + + steps: + - name: Log in to Docker Hub + uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Create and push multi-arch manifest + run: | + # Create manifest list combining both architectures + docker buildx imagetools create -t \ + ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }} \ + ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }}-amd64 \ + ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }}-arm64 + + # Also tag as 'latest' if requested + if [[ "${{ needs.set-parameters.outputs.image_tag }}" == "latest" ]]; then + docker buildx imagetools create -t \ + ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:latest \ + ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:latest-amd64 \ + ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:latest-arm64 + fi + + - name: Inspect multi-arch manifest + run: | + docker buildx imagetools inspect ${{ env.DOCKER_REGISTRY }}/${{ matrix.image_name }}:${{ needs.set-parameters.outputs.image_tag }} + + # Step 5: Summary report + build-summary: + runs-on: ubuntu-latest + needs: [set-parameters, prepare-matrix, build-amd64, build-arm64, create-manifests] + if: always() + steps: + - name: Generate build summary + run: | + echo "# Texera Multi-Arch Build Complete (Parallel)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "## Build Configuration" >> $GITHUB_STEP_SUMMARY + echo "- **Trigger:** ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY + echo "- **Branch:** \`${{ needs.set-parameters.outputs.branch }}\`" >> $GITHUB_STEP_SUMMARY + echo "- **Registry:** \`${{ needs.set-parameters.outputs.docker_registry }}\`" >> $GITHUB_STEP_SUMMARY + echo "- **Tag:** \`${{ needs.set-parameters.outputs.image_tag }}\`" >> $GITHUB_STEP_SUMMARY + echo "- **Services:** ${{ needs.set-parameters.outputs.services }}" >> $GITHUB_STEP_SUMMARY + echo "- **Platforms:** ${{ needs.set-parameters.outputs.platforms }}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "## Build Method" >> $GITHUB_STEP_SUMMARY + echo "**Parallel platform builds** (faster)" >> $GITHUB_STEP_SUMMARY + echo "- AMD64: Native build on \`ubuntu-latest\`" >> $GITHUB_STEP_SUMMARY + echo "- ARM64: QEMU emulation on \`ubuntu-latest\` (runs in parallel)" >> $GITHUB_STEP_SUMMARY + echo "- Manifests: Combined into multi-arch images" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "> **Performance:** AMD64 and ARM64 now build simultaneously instead of sequentially!" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "## Images Published" >> $GITHUB_STEP_SUMMARY + echo "All images are now available as multi-arch manifests at:" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "docker pull ${{ needs.set-parameters.outputs.docker_registry }}/:${{ needs.set-parameters.outputs.image_tag }}" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Usage" >> $GITHUB_STEP_SUMMARY + echo "The images will automatically use the correct architecture:" >> $GITHUB_STEP_SUMMARY + echo "- On x86_64/AMD64: pulls linux/amd64 variant" >> $GITHUB_STEP_SUMMARY + echo "- On ARM64/M1/M2: pulls linux/arm64 variant" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Build Status" >> $GITHUB_STEP_SUMMARY + echo "- AMD64 builds: ${{ needs.build-amd64.result }}" >> $GITHUB_STEP_SUMMARY + echo "- ARM64 builds: ${{ needs.build-arm64.result }}" >> $GITHUB_STEP_SUMMARY + echo "- Manifest creation: ${{ needs.create-manifests.result }}" >> $GITHUB_STEP_SUMMARY \ No newline at end of file diff --git a/.github/workflows/check-header.yml b/.github/workflows/check-header.yml index 0f781236e9e..c2cbddc1333 100644 --- a/.github/workflows/check-header.yml +++ b/.github/workflows/check-header.yml @@ -30,4 +30,4 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: apache/skywalking-eyes@v0.7.0 + - uses: apache/skywalking-eyes@5c5b974209f0de5d905f37deb69369068ebfc15c # v0.7.0 diff --git a/.github/workflows/create-release-candidate.yml b/.github/workflows/create-release-candidate.yml new file mode 100644 index 00000000000..719dc10f586 --- /dev/null +++ b/.github/workflows/create-release-candidate.yml @@ -0,0 +1,340 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Create and upload release candidate artifacts + +on: + workflow_dispatch: + inputs: + tag: + description: 'Existing Git tag (e.g., v1.1.0-incubating-rc1)' + required: true + type: string + rc_number: + description: 'Release candidate number for artifacts (e.g., 1 for RC1, 2 for RC2)' + required: true + type: string + default: '1' + +jobs: + create-rc: + runs-on: ubuntu-latest + outputs: + version: ${{ steps.vars.outputs.version }} + rc_num: ${{ steps.vars.outputs.rc_num }} + tag_name: ${{ steps.vars.outputs.tag_name }} + rc_dir: ${{ steps.vars.outputs.rc_dir }} + tarball_name: ${{ steps.vars.outputs.tarball_name }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Full history for proper tagging + + - name: Validate tag exists + run: | + TAG_NAME="${{ github.event.inputs.tag }}" + + # Check if tag exists + if ! git rev-parse "$TAG_NAME" >/dev/null 2>&1; then + echo "Error: Tag '$TAG_NAME' does not exist" + echo "Available tags:" + git tag -l | tail -10 + exit 1 + fi + + echo "✓ Tag validation passed: $TAG_NAME" + + - name: Set up variables + id: vars + run: | + TAG_NAME="${{ github.event.inputs.tag }}" + RC_NUM="${{ github.event.inputs.rc_number }}" + + # Parse version from tag (format: v1.1.0-incubating or v1.1.0-incubating-rcN) + # Both formats are accepted, but we use the input rc_number for artifacts + if [[ "$TAG_NAME" =~ ^v([0-9]+\.[0-9]+\.[0-9]+-incubating)(-rc[0-9]+)?$ ]]; then + VERSION="${BASH_REMATCH[1]}" + else + echo "Error: Tag must be in format vX.Y.Z-incubating or vX.Y.Z-incubating-rcN (e.g., v1.1.0-incubating-rc1)" + exit 1 + fi + + RC_DIR="${VERSION}-RC${RC_NUM}" + TARBALL_NAME="apache-texera-${VERSION}-rc${RC_NUM}-src.tar.gz" + + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "rc_num=$RC_NUM" >> $GITHUB_OUTPUT + echo "tag_name=$TAG_NAME" >> $GITHUB_OUTPUT + echo "rc_dir=$RC_DIR" >> $GITHUB_OUTPUT + echo "tarball_name=$TARBALL_NAME" >> $GITHUB_OUTPUT + + echo "Release Candidate: $TAG_NAME" + echo "Version: $VERSION" + echo "RC Number: $RC_NUM" + echo "Tarball: $TARBALL_NAME" + echo "Staging directory: dist/dev/incubator/texera/$RC_DIR" + + - name: Create source tarball + run: | + TAG_NAME="${{ steps.vars.outputs.tag_name }}" + TARBALL_NAME="${{ steps.vars.outputs.tarball_name }}" + VERSION="${{ steps.vars.outputs.version }}" + RC_NUM="${{ steps.vars.outputs.rc_num }}" + + # Create a temporary directory for the archive + TEMP_DIR=$(mktemp -d) + ARCHIVE_DIR="$TEMP_DIR/apache-texera-${VERSION}-rc${RC_NUM}-src" + + echo "Creating source archive in $ARCHIVE_DIR" + + # Export the git repository at the tag + git archive --format=tar --prefix="apache-texera-${VERSION}-rc${RC_NUM}-src/" "$TAG_NAME" | tar -x -C "$TEMP_DIR" + + # Create tarball + cd "$TEMP_DIR" + tar -czf "$GITHUB_WORKSPACE/$TARBALL_NAME" "apache-texera-${VERSION}-rc${RC_NUM}-src" + + cd "$GITHUB_WORKSPACE" + + # Verify tarball was created + if [[ ! -f "$TARBALL_NAME" ]]; then + echo "Error: Tarball was not created" + exit 1 + fi + + # Show tarball info + ls -lh "$TARBALL_NAME" + echo "✓ Created tarball: $TARBALL_NAME" + + - name: Import GPG key + run: | + echo "${{ secrets.GPG_PRIVATE_KEY }}" | gpg --batch --import + + # List imported keys + gpg --list-secret-keys + + echo "✓ GPG key imported successfully" + + - name: Create GPG signature (.asc) + run: | + TARBALL_NAME="${{ steps.vars.outputs.tarball_name }}" + + # Sign the tarball + echo "${{ secrets.GPG_PASSPHRASE }}" | gpg --batch --yes --pinentry-mode loopback --passphrase-fd 0 \ + --armor --detach-sign --output "${TARBALL_NAME}.asc" "$TARBALL_NAME" + + # Verify signature + gpg --verify "${TARBALL_NAME}.asc" "$TARBALL_NAME" + + echo "✓ Created GPG signature: ${TARBALL_NAME}.asc" + + - name: Create SHA512 checksum + run: | + TARBALL_NAME="${{ steps.vars.outputs.tarball_name }}" + + # Create SHA512 checksum + sha512sum "$TARBALL_NAME" > "${TARBALL_NAME}.sha512" + + # Display checksum + cat "${TARBALL_NAME}.sha512" + + echo "✓ Created SHA512 checksum: ${TARBALL_NAME}.sha512" + + - name: Generate vote email template + id: vote_email + run: | + VERSION="${{ steps.vars.outputs.version }}" + RC_NUM="${{ steps.vars.outputs.rc_num }}" + TAG_NAME="${{ steps.vars.outputs.tag_name }}" + RC_DIR="${{ steps.vars.outputs.rc_dir }}" + + # Get commit hash for the tag + COMMIT_HASH=$(git rev-parse "$TAG_NAME") + + # Get GPG key ID from the imported key + GPG_KEY_ID=$(gpg --list-secret-keys --keyid-format LONG | grep 'sec' | head -n1 | awk '{print $2}' | cut -d'/' -f2) + GPG_EMAIL=$(gpg --list-secret-keys | grep 'uid' | head -n1 | grep -oP '[\w\.-]+@[\w\.-]+') + + # Copy template from repository + cp .github/release/vote-email-template.md vote-email.txt + + # Substitute variables in the template + sed -i "s/\${VERSION}/${VERSION}/g" vote-email.txt + sed -i "s/\${RC_NUM}/${RC_NUM}/g" vote-email.txt + sed -i "s/\${RC_DIR}/${RC_DIR}/g" vote-email.txt + sed -i "s/\${TAG_NAME}/${TAG_NAME}/g" vote-email.txt + sed -i "s/\${COMMIT_HASH}/${COMMIT_HASH}/g" vote-email.txt + sed -i "s/\${GPG_KEY_ID}/${GPG_KEY_ID}/g" vote-email.txt + sed -i "s/\${GPG_EMAIL}/${GPG_EMAIL}/g" vote-email.txt + + echo "✓ Vote email template generated!" + + - name: Upload RC artifacts + uses: actions/upload-artifact@v4 + with: + name: rc-artifacts + path: | + ${{ steps.vars.outputs.tarball_name }} + ${{ steps.vars.outputs.tarball_name }}.asc + ${{ steps.vars.outputs.tarball_name }}.sha512 + vote-email.txt + retention-days: 7 + + upload-rc: + runs-on: ubuntu-latest + needs: create-rc + + steps: + - name: Download RC artifacts + uses: actions/download-artifact@v4 + with: + name: rc-artifacts + + - name: Verify downloaded artifacts + run: | + TARBALL_NAME="${{ needs.create-rc.outputs.tarball_name }}" + echo "Verifying downloaded artifacts..." + ls -lh + + if [[ ! -f "$TARBALL_NAME" ]] || [[ ! -f "${TARBALL_NAME}.asc" ]] || [[ ! -f "${TARBALL_NAME}.sha512" ]]; then + echo "Error: Missing required artifacts" + exit 1 + fi + + echo "✓ All artifacts downloaded successfully" + + - name: Install SVN + run: | + sudo apt-get update + sudo apt-get install -y subversion + svn --version + + - name: Checkout SVN dev directory + run: | + RC_DIR="${{ needs.create-rc.outputs.rc_dir }}" + + # Checkout the dev directory with depth=empty (lightweight) + svn co --depth=empty https://dist.apache.org/repos/dist/dev/incubator/texera svn-texera \ + --username "${{ secrets.SVN_USERNAME }}" \ + --password "${{ secrets.SVN_PASSWORD }}" \ + --no-auth-cache + + cd svn-texera + + # Create RC directory if it doesn't exist + if ! svn info "$RC_DIR" >/dev/null 2>&1; then + mkdir -p "$RC_DIR" + svn add "$RC_DIR" + echo "✓ Created new RC directory: $RC_DIR" + else + svn update "$RC_DIR" + echo "✓ RC directory already exists: $RC_DIR" + fi + + - name: Stage artifacts to SVN + run: | + TARBALL_NAME="${{ needs.create-rc.outputs.tarball_name }}" + RC_DIR="${{ needs.create-rc.outputs.rc_dir }}" + + cd svn-texera/"$RC_DIR" + + # Copy artifacts + cp "$GITHUB_WORKSPACE/$TARBALL_NAME" . + cp "$GITHUB_WORKSPACE/${TARBALL_NAME}.asc" . + cp "$GITHUB_WORKSPACE/${TARBALL_NAME}.sha512" . + + # Add files to SVN + svn add "$TARBALL_NAME" "${TARBALL_NAME}.asc" "${TARBALL_NAME}.sha512" --force + + # Check status + svn status + + echo "✓ Staged artifacts to SVN" + + - name: Commit artifacts to dist/dev + run: | + VERSION="${{ needs.create-rc.outputs.version }}" + RC_NUM="${{ needs.create-rc.outputs.rc_num }}" + RC_DIR="${{ needs.create-rc.outputs.rc_dir }}" + + cd svn-texera + + # Commit with descriptive message + svn commit -m "Add Apache Texera ${VERSION} RC${RC_NUM} artifacts" \ + --username "${{ secrets.SVN_USERNAME }}" \ + --password "${{ secrets.SVN_PASSWORD }}" \ + --no-auth-cache + + echo "✓ Committed artifacts to dist/dev/incubator/texera/$RC_DIR" + + - name: Generate release summary + run: | + VERSION="${{ needs.create-rc.outputs.version }}" + RC_NUM="${{ needs.create-rc.outputs.rc_num }}" + TAG_NAME="${{ needs.create-rc.outputs.tag_name }}" + RC_DIR="${{ needs.create-rc.outputs.rc_dir }}" + TARBALL_NAME="${{ needs.create-rc.outputs.tarball_name }}" + + echo "## Release Candidate Created Successfully! 🎉" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Release Information" >> $GITHUB_STEP_SUMMARY + echo "- **Version:** ${VERSION}" >> $GITHUB_STEP_SUMMARY + echo "- **RC Number:** RC${RC_NUM}" >> $GITHUB_STEP_SUMMARY + echo "- **Git Tag:** \`${TAG_NAME}\`" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Artifacts Location" >> $GITHUB_STEP_SUMMARY + echo "📦 **Staging Directory:** https://dist.apache.org/repos/dist/dev/incubator/texera/${RC_DIR}/" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Artifacts Created" >> $GITHUB_STEP_SUMMARY + echo "- \`${TARBALL_NAME}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`${TARBALL_NAME}.asc\` (GPG signature)" >> $GITHUB_STEP_SUMMARY + echo "- \`${TARBALL_NAME}.sha512\` (SHA512 checksum)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Next Steps" >> $GITHUB_STEP_SUMMARY + echo "1. Verify the artifacts at the staging directory" >> $GITHUB_STEP_SUMMARY + echo "2. Send [VOTE] email to dev@texera.apache.org" >> $GITHUB_STEP_SUMMARY + echo "3. Include links to:" >> $GITHUB_STEP_SUMMARY + echo " - Tag: https://github.com/${{ github.repository }}/releases/tag/${TAG_NAME}" >> $GITHUB_STEP_SUMMARY + echo " - Artifacts: https://dist.apache.org/repos/dist/dev/incubator/texera/${RC_DIR}/" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Verification Commands" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY + echo "# Download and verify signature" >> $GITHUB_STEP_SUMMARY + echo "wget https://dist.apache.org/repos/dist/dev/incubator/texera/${RC_DIR}/${TARBALL_NAME}" >> $GITHUB_STEP_SUMMARY + echo "wget https://dist.apache.org/repos/dist/dev/incubator/texera/${RC_DIR}/${TARBALL_NAME}.asc" >> $GITHUB_STEP_SUMMARY + echo "wget https://dist.apache.org/repos/dist/release/incubator/texera/KEYS" >> $GITHUB_STEP_SUMMARY + echo "gpg --import KEYS" >> $GITHUB_STEP_SUMMARY + echo "gpg --verify ${TARBALL_NAME}.asc ${TARBALL_NAME}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "# Verify SHA512" >> $GITHUB_STEP_SUMMARY + echo "wget https://dist.apache.org/repos/dist/dev/incubator/texera/${RC_DIR}/${TARBALL_NAME}.sha512" >> $GITHUB_STEP_SUMMARY + echo "sha512sum -c ${TARBALL_NAME}.sha512" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + + echo "✓ Release candidate workflow completed successfully!" + + - name: Display vote email template + run: | + echo "## 📧 Vote Email Template" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "Copy the content below to send to dev@texera.apache.org:" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + cat "$GITHUB_WORKSPACE/vote-email.txt" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/github-action-build.yml b/.github/workflows/github-action-build.yml index 14df23456a4..4c79fdce800 100644 --- a/.github/workflows/github-action-build.yml +++ b/.github/workflows/github-action-build.yml @@ -17,6 +17,9 @@ name: Build +env: + NX_CLOUD_ACCESS_TOKEN: ${{ secrets.NX_CLOUD_ACCESS_TOKEN }} + on: push: branches: @@ -30,7 +33,7 @@ concurrency: cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: - gui: + frontend: runs-on: ${{ matrix.os }} strategy: matrix: @@ -55,22 +58,22 @@ jobs: - uses: actions/cache@v4 with: path: | - core/gui/node_modules + frontend/node_modules key: ${{ runner.os }}-${{ matrix.arch }}-${{ matrix.node-version }}-yarn-cache-v1-${{ hashFiles('**/yarn.lock') }} restore-keys: | ${{ runner.os }}-${{ matrix.arch }}-${{ matrix.node-version }}-yarn-cache-v1- - name: Prepare Yarn 4.5.1 - run: corepack enable && corepack prepare yarn@4.5.1 --activate && yarn --cwd core/gui set version 4.5.1 + run: corepack enable && corepack prepare yarn@4.5.1 --activate && yarn --cwd frontend set version 4.5.1 - name: Install dependency - run: yarn --cwd core/gui install --immutable --network-timeout=100000 + run: yarn --cwd frontend install --immutable --network-timeout=100000 - name: Lint with Prettier & ESLint - run: yarn --cwd core/gui format:ci + run: yarn --cwd frontend format:ci - name: Run frontend unit tests - run: yarn --cwd core/gui run test:ci + run: yarn --cwd frontend run test:ci - name: Prod build - run: yarn --cwd core/gui run build:ci + run: yarn --cwd frontend run build:ci - core: + scala: strategy: matrix: os: [ ubuntu-22.04 ] @@ -102,26 +105,33 @@ jobs: distribution: 'temurin' java-version: 11 - name: Setup sbt launcher - uses: sbt/setup-sbt@v1 - - uses: coursier/cache-action@v6 + uses: sbt/setup-sbt@3e125ece5c3e5248e18da9ed8d2cce3d335ec8dd # v1.1.14 + - uses: coursier/cache-action@4e2615869d13561d626ed48655e1a39e5b192b3c # v6.4.9 with: - extraSbtFiles: '["core/*.sbt", "core/project/**.{scala,sbt}", "core/project/build.properties" ]' + extraSbtFiles: '["*.sbt", "project/**.{scala,sbt}", "project/build.properties" ]' - name: Lint with scalafmt - run: cd core && sbt scalafmtCheckAll - - name: Create Database - run: psql -h localhost -U postgres -f deployment/k8s/texera-helmchart/files/texera_ddl.sql + run: sbt scalafmtCheckAll + - name: Create Databases + run: | + psql -h localhost -U postgres -f sql/texera_ddl.sql + psql -h localhost -U postgres -f sql/iceberg_postgres_catalog.sql + psql -h localhost -U postgres -f sql/texera_lakefs.sql + env: + PGPASSWORD: postgres + - name: Create texera_db_for_test_cases + run: psql -h localhost -U postgres -v DB_NAME=texera_db_for_test_cases -f sql/texera_ddl.sql env: PGPASSWORD: postgres - name: Compile with sbt - run: cd core && sbt clean package + run: sbt clean package - name: Run backend tests - run: cd core && sbt test + run: sbt test - python_udf: + python: strategy: matrix: os: [ ubuntu-latest ] - python-version: [ '3.9', '3.10', '3.11', '3.12' ] + python-version: [ '3.10', '3.11', '3.12' ] runs-on: ${{ matrix.os }} steps: - name: Checkout Texera @@ -133,18 +143,18 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - if [ -f core/amber/requirements.txt ]; then pip install -r core/amber/requirements.txt; fi - if [ -f core/amber/operator-requirements.txt ]; then pip install -r core/amber/operator-requirements.txt; fi + if [ -f amber/requirements.txt ]; then pip install -r amber/requirements.txt; fi + if [ -f amber/operator-requirements.txt ]; then pip install -r amber/operator-requirements.txt; fi - name: Install PostgreSQL run: sudo apt-get update && sudo apt-get install -y postgresql - name: Start PostgreSQL Service run: sudo systemctl start postgresql - name: Create Database and User run: | - cd core/scripts/sql && sudo -u postgres psql -f iceberg_postgres_catalog.sql - - name: Lint with flake8 and black + cd sql && sudo -u postgres psql -f iceberg_postgres_catalog.sql + - name: Lint with Ruff run: | - cd core/amber/src/main/python && flake8 && black . --check + cd amber/src/main/python && ruff check . && ruff format --check . - name: Test with pytest run: | - cd core/amber/src/main/python && pytest -sv --ignore=core/models/test_RTableExecutor.py + cd amber/src/main/python && pytest -sv diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml new file mode 100644 index 00000000000..0846b98ebe8 --- /dev/null +++ b/.github/workflows/issue-triage.yml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Issue triage +on: + issues: + types: [assigned, unassigned] + +permissions: + issues: write + pull-requests: read + +jobs: + # -------------------------------------------------------- + # 1) Issue triage: add/remove "triage" on assignment changes + # -------------------------------------------------------- + issue-triage: + if: github.event_name == 'issues' + runs-on: ubuntu-latest + steps: + - name: Remove 'triage' label when issue is assigned + if: github.event.action == 'assigned' + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { owner, repo } = context.repo; + const issue_number = context.payload.issue.number; + + try { + await github.rest.issues.removeLabel({ + owner, + repo, + issue_number, + name: 'triage', + }); + core.info(`Removed 'triage' from issue #${issue_number}`); + } catch (e) { + if (e.status === 404) { + core.info(`Issue #${issue_number} has no 'triage' label, nothing to remove.`); + } else { + core.warning(`Failed to remove 'triage' from issue #${issue_number}: ${e.message}`); + throw e; + } + } + + - name: Add 'triage' label when issue is unassigned and has no assignees + if: github.event.action == 'unassigned' + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const issue = context.payload.issue; + const { owner, repo } = context.repo; + const issue_number = issue.number; + + const assignees = issue.assignees || []; + if (assignees.length > 0) { + core.info( + `Issue #${issue_number} still has ${assignees.length} assignee(s), not adding 'triage'.` + ); + return; + } + + core.info(`Issue #${issue_number} has no assignees, adding 'triage' label.`); + try { + await github.rest.issues.addLabels({ + owner, + repo, + issue_number, + labels: ['triage'], + }); + } catch (e) { + core.warning(`Failed to add 'triage' to issue #${issue_number}: ${e.message}`); + throw e; + } diff --git a/.github/workflows/lint-pr.yml b/.github/workflows/lint-pr.yml index f4151a19047..fcc5d46f0ca 100644 --- a/.github/workflows/lint-pr.yml +++ b/.github/workflows/lint-pr.yml @@ -31,6 +31,6 @@ jobs: permissions: pull-requests: read steps: - - uses: amannn/action-semantic-pull-request@v5 + - uses: amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017 # v5.5.3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 627c56f0874..62f85f0e2f0 100644 --- a/.gitignore +++ b/.gitignore @@ -87,7 +87,7 @@ rebel.xml *.log.gz # Ignoring the entire log folder -log/ +logs/ # Ignoring package-lock.json package-lock.json diff --git a/.licenserc.yaml b/.licenserc.yaml index 309f9d844fc..fed8faf136e 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -16,6 +16,7 @@ header: - 'NOTICE' - '.dockerignore' - '.gitattributes' + - '.github/PULL_REQUEST_TEMPLATE' - 'yarn.lock' - '.nvmrc' - '.htaccess' @@ -23,4 +24,5 @@ header: - 'site.webmanifest' - '.gitignore' - '.licenserc.yaml' - - 'core/gui/.yarn/**' + - 'frontend/.yarn/**' + - 'amber/src/main/python/proto/**' diff --git a/.run/AccessControlService.run.xml b/.run/AccessControlService.run.xml new file mode 100644 index 00000000000..56ffbc50fbb --- /dev/null +++ b/.run/AccessControlService.run.xml @@ -0,0 +1,34 @@ + + + + + \ No newline at end of file diff --git a/.run/ComputingUnitManagingService.run.xml b/.run/ComputingUnitManagingService.run.xml new file mode 100644 index 00000000000..8bbb5c1966f --- /dev/null +++ b/.run/ComputingUnitManagingService.run.xml @@ -0,0 +1,34 @@ + + + + + \ No newline at end of file diff --git a/.run/ComputingUnitMaster.run.xml b/.run/ComputingUnitMaster.run.xml new file mode 100644 index 00000000000..5ffffb79e88 --- /dev/null +++ b/.run/ComputingUnitMaster.run.xml @@ -0,0 +1,34 @@ + + + + + \ No newline at end of file diff --git a/.run/ComputingUnitWorker.run.xml b/.run/ComputingUnitWorker.run.xml new file mode 100644 index 00000000000..854da672287 --- /dev/null +++ b/.run/ComputingUnitWorker.run.xml @@ -0,0 +1,34 @@ + + + + + \ No newline at end of file diff --git a/.run/ConfigService.run.xml b/.run/ConfigService.run.xml new file mode 100644 index 00000000000..395688afca4 --- /dev/null +++ b/.run/ConfigService.run.xml @@ -0,0 +1,34 @@ + + + + + \ No newline at end of file diff --git a/.run/FileService.run.xml b/.run/FileService.run.xml new file mode 100644 index 00000000000..0a54e70aac8 --- /dev/null +++ b/.run/FileService.run.xml @@ -0,0 +1,34 @@ + + + + + \ No newline at end of file diff --git a/.run/TexeraWebApplication.run.xml b/.run/TexeraWebApplication.run.xml new file mode 100644 index 00000000000..d9e3829fc22 --- /dev/null +++ b/.run/TexeraWebApplication.run.xml @@ -0,0 +1,34 @@ + + + + + \ No newline at end of file diff --git a/.run/WorkflowCompilingService.run.xml b/.run/WorkflowCompilingService.run.xml new file mode 100644 index 00000000000..d41f60d9c54 --- /dev/null +++ b/.run/WorkflowCompilingService.run.xml @@ -0,0 +1,34 @@ + + + + + \ No newline at end of file diff --git a/.run/frontend.run.xml b/.run/frontend.run.xml new file mode 100644 index 00000000000..8132f576027 --- /dev/null +++ b/.run/frontend.run.xml @@ -0,0 +1,30 @@ + + + + + + +